musées mapping, fix key_converted and convert_to_phone bug

convert_to_osm_tags.ts

@@ -15,15 +15,18 @@ import utils from './mappings/utils'
 import mappingRouenParkingVelos from "./mappings/converters/configRouen_OpenData_velo_parkings";
 import mappingFINESS from "./mappings/converters/configFINESS";
 import MappingArbresIssy from "./mappings/converters/configArbresIssy";
+import MappingArbres92 from "./mappings/converters/configArbresHautsDeSeine";
+import MappingMuseums from "./mappings/converters/configMuseums";
 
 const limitWarningPercentageChangeInPoints = 5; // show a warning when more than N percent of the number of points changed
 const allowed_configs = [
     'mappingIssy2Roues', 'mappingConfigIRVE', 'mappingConfigIRVEFromOsmose',
     'mappingConfigIRVE_simple', 'mappingTest', 'ConfigIRVE',
     'mappingRouenParkingVelos', 'mappingFINESS', 'MappingArbresIssy',
+    'MappingArbres92', 'MappingMuseums'
 ];
 
-const minimist = require('minimist')
+import minimist = require('minimist')
 
 const debugLog = utils.debugLog;
 
@@ -387,6 +390,10 @@ function setMappingConfigFromName(engine_conf_choice: string) {
                     Mapping_engine.setConfig(MappingArbresIssy)
                 } else if (engine_conf_choice == 'mappingFINESS') {
                     Mapping_engine.setConfig(mappingFINESS)
+                } else if (engine_conf_choice == 'MappingArbres92') {
+                    Mapping_engine.setConfig(MappingArbres92)
+                } else if (engine_conf_choice == 'MappingMuseums') {
+                    Mapping_engine.setConfig(MappingMuseums)
                 }
             } else {
                 Mapping_engine.setConfig(mappingConfigIRVE)

docs/arbres.md

@@ -14,7 +14,14 @@ Des jeux de données sont disponibles pour lister les abres, suivre leur évolut
 - FORME: tree_shape = curtain / free / half_free  <= Architecturé, rideau / Libre / Semi-libre
 
 # conversion
-
+Abres d'issy les moulineaux
 ```bash 
 ts-node convert_to_osm_tags.ts --source=etalab_data/arbres/arbres_issy.geojson --output-file=arbres_issy.geojson --engine-config=MappingArbresIssy
 ```
+
+Cadastre vert:
+
+```bash 
+ts-node convert_to_osm_t
+ags.ts --source=etalab_data/arbres/arbres_92.json --output-file=arbres_92.geojson --engine-config=MappingArbres92
+```

docs/musées.md

@@ -0,0 +1,7 @@
+# musées
+
+Jeu de données des musées en île de france.
+# conversion
+```bash 
+ts-node convert_to_osm_tags.ts --source=etalab_data/arbres/musées/fr.geojson --output-file=musées_idf.geojson --engine-config=MappingMuseums
+```

etalab_data/arbres/arbres_92_small.json

@@ -0,0 +1,71 @@
+{
+  "type": "FeatureCollection",
+  "features": [
+    {
+      "type": "Feature",
+      "geometry": {
+        "coordinates": [
+          2.169019488063191,
+          48.863711749353044
+        ],
+        "type": "Point"
+      },
+      "properties": {
+        "idelement_vert": 301393,
+        "commune": "RUEIL MALMAISON",
+        "code_insee": "92063",
+        "diametre": 3.208899,
+        "circonference": 10.075946,
+        "type": "I",
+        "geo_point_2d": {
+          "lon": 2.169019488063191,
+          "lat": 48.863711749353044
+        }
+      }
+    },
+    {
+      "type": "Feature",
+      "geometry": {
+        "coordinates": [
+          2.1809078969328555,
+          48.86379786822936
+        ],
+        "type": "Point"
+      },
+      "properties": {
+        "idelement_vert": 301396,
+        "commune": "RUEIL MALMAISON",
+        "code_insee": "92063",
+        "diametre": 0.0,
+        "circonference": 0.0,
+        "type": "A",
+        "geo_point_2d": {
+          "lon": 2.1809078969328555,
+          "lat": 48.86379786822936
+        }
+      }
+    },
+    {
+      "type": "Feature",
+      "geometry": {
+        "coordinates": [
+          2.197033918443427,
+          48.863914840527364
+        ],
+        "type": "Point"
+      },
+      "properties": {
+        "idelement_vert": 301408,
+        "commune": "RUEIL MALMAISON",
+        "code_insee": "92063",
+        "diametre": 3.677071,
+        "circonference": 11.54600608,
+        "type": "A",
+        "geo_point_2d": {
+          "lon": 2.197033918443427,
+          "lat": 48.863914840527364
+        }
+      }
+    }
+  ]
+}

mappings/converters/configArbresHautsDeSeine.ts

@@ -0,0 +1,51 @@
+import MappingConfigType from "../mapping-config.type";
+
+const MappingArbres92: MappingConfigType = {
+    config_name: 'Mapping des arbres d\'Issy les Moulineaux',
+    config_author: 'tykayn <contact+geojson2osm@cipherbliss.com>',
+    default_properties_of_point: {natural: 'tree'},
+    tags: {
+        // ******* booléens
+        // ******* nombres
+        //         - ID_ARBRE : ref:FR:issy_les_mx:id_arbre identifiant interne à Issy les Moulineaux
+        "idelement_vert": "ref:FR:cadastre_vert:id_element_vert",
+        "type": "ref:FR:cadastre_vert:type",
+        // ******* textes
+        // ******* textes
+
+// - HAUTEUR est_height=* hauteur estimée en intervalle de mètres
+        //"class_hauteur": "est_height",
+        // - VERNACULAIRE : species:FR=*
+        //"vernaculaire": "species:FR",
+        //  https://wiki.openstreetmap.org/wiki/Key:circumference
+        "diametre": "circumference",// tour du tronc à hauteur de poitrine
+        // "diametre": {// tour du tronc à hauteur de poitrine
+        //     key_converted: "circumference",
+        //     conditionnal_values: {
+        //         "0 m": {
+        //             ignore_this_data: true,
+        //         }
+        //     }
+        // },
+        "circonference": "diameter_crown",// tour de toutes les branches, la couronne
+        // "circonference": {// tour de toutes les branches, la couronne
+        //     key_converted: "diameter_crown",
+        //     conditionnal_values: {
+        //         "0 m": {
+        //             ignore_this_data: true,
+        //         }
+        //     }
+        //
+        // },
+
+
+
+    },
+    add_not_mapped_tags_too: false,
+    source: {
+        geojson_path: 'https://opendata.hauts-de-seine.fr/api/explore/v2.1/catalog/datasets/cadastre-vert-les-arbres/exports/geojson?lang=fr&timezone=Europe%2FBerlin',
+        url: 'https://opendata.hauts-de-seine.fr/explore/dataset/cadastre-vert-les-arbres'
+    },
+    filters: {},
+}
+export default MappingArbres92

mappings/converters/configArbresIssy.ts

@@ -96,6 +96,8 @@ const MappingArbresIssy: MappingConfigType = {
         geojson_path: 'https://data.issy.com/api/explore/v2.1/catalog/datasets/arbres-remarquables-issy-les-moulineaux/exports/geojson?lang=fr&timezone=Europe%2FBerlin',
         url: 'https://www.data.gouv.fr/fr/datasets/arbres-remarquables'
     },
-    filters: {},
+    filters: {
+        offset:10
+    },
 }
 export default MappingArbresIssy

mappings/converters/configMuseums.ts

@@ -0,0 +1,32 @@
+import MappingConfigType from "../mapping-config.type";
+
+const MappingMuseums: MappingConfigType = {
+    config_name: 'Mapping des musées en Île de France',
+    config_author: 'tykayn <contact+geojson2osm@cipherbliss.com>',
+    default_properties_of_point: {tourism: 'museum'},
+    tags: {
+        // ******* booléens
+        // ******* nombres
+        "URL": "contact:website",
+        "Identifiant Muséofile": "ref:FR:museums:museofile",
+        "REF_Deps": "ref:FR:museums:deps",
+        // ******* textes
+        'Téléphone': {
+            key_converted: "contact:phone",
+            convert_to_phone: true,
+        },
+        "Nom officiel du musée": {
+            key_converted: "name",
+            convert_to_name: true,
+        }
+    },
+    add_not_mapped_tags_too: false,
+    source: {
+        geojson_path: '',
+        url: ''
+    },
+    filters: {
+        // offset:2 // uncomment to run small tests
+    },
+}
+export default MappingMuseums

mappings/engine.ts

@@ -41,7 +41,7 @@ export default class {
     }
 
     setConfig(mappingConfig: MappingConfigType) {
-        console.log('load config', mappingConfig.config_name)
+        debugLog('load config', mappingConfig.config_name)
         debugLog('load config', mappingConfig.config_name)
         this.mapping_config = mappingConfig
     }
@@ -108,15 +108,15 @@ export default class {
      * @param list
      * @param excludedKeys
      */
-    filterListOfPointsByExcludingIfKeyFilled(list: any, excludedKeys: Array<string>):any[] {
+    filterListOfPointsByExcludingIfKeyFilled(list: any, excludedKeys: Array<string>): any[] {
         let newList: Array<any> = []
         list.forEach((geojsonPoint: any) => {
             let pointProperties = Object.keys(geojsonPoint.properties)
             let addPoint = true;
             excludedKeys.forEach((key: any) => {
                 debugLog(key, 'pointProperties[key]', pointProperties[key])
-                let foundProperty:string = pointProperties[key]
+                let foundProperty: string = pointProperties[key]
-                if ( foundProperty && foundProperty !== 'null') {
+                if (foundProperty && foundProperty !== 'null') {
                     addPoint = false
                 }
             })
@@ -213,9 +213,11 @@ export default class {
         // debugLog('convertProperty: mappingKeys', mappingKeys)
 
         let remove_original_key = false;
-        if (this.mapping_config.tags_to_ignore_if_value_is?.indexOf(originalValue) !== -1) {
+        debugLog('tags_to_ignore_if_value_is', this.mapping_config.tags_to_ignore_if_value_is)
+        if (this.mapping_config.tags_to_ignore_if_value_is && this.mapping_config.tags_to_ignore_if_value_is.length && this.mapping_config.tags_to_ignore_if_value_is?.indexOf(originalValue) !== -1) {
             debugLog('(x) => ignore', originalValue, ' in ', pointKeyName)
             remove_original_key = true;
+
         }
 
 
@@ -293,6 +295,7 @@ export default class {
 
                     if (configObject.key_converted) {
                         newKey = configObject.key_converted
+                        debugLog('key_converted newKey', newKey)
                     }
                     if (configObject.transform_function) {
                         convertedValue = configObject.transform_function(originalValue)
@@ -414,7 +417,6 @@ export default class {
                                 }
                             } else {
                                 debugLog('no sockets', this.current_geojson_point.properties.ref)
-                                // console.log('sockets', countOfSockets, this.current_geojson_point.properties)
                             }
                         }
                         return out
@@ -434,8 +436,8 @@ export default class {
                             this.stats.phones_updated_list.push(convertedValue)
                         } else {
                             this.stats.phones_not_updated++
-
                         }
+                        debugLog('convertedValue convert_to_phone', originalValue ,'=>', convertedValue)
 
                     }
                     if (configObject.convert_to_name) {
@@ -457,9 +459,8 @@ export default class {
                     if (configObject.conditional_values) {
 
 
-
                         // convert numbers from json to string to compare them correctly
-                        originalValue = ''+originalValue
+                        originalValue = '' + originalValue
 
                         let keysConditionnalValues: any = Object.keys(configObject.conditional_values)
                         let isFoundValue = keysConditionnalValues.indexOf(originalValue)
@@ -477,7 +478,7 @@ export default class {
 
 
                             if (isFoundValue !== -1) {
-                                debugLog('found condition',isFoundValue )
+                                debugLog('found condition', isFoundValue)
 
                                 /** ----------------------
                                  *  gestion des valeurs conditionnelles
@@ -518,7 +519,7 @@ export default class {
                             // on peut définir un ensemble de tags à rajouter
                             let tagKeys = Object.keys(conditionnalConfig.tags_to_add)
                             debugLog('conditionnalConfig.tags_to_add', conditionnalConfig.tags_to_add)
-                            tagKeys.forEach((index:any) => {
+                            tagKeys.forEach((index: any) => {
                                 debugLog('key', index)
                                 debugLog('value', conditionnalConfig.tags_to_add[index])
                                 newProperties[index] = conditionnalConfig.tags_to_add[index]
@@ -530,7 +531,19 @@ export default class {
 
                     debugLog('convertProperty: convertedValue ==========> {', newKey, ':', convertedValue, '}')
                     debugLog(' =============== remove_original_key', newKey, remove_original_key)
-                    if (!remove_original_key && newKey && convertedValue && !configObject.ignore_this_data) {
+
+                    let keysOfConfigObject = [];
+                    let hasKeyIgnoreThisData = false;
+                    if (configObject) {
+                        keysOfConfigObject = Object.keys(configObject)
+                        debugLog('keysOfConfigObject', keysOfConfigObject)
+                        hasKeyIgnoreThisData = (keysOfConfigObject.indexOf('ignore_this_data') !== -1)
+                    }
+
+                    debugLog('remove_original_key && newKey && convertedValue && hasKeyIgnoreThisData', remove_original_key , newKey , convertedValue ,hasKeyIgnoreThisData)
+                    if (!remove_original_key && newKey && convertedValue && !hasKeyIgnoreThisData
+                    ) {
+                        debugLog('convertedValue', convertedValue)
 
                         debugLog('convertProperty: added', newKey, (`${convertedValue}`).trim())
                         newProperties[newKey] = (`${convertedValue}`).trim()

mappings/formatters.ts

@@ -1,21 +1,22 @@
 import custom_utils from "./utils";
 
-const {debugLog} = custom_utils
+const {debugLog,prefix_phone_fr_only} = custom_utils
 
 /**
  * Class that helps to convert values into predefined formats
  */
 export default class Formatters {
 
-    static convertToPhone(originalValue: string) {
+    static convertToPhone(originalValue: string) :string{
         /**
          * nettoyer les numéros de téléphone en ne gardant que les nombres et le préfixe de pays
          */
+        debugLog("convertToPhone:" , originalValue);
         // debugLog('originalValue', originalValue.substring(1))
         if (!originalValue) {
             originalValue = ''
         }
-        let original_without_spaces = originalValue.replace(/ /g, '')
+        let original_without_spaces = originalValue.replace(' ', '')
         let cleaned_value = `${original_without_spaces}`
         cleaned_value = cleaned_value
             .trim()
@@ -31,6 +32,8 @@ export default class Formatters {
         }
         cleaned_value = cleaned_value.replace('+33', '')
 
+        debugLog("convertToPhone: cleaned_value" , cleaned_value);
+
         if (/^0/.test(cleaned_value)) {
             cleaned_value = cleaned_value.substring(1)
         }
@@ -60,14 +63,14 @@ export default class Formatters {
         ) {
             add_prefix = true
         }
-        if (add_prefix) {
+        if (add_prefix && prefix_phone_fr_only) {
             convertedValue = `+33 ` + convertedValue
         }
 
         debugLog('phone: ', originalValue, '=>', convertedValue)
 
 
-        return "";
+        return ""+convertedValue;
     }
 
     static convertToName(originalValue: string) {

mappings/utils.ts

@@ -3,6 +3,7 @@ import * as fs from 'node:fs'
 let show_debug = 0
 // show_debug = 1
 let output_folder = 'output';
+const prefix_phone_fr_only = true
 
 // console.log('----------------------show_debug', show_debug)
 
@@ -72,5 +73,6 @@ function writeFile(fileName: string, fileContent: any, outputPathOverride: strin
 export default {
     debugLog,
     isBooleanKey,
-    writeFile
+    writeFile,
+    prefix_phone_fr_only
 }