⚡ change file output name, fix tags to add conditionnal handling

convert_to_osm_tags.ts

@@ -20,19 +20,13 @@ const minimist = require('minimist')
 
 const debugLog = utils.debugLog;
 
-let use_mapping_engine = true
-
 let mini_arguments: any = minimist(process.argv.slice(2))
-
+let use_mapping_engine = true
-
 let output_supplement = '';
-
-
 let sourceFilePathGeoJson = './data_other/' + output_supplement + '.json'
 // wip filter
 let filterOnBoundingBox = true
 filterOnBoundingBox = false
-
 let boundingBoxCoordinates: BoundingBoxCoordinatesType = {
     xMin: 1.91,
     xMax: 2.38,
@@ -41,7 +35,6 @@ let boundingBoxCoordinates: BoundingBoxCoordinatesType = {
 }
 let default_engine_conf_choice: string = 'ConfigIRVE';
 let engine_conf_choice: string = 'ConfigIRVE';
-
 let filterCoordinates = true
 filterCoordinates = false
 
@@ -70,7 +63,7 @@ if (mini_arguments['engine-config']) {
     output_supplement += "_" + engine_conf_choice
 }
 let Mapping_engine: any;
-let outname = 'my_data_set_'
+let outname = 'converted_'
 
 if (mini_arguments['outname']) {
     outname = mini_arguments['outname'];
@@ -278,7 +271,7 @@ function convertDataFromSource(sourceFilePath: string, mapping: MappingConfigTyp
                 if (output_supplement) {
                     output_supplement = '_' + output_supplement;
                 }
-                let fileNameToWrite = 'my_converted_data_set' + filteredName + output_supplement + '.json'
+                let fileNameToWrite = '_' + filteredName + output_supplement + '.json'
                 console.log('converted features:', converted_geo_json.features.length)
                 console.log('diff: ', data_transformed.features.length - converted_geo_json.features.length)
                 debugLog('convert : write file ', fileNameToWrite)
@@ -388,15 +381,13 @@ function setMappingConfigFromName(engine_conf_choice: string) {
         convertDataFromSource(sourceFilePathGeoJson, mappingConfigIRVE, pointCounterMax, boundingBoxCoordinates)
     }
 
-
 }
 
-
+/**
+ * Launch conversion of dataset
+ */
 function init() {
     setMappingConfigFromName(engine_conf_choice)
-
-
-
 }
 
 init()

mappings/converters/configRouen_OpenData_velo_parkings.ts

@@ -17,7 +17,7 @@ const mappingRouenParkingVelos: MappingConfigType = {
     },
     filters: {
         exclude_point_if_tag_not_empty: ['osm_id'],
-        offset: 1
+        // offset: 1
     },
     add_not_mapped_tags_too: false,
     boolean_keys: [
@@ -57,21 +57,21 @@ const mappingRouenParkingVelos: MappingConfigType = {
             remove_original_key: true, // TODO trouver ce qui empêche la conversion quand remove_original_key est a true
             conditional_values: {
                 "POTELET": {
-                    "tags_to_add": [
+                    tags_to_add: [
                         {
                             "bicycle_parking": "bollard"
                         },
                     ]
                 },
                 "ARCEAU": {
-                    "tags_to_add": [
+                    tags_to_add: [
                         {
                             "bicycle_parking": "stands"
                         },
                     ]
                 },
                 "RATELIER": {
-                    "tags_to_add": [
+                    tags_to_add: [
                         {
                             "bicycle_parking": "rack"
                         },

mappings/engine.ts

@@ -22,7 +22,7 @@ function boolToAddable(someBooleanValue: boolean) {
 
 export default class {
     mapping_config: any = {}
-    private jardinage = true;
+    private jardinage = false;
     public stats: any;
     private current_converted_geojson_point: any;
     private current_geojson_point: any;
@@ -140,7 +140,7 @@ export default class {
             debugLog('mapElementFromConf: convert', pointKeyName)
             debugLog('mapElementFromConf: mapping keys:', mappingKeys)
 
-                this.convertProperty(pointKeyName, mappingKeys, featurePoint, newProperties)
+            this.convertProperty(pointKeyName, mappingKeys, featurePoint, newProperties)
 
 
         })
@@ -182,6 +182,7 @@ export default class {
 
         let remove_original_key = false;
         if (this.mapping_config.tags_to_ignore_if_value_is.indexOf(originalValue) !== -1) {
+            debugLog('(x) => ignore', originalValue, ' in ', pointKeyName)
             remove_original_key = true;
         }
 
@@ -204,6 +205,7 @@ export default class {
             /**
              * only use existing keys
              */
+            debugLog("only use existing keys,", pointKeyName)
             if (mappingKeys.indexOf(pointKeyName) !== -1) {
                 let valueConvertedFromMapping = featurePoint.properties[pointKeyName]
                 let keyConvertedFromMapping = mappingKeys[mappingKeys.indexOf(pointKeyName)]
@@ -394,7 +396,7 @@ export default class {
 
 
                     if (configObject.convert_to_phone) {
-                       convertedValue = Formatters.convertToPhone(originalValue)
+                        convertedValue = Formatters.convertToPhone(originalValue)
                         if (originalValue !== convertedValue) {
                             this.stats.phones_updated++
                             this.stats.phones_updated_list.push(convertedValue)
@@ -417,73 +419,82 @@ export default class {
                      * config pour une clé
                      * nous pouvons renseigner une string ou un objet décrivant les transformations à réaliser
                      */
-                    if (!remove_original_key && configObject.conditional_values) {
+                    if (configObject.conditional_values) {
+
+                        let keysConditionnalValues: any = Object.keys(configObject.conditional_values)
+                        let isFoundValue = keysConditionnalValues.indexOf(originalValue)
+                        let conditionnalConfig: any = configObject.conditional_values[keysConditionnalValues[isFoundValue]]
                         debugLog('convertProperty: conditional_values__________',
                             configObject.conditional_values)
 
-                        let keysConditionnalValues: any = Object.keys(configObject.conditional_values)
-
-                        let isFoundValue = keysConditionnalValues.indexOf(originalValue)
                         debugLog('isFoundValue', isFoundValue, originalValue)
                         debugLog('keysConditionnalValues', keysConditionnalValues)
 
-
+                        if (!remove_original_key) {
-                        if (isFoundValue > -1) {
-                            let conditionnalConfig: any = configObject.conditional_values[keysConditionnalValues[isFoundValue]]
-                            /** ----------------------
-                             *  gestion des valeurs conditionnelles
-                             *  ---------------------- */
-                            debugLog('conditionnalConfig', conditionnalConfig)
 
 
-                            if (conditionnalConfig.ignore_this_data) {
+                            if (isFoundValue > -1) {
-                                debugLog(`on ignore cette clé car sa valeur "${originalValue}" est à exclure: `, pointKeyName, '=>', newKey)
-                                remove_original_key = true;
-                            }
-                            if (conditionnalConfig.tags_to_add) {
-                                // on peut définir un ensemble de tags à rajouter
-                                let tagKeys = Object.keys(conditionnalConfig.tags_to_add)
-                                debugLog('conditionnalConfig.tags_to_add', conditionnalConfig.tags_to_add)
-                                conditionnalConfig.tags_to_add.forEach((object: any, pair: any) => {
-                                    debugLog('object', object)
-                                    debugLog('pair', pair)
-                                    let key: any = Object.keys(object)
-                                    key = key[0]
-                                    let value = object[key]
 
-                                    debugLog('key', key)
+                                /** ----------------------
-                                    debugLog('value', value)
+                                 *  gestion des valeurs conditionnelles
-                                    newProperties[key] = value
+                                 *  ---------------------- */
-                                })
+                                debugLog('conditionnalConfig', conditionnalConfig)
-                            }
+
-                            if (conditionnalConfig.truthy_value) {
+
-                                // convertir la valeur, si elle est truthy, la transformer en ce que donne la propriété truthy_value
+                                if (conditionnalConfig.ignore_this_data) {
-                                // exemple: le jeu de données dit que la colonne cable_t2_attache vaut "True", mais on veut le convertir en "1".
+                                    debugLog(`on ignore cette clé car sa valeur "${originalValue}" est à exclure: `, pointKeyName, '=>', newKey)
-                                // on met donc truthy_value: '1'
+                                    remove_original_key = true;
-                                if (this.truthyValues.indexOf(originalValue) !== -1) {
-                                    convertedValue = conditionnalConfig.truthy_value
                                 }
-                            }
+
-                            if (conditionnalConfig.falsy_value) {
+                                if (conditionnalConfig.truthy_value) {
-                                if (this.falsyValues.indexOf(originalValue) !== -1) {
+                                    // convertir la valeur, si elle est truthy, la transformer en ce que donne la propriété truthy_value
-                                    convertedValue = conditionnalConfig.falsy_value
+                                    // exemple: le jeu de données dit que la colonne cable_t2_attache vaut "True", mais on veut le convertir en "1".
+                                    // on met donc truthy_value: '1'
+                                    if (this.truthyValues.indexOf(originalValue) !== -1) {
+                                        convertedValue = conditionnalConfig.truthy_value
+                                    }
                                 }
+                                if (conditionnalConfig.falsy_value) {
+                                    if (this.falsyValues.indexOf(originalValue) !== -1) {
+                                        convertedValue = conditionnalConfig.falsy_value
+                                    }
+                                }
+
+                                // use the value converted
+                                else if (conditionnalConfig.value_converted) {
+                                    convertedValue = conditionnalConfig.value_converted
+                                }
+
+
                             }
 
-                            // use the value converted
+                        }
-                            else if (conditionnalConfig.value_converted) {
+                        if (conditionnalConfig?.tags_to_add) {
-                                convertedValue = conditionnalConfig.value_converted
+                            debugLog('on ajoute des tags', conditionnalConfig.tags_to_add)
-                            }
+                            // on peut définir un ensemble de tags à rajouter
-
+                            let tagKeys = Object.keys(conditionnalConfig.tags_to_add)
+                            debugLog('conditionnalConfig.tags_to_add', conditionnalConfig.tags_to_add)
+                            conditionnalConfig.tags_to_add.forEach((object: any, pair: any) => {
+                                debugLog('object', object)
+                                debugLog('pair', pair)
+                                let key: any = Object.keys(object)
+                                key = key[0]
+                                let value = object[key]
 
+                                debugLog('key', key)
+                                debugLog('value', value)
+                                newProperties[key] = value
+                            })
                         }
 
                     }
+
+
                     debugLog('convertProperty: convertedValue ==========> {', newKey, ':', convertedValue, '}')
                     debugLog(' =============== remove_original_key', newKey, remove_original_key)
                     if (!remove_original_key && newKey && convertedValue && !configObject.ignore_this_data) {
 
-                        debugLog('convertProperty: added', newKey, convertedValue)
+                        debugLog('convertProperty: added', newKey, (`${convertedValue}`).trim())
                         newProperties[newKey] = (`${convertedValue}`).trim()
                     }
 
@@ -493,6 +504,7 @@ export default class {
             }
         }
 
+        debugLog('newProperties', newProperties)
 
         return newProperties;
     }

output/my_converted_data_set__mappingRouenParkingVelos.json

@@ -14,6 +14,8 @@
         "amenity": "bicycle_parking",
         "ref:FR:rouen_veloparking_id": 775,
         "capacity": 8,
+        "bicycle_parking": "stands",
+        "mobilier": "ARCEAU",
         "check_date": "10/08/2021"
       }
     }

tests/data/mappings_to_test.ts

@@ -9,7 +9,10 @@ export const mappingRemoveAll: MappingConfigType = {
     default_properties_of_point: {},
     tags: {},
     add_not_mapped_tags_too: false,
-    source: {},
+    source: {
+        geojson_path: "",
+        url: ""
+    },
     filters: {},
 }
 export const mappingSame: MappingConfigType = {
@@ -17,10 +20,13 @@ export const mappingSame: MappingConfigType = {
     config_author: 'tykayn <contact@cipherbliss.com>',
     default_properties_of_point: {},
     tags: {
-        equal : 'equal'
+        equal: 'equal'
     },
     add_not_mapped_tags_too: false,
-    source: {},
+    source: {
+        geojson_path: "",
+        url: ""
+    },
     filters: {},
 }
 export const mappingName: MappingConfigType = {
@@ -29,9 +35,12 @@ export const mappingName: MappingConfigType = {
     default_properties_of_point: {},
     add_not_mapped_tags_too: false,
     tags: {
-        nom_amenageur : 'name'
+        nom_amenageur: 'name'
+    },
+    source: {
+        geojson_path: "",
+        url: ""
     },
-    source: {},
     filters: {},
 }
 export const mappingPhone: MappingConfigType = {
@@ -46,7 +55,10 @@ export const mappingPhone: MappingConfigType = {
         },
     },
     add_not_mapped_tags_too: false,
-    source: {},
+    source: {
+        geojson_path: "",
+        url: ""
+    },
     filters: {},
 }
 export const mappingBoolean: MappingConfigType = {
@@ -60,7 +72,10 @@ export const mappingBoolean: MappingConfigType = {
         },
     },
     add_not_mapped_tags_too: false,
-    source: {},
+    source: {
+        geojson_path: "",
+        url: ""
+    },
     filters: {},
 }
 export const mappingTruthy: MappingConfigType = {
@@ -74,7 +89,10 @@ export const mappingTruthy: MappingConfigType = {
         },
     },
     add_not_mapped_tags_too: false,
-    source: {},
+    source: {
+        geojson_path: "",
+        url: ""
+    },
     filters: {},
 }
 export const mappingFalsy: MappingConfigType = {
@@ -88,7 +106,10 @@ export const mappingFalsy: MappingConfigType = {
         },
     },
     add_not_mapped_tags_too: false,
-    source: {},
+    source: {
+        geojson_path: "",
+        url: ""
+    },
     filters: {},
 }
 export const mappingIgnoreFalsy: MappingConfigType = {
@@ -102,7 +123,10 @@ export const mappingIgnoreFalsy: MappingConfigType = {
         },
     },
     add_not_mapped_tags_too: false,
-    source: {},
+    source: {
+        geojson_path: "",
+        url: ""
+    },
     filters: {},
 }
 export const mappingIgnoreTruthy: MappingConfigType = {
@@ -116,6 +140,9 @@ export const mappingIgnoreTruthy: MappingConfigType = {
         },
     },
     add_not_mapped_tags_too: false,
-    source: {},
+    source: {
+        geojson_path: "",
+        url: ""
+    },
     filters: {},
 }