split engine formatter to dedicated class, add filter offset

convert_to_osm_tags.ts

@@ -168,8 +168,22 @@ function convertDataFromSource(sourceFilePath: string, mapping: MappingConfigTyp
             let list_of_points: any = data_transformed.features
             debugLog('listOfPoints.length', list_of_points.length)
 
+            if(limitConversionToFirstPoint){
+                debugLog('limitConversionToFirstPoint enabled')
+                if(mapping?.filters?.offset){
+                mapping.filters.offset = 1
+                }
+            }
+            if(mapping?.filters?.offset){
+                debugLog('filter offset enabled', mapping.filters.offset)
+                list_of_points = list_of_points.splice(0,mapping.filters.offset)
+            }
+            console.log('list_of_points.length', list_of_points.length)
+
+
             /**
              * filtering
+             * run filters before mapping OSM tags
              * TODO: do it in the engine
              */
             // for each point from the data source, filter if we take it or not

mappings/converters/configRouen_OpenData_velo_parkings.ts

@@ -16,7 +16,8 @@ const mappingRouenParkingVelos: MappingConfigType = {
         url: 'https://data.metropole-rouen-normandie.fr/api/explore/v2.1/catalog/datasets/liste-des-stationnements-cyclables-metropole-rouen-normandie/exports/geojson?lang=fr&timezone=Europe%2FBerlin'
     },
     filters: {
-        exclude_point_if_tag_not_empty: ['osm_id']
+        exclude_point_if_tag_not_empty: ['osm_id'],
+        offset: 1
     },
     add_not_mapped_tags_too: false,
     boolean_keys: [
@@ -53,7 +54,7 @@ const mappingRouenParkingVelos: MappingConfigType = {
         },
         mobilier: {
             key_converted: 'mobilier',
-            remove_original_key: true,
+            remove_original_key: true, // TODO trouver ce qui empêche la conversion quand remove_original_key est a true
             conditional_values: {
                 "POTELET": {
                     "tags_to_add": [

mappings/engine.ts

@@ -1,5 +1,6 @@
 import custom_utils from './utils'
 import MappingConfigType from "./mapping-config.type";
+import Formatters from "./formatters";
 
 const {debugLog} = custom_utils
 
@@ -310,7 +311,6 @@ export default class {
                         let countOfSockets = (boolToAddable(has_prise_type_2) + boolToAddable(has_prise_type_combo_ccs) + boolToAddable(prise_type_chademo) +
                             boolToAddable(prise_type_ef) + boolToAddable(prise_type_autre) + boolToAddable(prise_type_e)
                         );
-                        // console.log('this.current_geojson_point.properties.prise_type_2', this.current_geojson_point.properties.prise_type_2, this.isTruthyValue(this.current_geojson_point.properties.prise_type_2), 'countOfSockets:', countOfSockets)
                         if (countOfSockets > 0) {
                             we_use_max_output = true;
                         }
@@ -319,24 +319,21 @@ export default class {
                         let converted_value = originalValue.replace(/[^\d\.\,]/g, '').replace(',', '.')
                         let max_output = 401
                         // do not limit accepted values
-                        // let accepted_values = [3, 7, 22, 50, 150, 300]
-                        // let accepted_values = [3, 7, 22, 50, 150, 300]
-                        // if (accepted_values.includes(converted_value)) {
                         let out = ''
 
                         if (intOriginalValue < max_output) {
 
-                            // enlever les lettres dans la valeur
+                            // rajouter l'unité de puissance kW dans la valeur
                             out = converted_value + ' kW'
 
                         } else {
-                            // console.log('too high kW value detected', originalValue)
                             // prise en charge des valeurs en Watts et non en kW.
+                            debugLog('too high kW value detected', originalValue)
                             if (intOriginalValue > 1000 && intOriginalValue < 401000) {
 
                                 let kilowatts = (parseFloat(converted_value) / 1000).toFixed(2).replace('.00', '');
                                 out = ('' + kilowatts + ' kW').replace('.00', '')
-                                // console.log('valeurs en Watts out', out, 'original:', originalValue)
+                                debugLog('valeurs en Watts out', out, 'original:', originalValue)
                                 this.stats.power_output++
                             }
                         }
@@ -397,65 +394,7 @@ export default class {
 
 
                     if (configObject.convert_to_phone) {
-                        /**
+                       convertedValue = Formatters.convertToPhone(originalValue)
-                         * nettoyer les numéros de téléphone en ne gardant que les nombres et le préfixe de pays
-                         */
-                        // debugLog('originalValue', originalValue.substring(1))
-                        if (!originalValue) {
-                            originalValue = ''
-                        }
-                        let original_without_spaces = originalValue.replace(/ /g, '')
-                        let cleaned_value = `${original_without_spaces}`
-                        cleaned_value = cleaned_value
-                            .trim()
-                            .replace('Stations-e', '')
-                            .replace(/[a-zA-Zéèà]/ig, '')
-                            .replace(/[\(\)\.\- ]/g, '')
-                        let add_prefix = false;
-                        if (
-                            /^\d/.test(cleaned_value) &&
-                            !/^\+33 /.test(original_without_spaces)
-                        ) {
-                            add_prefix = true
-                        }
-                        cleaned_value = cleaned_value.replace('+33', '')
-
-                        if (/^0/.test(cleaned_value)) {
-                            cleaned_value = cleaned_value.substring(1)
-                        }
-                        let array_of_numbers = cleaned_value
-                            .split('')
-
-
-                        let ii = 0;
-                        if (cleaned_value.length == 4) {
-                            ii = 1
-                        }
-                        convertedValue = ''
-                        array_of_numbers.forEach((num: string) => {
-                            if (ii % 2) {
-                                convertedValue += ' ';
-                            }
-
-                            convertedValue += num;
-                            ii++;
-                        })
-
-
-                        convertedValue = convertedValue.replace('  ', ' ').trim();
-                        debugLog('convertedValue', convertedValue)
-                        if (
-                            /^\d/.test(convertedValue) &&
-                            !/^\+33 /.test(convertedValue)
-                        ) {
-                            add_prefix = true
-                        }
-                        if (add_prefix) {
-                            convertedValue = `+33 ` + convertedValue
-                        }
-
-                        debugLog('phone: ', originalValue, '=>', convertedValue)
-
                         if (originalValue !== convertedValue) {
                             this.stats.phones_updated++
                             this.stats.phones_updated_list.push(convertedValue)
@@ -540,7 +479,6 @@ export default class {
                         }
 
                     }
-
                     debugLog('convertProperty: convertedValue ==========> {', newKey, ':', convertedValue, '}')
                     debugLog(' =============== remove_original_key', newKey, remove_original_key)
                     if (!remove_original_key && newKey && convertedValue && !configObject.ignore_this_data) {
@@ -548,6 +486,7 @@ export default class {
                         debugLog('convertProperty: added', newKey, convertedValue)
                         newProperties[newKey] = (`${convertedValue}`).trim()
                     }
+
                 }
             } else {
                 debugLog('!!!!!! property not found in mappingKeys: ', pointKeyName)

mappings/formatters.ts

@@ -0,0 +1,72 @@
+import custom_utils from "./utils";
+
+const {debugLog} = custom_utils
+
+/**
+ * Class that helps to convert values into predefined formats
+ */
+export default class Formatters {
+
+    static convertToPhone(originalValue: string) {
+        /**
+         * nettoyer les numéros de téléphone en ne gardant que les nombres et le préfixe de pays
+         */
+        // debugLog('originalValue', originalValue.substring(1))
+        if (!originalValue) {
+            originalValue = ''
+        }
+        let original_without_spaces = originalValue.replace(/ /g, '')
+        let cleaned_value = `${original_without_spaces}`
+        cleaned_value = cleaned_value
+            .trim()
+            .replace('Stations-e', '')
+            .replace(/[a-zA-Zéèà]/ig, '')
+            .replace(/[\(\)\.\- ]/g, '')
+        let add_prefix = false;
+        if (
+            /^\d/.test(cleaned_value) &&
+            !/^\+33 /.test(original_without_spaces)
+        ) {
+            add_prefix = true
+        }
+        cleaned_value = cleaned_value.replace('+33', '')
+
+        if (/^0/.test(cleaned_value)) {
+            cleaned_value = cleaned_value.substring(1)
+        }
+        let array_of_numbers = cleaned_value
+            .split('')
+
+
+        let ii = 0;
+        if (cleaned_value.length == 4) {
+            ii = 1
+        }
+        let convertedValue = ''
+        array_of_numbers.forEach((num: string) => {
+            if (ii % 2) {
+                convertedValue += ' ';
+            }
+            convertedValue += num;
+            ii++;
+        })
+
+
+        convertedValue = convertedValue.replace('  ', ' ').trim();
+        debugLog('convertedValue', convertedValue)
+        if (
+            /^\d/.test(convertedValue) &&
+            !/^\+33 /.test(convertedValue)
+        ) {
+            add_prefix = true
+        }
+        if (add_prefix) {
+            convertedValue = `+33 ` + convertedValue
+        }
+
+        debugLog('phone: ', originalValue, '=>', convertedValue)
+
+
+        return "";
+    }
+}

mappings/mapping-config.type.ts

@@ -33,6 +33,7 @@ export interface filteringConfig {
     enable_properties_filter?: boolean;
     properties?: object;
     bounding_box?: object;
+    offset?:number;
     exclude_point_if_tag_not_empty?: Array<string>;
     exclude_point_if_tag_truthy?: Array<string>;
     exclude_point_if_tag_falsy?: Array<string>;

mappings/utils.ts

@@ -1,4 +1,4 @@
-import fs from 'fs'
+import * as fs from 'node:fs'
 
 let show_debug = 0
 // show_debug = 1