split engine formatter to dedicated class, add filter offset
This commit is contained in:
parent
0ea1f6bdc9
commit
77ee5cc28e
@ -168,8 +168,22 @@ function convertDataFromSource(sourceFilePath: string, mapping: MappingConfigTyp
|
||||
let list_of_points: any = data_transformed.features
|
||||
debugLog('listOfPoints.length', list_of_points.length)
|
||||
|
||||
if(limitConversionToFirstPoint){
|
||||
debugLog('limitConversionToFirstPoint enabled')
|
||||
if(mapping?.filters?.offset){
|
||||
mapping.filters.offset = 1
|
||||
}
|
||||
}
|
||||
if(mapping?.filters?.offset){
|
||||
debugLog('filter offset enabled', mapping.filters.offset)
|
||||
list_of_points = list_of_points.splice(0,mapping.filters.offset)
|
||||
}
|
||||
console.log('list_of_points.length', list_of_points.length)
|
||||
|
||||
|
||||
/**
|
||||
* filtering
|
||||
* run filters before mapping OSM tags
|
||||
* TODO: do it in the engine
|
||||
*/
|
||||
// for each point from the data source, filter if we take it or not
|
||||
|
||||
@ -16,7 +16,8 @@ const mappingRouenParkingVelos: MappingConfigType = {
|
||||
url: 'https://data.metropole-rouen-normandie.fr/api/explore/v2.1/catalog/datasets/liste-des-stationnements-cyclables-metropole-rouen-normandie/exports/geojson?lang=fr&timezone=Europe%2FBerlin'
|
||||
},
|
||||
filters: {
|
||||
exclude_point_if_tag_not_empty: ['osm_id']
|
||||
exclude_point_if_tag_not_empty: ['osm_id'],
|
||||
offset: 1
|
||||
},
|
||||
add_not_mapped_tags_too: false,
|
||||
boolean_keys: [
|
||||
@ -53,7 +54,7 @@ const mappingRouenParkingVelos: MappingConfigType = {
|
||||
},
|
||||
mobilier: {
|
||||
key_converted: 'mobilier',
|
||||
remove_original_key: true,
|
||||
remove_original_key: true, // TODO trouver ce qui empêche la conversion quand remove_original_key est a true
|
||||
conditional_values: {
|
||||
"POTELET": {
|
||||
"tags_to_add": [
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
import custom_utils from './utils'
|
||||
import MappingConfigType from "./mapping-config.type";
|
||||
import Formatters from "./formatters";
|
||||
|
||||
const {debugLog} = custom_utils
|
||||
|
||||
@ -310,7 +311,6 @@ export default class {
|
||||
let countOfSockets = (boolToAddable(has_prise_type_2) + boolToAddable(has_prise_type_combo_ccs) + boolToAddable(prise_type_chademo) +
|
||||
boolToAddable(prise_type_ef) + boolToAddable(prise_type_autre) + boolToAddable(prise_type_e)
|
||||
);
|
||||
// console.log('this.current_geojson_point.properties.prise_type_2', this.current_geojson_point.properties.prise_type_2, this.isTruthyValue(this.current_geojson_point.properties.prise_type_2), 'countOfSockets:', countOfSockets)
|
||||
if (countOfSockets > 0) {
|
||||
we_use_max_output = true;
|
||||
}
|
||||
@ -319,24 +319,21 @@ export default class {
|
||||
let converted_value = originalValue.replace(/[^\d\.\,]/g, '').replace(',', '.')
|
||||
let max_output = 401
|
||||
// do not limit accepted values
|
||||
// let accepted_values = [3, 7, 22, 50, 150, 300]
|
||||
// let accepted_values = [3, 7, 22, 50, 150, 300]
|
||||
// if (accepted_values.includes(converted_value)) {
|
||||
let out = ''
|
||||
|
||||
if (intOriginalValue < max_output) {
|
||||
|
||||
// enlever les lettres dans la valeur
|
||||
// rajouter l'unité de puissance kW dans la valeur
|
||||
out = converted_value + ' kW'
|
||||
|
||||
} else {
|
||||
// console.log('too high kW value detected', originalValue)
|
||||
// prise en charge des valeurs en Watts et non en kW.
|
||||
debugLog('too high kW value detected', originalValue)
|
||||
if (intOriginalValue > 1000 && intOriginalValue < 401000) {
|
||||
|
||||
let kilowatts = (parseFloat(converted_value) / 1000).toFixed(2).replace('.00', '');
|
||||
out = ('' + kilowatts + ' kW').replace('.00', '')
|
||||
// console.log('valeurs en Watts out', out, 'original:', originalValue)
|
||||
debugLog('valeurs en Watts out', out, 'original:', originalValue)
|
||||
this.stats.power_output++
|
||||
}
|
||||
}
|
||||
@ -397,65 +394,7 @@ export default class {
|
||||
|
||||
|
||||
if (configObject.convert_to_phone) {
|
||||
/**
|
||||
* nettoyer les numéros de téléphone en ne gardant que les nombres et le préfixe de pays
|
||||
*/
|
||||
// debugLog('originalValue', originalValue.substring(1))
|
||||
if (!originalValue) {
|
||||
originalValue = ''
|
||||
}
|
||||
let original_without_spaces = originalValue.replace(/ /g, '')
|
||||
let cleaned_value = `${original_without_spaces}`
|
||||
cleaned_value = cleaned_value
|
||||
.trim()
|
||||
.replace('Stations-e', '')
|
||||
.replace(/[a-zA-Zéèà]/ig, '')
|
||||
.replace(/[\(\)\.\- ]/g, '')
|
||||
let add_prefix = false;
|
||||
if (
|
||||
/^\d/.test(cleaned_value) &&
|
||||
!/^\+33 /.test(original_without_spaces)
|
||||
) {
|
||||
add_prefix = true
|
||||
}
|
||||
cleaned_value = cleaned_value.replace('+33', '')
|
||||
|
||||
if (/^0/.test(cleaned_value)) {
|
||||
cleaned_value = cleaned_value.substring(1)
|
||||
}
|
||||
let array_of_numbers = cleaned_value
|
||||
.split('')
|
||||
|
||||
|
||||
let ii = 0;
|
||||
if (cleaned_value.length == 4) {
|
||||
ii = 1
|
||||
}
|
||||
convertedValue = ''
|
||||
array_of_numbers.forEach((num: string) => {
|
||||
if (ii % 2) {
|
||||
convertedValue += ' ';
|
||||
}
|
||||
|
||||
convertedValue += num;
|
||||
ii++;
|
||||
})
|
||||
|
||||
|
||||
convertedValue = convertedValue.replace(' ', ' ').trim();
|
||||
debugLog('convertedValue', convertedValue)
|
||||
if (
|
||||
/^\d/.test(convertedValue) &&
|
||||
!/^\+33 /.test(convertedValue)
|
||||
) {
|
||||
add_prefix = true
|
||||
}
|
||||
if (add_prefix) {
|
||||
convertedValue = `+33 ` + convertedValue
|
||||
}
|
||||
|
||||
debugLog('phone: ', originalValue, '=>', convertedValue)
|
||||
|
||||
convertedValue = Formatters.convertToPhone(originalValue)
|
||||
if (originalValue !== convertedValue) {
|
||||
this.stats.phones_updated++
|
||||
this.stats.phones_updated_list.push(convertedValue)
|
||||
@ -540,7 +479,6 @@ export default class {
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
debugLog('convertProperty: convertedValue ==========> {', newKey, ':', convertedValue, '}')
|
||||
debugLog(' =============== remove_original_key', newKey, remove_original_key)
|
||||
if (!remove_original_key && newKey && convertedValue && !configObject.ignore_this_data) {
|
||||
@ -548,6 +486,7 @@ export default class {
|
||||
debugLog('convertProperty: added', newKey, convertedValue)
|
||||
newProperties[newKey] = (`${convertedValue}`).trim()
|
||||
}
|
||||
|
||||
}
|
||||
} else {
|
||||
debugLog('!!!!!! property not found in mappingKeys: ', pointKeyName)
|
||||
|
||||
72
mappings/formatters.ts
Normal file
72
mappings/formatters.ts
Normal file
@ -0,0 +1,72 @@
|
||||
import custom_utils from "./utils";
|
||||
|
||||
const {debugLog} = custom_utils
|
||||
|
||||
/**
|
||||
* Class that helps to convert values into predefined formats
|
||||
*/
|
||||
export default class Formatters {
|
||||
|
||||
static convertToPhone(originalValue: string) {
|
||||
/**
|
||||
* nettoyer les numéros de téléphone en ne gardant que les nombres et le préfixe de pays
|
||||
*/
|
||||
// debugLog('originalValue', originalValue.substring(1))
|
||||
if (!originalValue) {
|
||||
originalValue = ''
|
||||
}
|
||||
let original_without_spaces = originalValue.replace(/ /g, '')
|
||||
let cleaned_value = `${original_without_spaces}`
|
||||
cleaned_value = cleaned_value
|
||||
.trim()
|
||||
.replace('Stations-e', '')
|
||||
.replace(/[a-zA-Zéèà]/ig, '')
|
||||
.replace(/[\(\)\.\- ]/g, '')
|
||||
let add_prefix = false;
|
||||
if (
|
||||
/^\d/.test(cleaned_value) &&
|
||||
!/^\+33 /.test(original_without_spaces)
|
||||
) {
|
||||
add_prefix = true
|
||||
}
|
||||
cleaned_value = cleaned_value.replace('+33', '')
|
||||
|
||||
if (/^0/.test(cleaned_value)) {
|
||||
cleaned_value = cleaned_value.substring(1)
|
||||
}
|
||||
let array_of_numbers = cleaned_value
|
||||
.split('')
|
||||
|
||||
|
||||
let ii = 0;
|
||||
if (cleaned_value.length == 4) {
|
||||
ii = 1
|
||||
}
|
||||
let convertedValue = ''
|
||||
array_of_numbers.forEach((num: string) => {
|
||||
if (ii % 2) {
|
||||
convertedValue += ' ';
|
||||
}
|
||||
convertedValue += num;
|
||||
ii++;
|
||||
})
|
||||
|
||||
|
||||
convertedValue = convertedValue.replace(' ', ' ').trim();
|
||||
debugLog('convertedValue', convertedValue)
|
||||
if (
|
||||
/^\d/.test(convertedValue) &&
|
||||
!/^\+33 /.test(convertedValue)
|
||||
) {
|
||||
add_prefix = true
|
||||
}
|
||||
if (add_prefix) {
|
||||
convertedValue = `+33 ` + convertedValue
|
||||
}
|
||||
|
||||
debugLog('phone: ', originalValue, '=>', convertedValue)
|
||||
|
||||
|
||||
return "";
|
||||
}
|
||||
}
|
||||
@ -33,6 +33,7 @@ export interface filteringConfig {
|
||||
enable_properties_filter?: boolean;
|
||||
properties?: object;
|
||||
bounding_box?: object;
|
||||
offset?:number;
|
||||
exclude_point_if_tag_not_empty?: Array<string>;
|
||||
exclude_point_if_tag_truthy?: Array<string>;
|
||||
exclude_point_if_tag_falsy?: Array<string>;
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import fs from 'fs'
|
||||
import * as fs from 'node:fs'
|
||||
|
||||
let show_debug = 0
|
||||
// show_debug = 1
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue
Block a user