change file output name, fix tags to add conditionnal handling

This commit is contained in:
Tykayn 2024-10-08 10:37:58 +02:00 committed by tykayn
parent 77ee5cc28e
commit 04c3bd8d53
6 changed files with 19010 additions and 79 deletions

View File

@ -20,19 +20,13 @@ const minimist = require('minimist')
const debugLog = utils.debugLog; const debugLog = utils.debugLog;
let use_mapping_engine = true
let mini_arguments: any = minimist(process.argv.slice(2)) let mini_arguments: any = minimist(process.argv.slice(2))
let use_mapping_engine = true
let output_supplement = ''; let output_supplement = '';
let sourceFilePathGeoJson = './data_other/' + output_supplement + '.json' let sourceFilePathGeoJson = './data_other/' + output_supplement + '.json'
// wip filter // wip filter
let filterOnBoundingBox = true let filterOnBoundingBox = true
filterOnBoundingBox = false filterOnBoundingBox = false
let boundingBoxCoordinates: BoundingBoxCoordinatesType = { let boundingBoxCoordinates: BoundingBoxCoordinatesType = {
xMin: 1.91, xMin: 1.91,
xMax: 2.38, xMax: 2.38,
@ -41,7 +35,6 @@ let boundingBoxCoordinates: BoundingBoxCoordinatesType = {
} }
let default_engine_conf_choice: string = 'ConfigIRVE'; let default_engine_conf_choice: string = 'ConfigIRVE';
let engine_conf_choice: string = 'ConfigIRVE'; let engine_conf_choice: string = 'ConfigIRVE';
let filterCoordinates = true let filterCoordinates = true
filterCoordinates = false filterCoordinates = false
@ -70,7 +63,7 @@ if (mini_arguments['engine-config']) {
output_supplement += "_" + engine_conf_choice output_supplement += "_" + engine_conf_choice
} }
let Mapping_engine: any; let Mapping_engine: any;
let outname = 'my_data_set_' let outname = 'converted_'
if (mini_arguments['outname']) { if (mini_arguments['outname']) {
outname = mini_arguments['outname']; outname = mini_arguments['outname'];
@ -278,7 +271,7 @@ function convertDataFromSource(sourceFilePath: string, mapping: MappingConfigTyp
if (output_supplement) { if (output_supplement) {
output_supplement = '_' + output_supplement; output_supplement = '_' + output_supplement;
} }
let fileNameToWrite = 'my_converted_data_set' + filteredName + output_supplement + '.json' let fileNameToWrite = '_' + filteredName + output_supplement + '.json'
console.log('converted features:', converted_geo_json.features.length) console.log('converted features:', converted_geo_json.features.length)
console.log('diff: ', data_transformed.features.length - converted_geo_json.features.length) console.log('diff: ', data_transformed.features.length - converted_geo_json.features.length)
debugLog('convert : write file ', fileNameToWrite) debugLog('convert : write file ', fileNameToWrite)
@ -388,15 +381,13 @@ function setMappingConfigFromName(engine_conf_choice: string) {
convertDataFromSource(sourceFilePathGeoJson, mappingConfigIRVE, pointCounterMax, boundingBoxCoordinates) convertDataFromSource(sourceFilePathGeoJson, mappingConfigIRVE, pointCounterMax, boundingBoxCoordinates)
} }
} }
/**
* Launch conversion of dataset
*/
function init() { function init() {
setMappingConfigFromName(engine_conf_choice) setMappingConfigFromName(engine_conf_choice)
} }
init() init()

View File

@ -17,7 +17,7 @@ const mappingRouenParkingVelos: MappingConfigType = {
}, },
filters: { filters: {
exclude_point_if_tag_not_empty: ['osm_id'], exclude_point_if_tag_not_empty: ['osm_id'],
offset: 1 // offset: 1
}, },
add_not_mapped_tags_too: false, add_not_mapped_tags_too: false,
boolean_keys: [ boolean_keys: [
@ -57,21 +57,21 @@ const mappingRouenParkingVelos: MappingConfigType = {
remove_original_key: true, // TODO trouver ce qui empêche la conversion quand remove_original_key est a true remove_original_key: true, // TODO trouver ce qui empêche la conversion quand remove_original_key est a true
conditional_values: { conditional_values: {
"POTELET": { "POTELET": {
"tags_to_add": [ tags_to_add: [
{ {
"bicycle_parking": "bollard" "bicycle_parking": "bollard"
}, },
] ]
}, },
"ARCEAU": { "ARCEAU": {
"tags_to_add": [ tags_to_add: [
{ {
"bicycle_parking": "stands" "bicycle_parking": "stands"
}, },
] ]
}, },
"RATELIER": { "RATELIER": {
"tags_to_add": [ tags_to_add: [
{ {
"bicycle_parking": "rack" "bicycle_parking": "rack"
}, },

View File

@ -22,7 +22,7 @@ function boolToAddable(someBooleanValue: boolean) {
export default class { export default class {
mapping_config: any = {} mapping_config: any = {}
private jardinage = true; private jardinage = false;
public stats: any; public stats: any;
private current_converted_geojson_point: any; private current_converted_geojson_point: any;
private current_geojson_point: any; private current_geojson_point: any;
@ -140,7 +140,7 @@ export default class {
debugLog('mapElementFromConf: convert', pointKeyName) debugLog('mapElementFromConf: convert', pointKeyName)
debugLog('mapElementFromConf: mapping keys:', mappingKeys) debugLog('mapElementFromConf: mapping keys:', mappingKeys)
this.convertProperty(pointKeyName, mappingKeys, featurePoint, newProperties) this.convertProperty(pointKeyName, mappingKeys, featurePoint, newProperties)
}) })
@ -182,6 +182,7 @@ export default class {
let remove_original_key = false; let remove_original_key = false;
if (this.mapping_config.tags_to_ignore_if_value_is.indexOf(originalValue) !== -1) { if (this.mapping_config.tags_to_ignore_if_value_is.indexOf(originalValue) !== -1) {
debugLog('(x) => ignore', originalValue, ' in ', pointKeyName)
remove_original_key = true; remove_original_key = true;
} }
@ -204,6 +205,7 @@ export default class {
/** /**
* only use existing keys * only use existing keys
*/ */
debugLog("only use existing keys,", pointKeyName)
if (mappingKeys.indexOf(pointKeyName) !== -1) { if (mappingKeys.indexOf(pointKeyName) !== -1) {
let valueConvertedFromMapping = featurePoint.properties[pointKeyName] let valueConvertedFromMapping = featurePoint.properties[pointKeyName]
let keyConvertedFromMapping = mappingKeys[mappingKeys.indexOf(pointKeyName)] let keyConvertedFromMapping = mappingKeys[mappingKeys.indexOf(pointKeyName)]
@ -394,7 +396,7 @@ export default class {
if (configObject.convert_to_phone) { if (configObject.convert_to_phone) {
convertedValue = Formatters.convertToPhone(originalValue) convertedValue = Formatters.convertToPhone(originalValue)
if (originalValue !== convertedValue) { if (originalValue !== convertedValue) {
this.stats.phones_updated++ this.stats.phones_updated++
this.stats.phones_updated_list.push(convertedValue) this.stats.phones_updated_list.push(convertedValue)
@ -417,73 +419,82 @@ export default class {
* config pour une clé * config pour une clé
* nous pouvons renseigner une string ou un objet décrivant les transformations à réaliser * nous pouvons renseigner une string ou un objet décrivant les transformations à réaliser
*/ */
if (!remove_original_key && configObject.conditional_values) { if (configObject.conditional_values) {
let keysConditionnalValues: any = Object.keys(configObject.conditional_values)
let isFoundValue = keysConditionnalValues.indexOf(originalValue)
let conditionnalConfig: any = configObject.conditional_values[keysConditionnalValues[isFoundValue]]
debugLog('convertProperty: conditional_values__________', debugLog('convertProperty: conditional_values__________',
configObject.conditional_values) configObject.conditional_values)
let keysConditionnalValues: any = Object.keys(configObject.conditional_values)
let isFoundValue = keysConditionnalValues.indexOf(originalValue)
debugLog('isFoundValue', isFoundValue, originalValue) debugLog('isFoundValue', isFoundValue, originalValue)
debugLog('keysConditionnalValues', keysConditionnalValues) debugLog('keysConditionnalValues', keysConditionnalValues)
if (!remove_original_key) {
if (isFoundValue > -1) {
let conditionnalConfig: any = configObject.conditional_values[keysConditionnalValues[isFoundValue]]
/** ----------------------
* gestion des valeurs conditionnelles
* ---------------------- */
debugLog('conditionnalConfig', conditionnalConfig)
if (conditionnalConfig.ignore_this_data) { if (isFoundValue > -1) {
debugLog(`on ignore cette clé car sa valeur "${originalValue}" est à exclure: `, pointKeyName, '=>', newKey)
remove_original_key = true;
}
if (conditionnalConfig.tags_to_add) {
// on peut définir un ensemble de tags à rajouter
let tagKeys = Object.keys(conditionnalConfig.tags_to_add)
debugLog('conditionnalConfig.tags_to_add', conditionnalConfig.tags_to_add)
conditionnalConfig.tags_to_add.forEach((object: any, pair: any) => {
debugLog('object', object)
debugLog('pair', pair)
let key: any = Object.keys(object)
key = key[0]
let value = object[key]
debugLog('key', key) /** ----------------------
debugLog('value', value) * gestion des valeurs conditionnelles
newProperties[key] = value * ---------------------- */
}) debugLog('conditionnalConfig', conditionnalConfig)
}
if (conditionnalConfig.truthy_value) {
// convertir la valeur, si elle est truthy, la transformer en ce que donne la propriété truthy_value if (conditionnalConfig.ignore_this_data) {
// exemple: le jeu de données dit que la colonne cable_t2_attache vaut "True", mais on veut le convertir en "1". debugLog(`on ignore cette clé car sa valeur "${originalValue}" est à exclure: `, pointKeyName, '=>', newKey)
// on met donc truthy_value: '1' remove_original_key = true;
if (this.truthyValues.indexOf(originalValue) !== -1) {
convertedValue = conditionnalConfig.truthy_value
} }
}
if (conditionnalConfig.falsy_value) { if (conditionnalConfig.truthy_value) {
if (this.falsyValues.indexOf(originalValue) !== -1) { // convertir la valeur, si elle est truthy, la transformer en ce que donne la propriété truthy_value
convertedValue = conditionnalConfig.falsy_value // exemple: le jeu de données dit que la colonne cable_t2_attache vaut "True", mais on veut le convertir en "1".
// on met donc truthy_value: '1'
if (this.truthyValues.indexOf(originalValue) !== -1) {
convertedValue = conditionnalConfig.truthy_value
}
} }
if (conditionnalConfig.falsy_value) {
if (this.falsyValues.indexOf(originalValue) !== -1) {
convertedValue = conditionnalConfig.falsy_value
}
}
// use the value converted
else if (conditionnalConfig.value_converted) {
convertedValue = conditionnalConfig.value_converted
}
} }
// use the value converted }
else if (conditionnalConfig.value_converted) { if (conditionnalConfig?.tags_to_add) {
convertedValue = conditionnalConfig.value_converted debugLog('on ajoute des tags', conditionnalConfig.tags_to_add)
} // on peut définir un ensemble de tags à rajouter
let tagKeys = Object.keys(conditionnalConfig.tags_to_add)
debugLog('conditionnalConfig.tags_to_add', conditionnalConfig.tags_to_add)
conditionnalConfig.tags_to_add.forEach((object: any, pair: any) => {
debugLog('object', object)
debugLog('pair', pair)
let key: any = Object.keys(object)
key = key[0]
let value = object[key]
debugLog('key', key)
debugLog('value', value)
newProperties[key] = value
})
} }
} }
debugLog('convertProperty: convertedValue ==========> {', newKey, ':', convertedValue, '}') debugLog('convertProperty: convertedValue ==========> {', newKey, ':', convertedValue, '}')
debugLog(' =============== remove_original_key', newKey, remove_original_key) debugLog(' =============== remove_original_key', newKey, remove_original_key)
if (!remove_original_key && newKey && convertedValue && !configObject.ignore_this_data) { if (!remove_original_key && newKey && convertedValue && !configObject.ignore_this_data) {
debugLog('convertProperty: added', newKey, convertedValue) debugLog('convertProperty: added', newKey, (`${convertedValue}`).trim())
newProperties[newKey] = (`${convertedValue}`).trim() newProperties[newKey] = (`${convertedValue}`).trim()
} }
@ -493,6 +504,7 @@ export default class {
} }
} }
debugLog('newProperties', newProperties)
return newProperties; return newProperties;
} }

File diff suppressed because it is too large Load Diff

View File

@ -14,6 +14,8 @@
"amenity": "bicycle_parking", "amenity": "bicycle_parking",
"ref:FR:rouen_veloparking_id": 775, "ref:FR:rouen_veloparking_id": 775,
"capacity": 8, "capacity": 8,
"bicycle_parking": "stands",
"mobilier": "ARCEAU",
"check_date": "10/08/2021" "check_date": "10/08/2021"
} }
} }

View File

@ -9,7 +9,10 @@ export const mappingRemoveAll: MappingConfigType = {
default_properties_of_point: {}, default_properties_of_point: {},
tags: {}, tags: {},
add_not_mapped_tags_too: false, add_not_mapped_tags_too: false,
source: {}, source: {
geojson_path: "",
url: ""
},
filters: {}, filters: {},
} }
export const mappingSame: MappingConfigType = { export const mappingSame: MappingConfigType = {
@ -17,10 +20,13 @@ export const mappingSame: MappingConfigType = {
config_author: 'tykayn <contact@cipherbliss.com>', config_author: 'tykayn <contact@cipherbliss.com>',
default_properties_of_point: {}, default_properties_of_point: {},
tags: { tags: {
equal : 'equal' equal: 'equal'
}, },
add_not_mapped_tags_too: false, add_not_mapped_tags_too: false,
source: {}, source: {
geojson_path: "",
url: ""
},
filters: {}, filters: {},
} }
export const mappingName: MappingConfigType = { export const mappingName: MappingConfigType = {
@ -29,9 +35,12 @@ export const mappingName: MappingConfigType = {
default_properties_of_point: {}, default_properties_of_point: {},
add_not_mapped_tags_too: false, add_not_mapped_tags_too: false,
tags: { tags: {
nom_amenageur : 'name' nom_amenageur: 'name'
},
source: {
geojson_path: "",
url: ""
}, },
source: {},
filters: {}, filters: {},
} }
export const mappingPhone: MappingConfigType = { export const mappingPhone: MappingConfigType = {
@ -46,7 +55,10 @@ export const mappingPhone: MappingConfigType = {
}, },
}, },
add_not_mapped_tags_too: false, add_not_mapped_tags_too: false,
source: {}, source: {
geojson_path: "",
url: ""
},
filters: {}, filters: {},
} }
export const mappingBoolean: MappingConfigType = { export const mappingBoolean: MappingConfigType = {
@ -60,7 +72,10 @@ export const mappingBoolean: MappingConfigType = {
}, },
}, },
add_not_mapped_tags_too: false, add_not_mapped_tags_too: false,
source: {}, source: {
geojson_path: "",
url: ""
},
filters: {}, filters: {},
} }
export const mappingTruthy: MappingConfigType = { export const mappingTruthy: MappingConfigType = {
@ -74,7 +89,10 @@ export const mappingTruthy: MappingConfigType = {
}, },
}, },
add_not_mapped_tags_too: false, add_not_mapped_tags_too: false,
source: {}, source: {
geojson_path: "",
url: ""
},
filters: {}, filters: {},
} }
export const mappingFalsy: MappingConfigType = { export const mappingFalsy: MappingConfigType = {
@ -88,7 +106,10 @@ export const mappingFalsy: MappingConfigType = {
}, },
}, },
add_not_mapped_tags_too: false, add_not_mapped_tags_too: false,
source: {}, source: {
geojson_path: "",
url: ""
},
filters: {}, filters: {},
} }
export const mappingIgnoreFalsy: MappingConfigType = { export const mappingIgnoreFalsy: MappingConfigType = {
@ -102,7 +123,10 @@ export const mappingIgnoreFalsy: MappingConfigType = {
}, },
}, },
add_not_mapped_tags_too: false, add_not_mapped_tags_too: false,
source: {}, source: {
geojson_path: "",
url: ""
},
filters: {}, filters: {},
} }
export const mappingIgnoreTruthy: MappingConfigType = { export const mappingIgnoreTruthy: MappingConfigType = {
@ -116,6 +140,9 @@ export const mappingIgnoreTruthy: MappingConfigType = {
}, },
}, },
add_not_mapped_tags_too: false, add_not_mapped_tags_too: false,
source: {}, source: {
geojson_path: "",
url: ""
},
filters: {}, filters: {},
} }