tykayn/mapping-geojson-osm
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

⚡ change file output name, fix tags to add conditionnal handling

Browse Source
This commit is contained in:
Tykayn 2024-10-08 10:37:58 +02:00 committed by tykayn
parent 77ee5cc28e
commit 04c3bd8d53
6 changed files with 19010 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
convert_to_osm_tags.ts
View File

@ -20,19 +20,13 @@ const minimist = require('minimist')
const debugLog = utils.debugLog;
let use_mapping_engine = true
let mini_arguments: any = minimist(process.argv.slice(2))
let use_mapping_engine = true
let output_supplement = '';
let sourceFilePathGeoJson = './data_other/' + output_supplement + '.json'
// wip filter
let filterOnBoundingBox = true
filterOnBoundingBox = false
let boundingBoxCoordinates: BoundingBoxCoordinatesType = {
xMin: 1.91,
xMax: 2.38,
@ -41,7 +35,6 @@ let boundingBoxCoordinates: BoundingBoxCoordinatesType = {
}
let default_engine_conf_choice: string = 'ConfigIRVE';
let engine_conf_choice: string = 'ConfigIRVE';
let filterCoordinates = true
filterCoordinates = false
@ -70,7 +63,7 @@ if (mini_arguments['engine-config']) {
output_supplement += "_" + engine_conf_choice
}
let Mapping_engine: any;
let outname = 'my_data_set_'
let outname = 'converted_'
if (mini_arguments['outname']) {
outname = mini_arguments['outname'];
@ -278,7 +271,7 @@ function convertDataFromSource(sourceFilePath: string, mapping: MappingConfigTyp
if (output_supplement) {
output_supplement = '_' + output_supplement;
}
let fileNameToWrite = 'my_converted_data_set' + filteredName + output_supplement + '.json'
let fileNameToWrite = '_' + filteredName + output_supplement + '.json'
console.log('converted features:', converted_geo_json.features.length)
console.log('diff: ', data_transformed.features.length - converted_geo_json.features.length)
debugLog('convert : write file ', fileNameToWrite)
@ -388,15 +381,13 @@ function setMappingConfigFromName(engine_conf_choice: string) {
convertDataFromSource(sourceFilePathGeoJson, mappingConfigIRVE, pointCounterMax, boundingBoxCoordinates)
}
}
/**
* Launch conversion of dataset
*/
function init() {
setMappingConfigFromName(engine_conf_choice)
}
init()

8
mappings/converters/configRouen_OpenData_velo_parkings.ts
View File

@ -17,7 +17,7 @@ const mappingRouenParkingVelos: MappingConfigType = {
},
filters: {
exclude_point_if_tag_not_empty: ['osm_id'],
offset: 1
// offset: 1
},
add_not_mapped_tags_too: false,
boolean_keys: [
@ -57,21 +57,21 @@ const mappingRouenParkingVelos: MappingConfigType = {
remove_original_key: true, // TODO trouver ce qui empêche la conversion quand remove_original_key est a true
conditional_values: {
"POTELET": {
"tags_to_add": [
tags_to_add: [
{
"bicycle_parking": "bollard"
},
]
},
"ARCEAU": {
"tags_to_add": [
tags_to_add: [
{
"bicycle_parking": "stands"
},
]
},
"RATELIER": {
"tags_to_add": [
tags_to_add: [
{
"bicycle_parking": "rack"
},

110
mappings/engine.ts
View File

@ -22,7 +22,7 @@ function boolToAddable(someBooleanValue: boolean) {
export default class {
mapping_config: any = {}
private jardinage = true;
private jardinage = false;
public stats: any;
private current_converted_geojson_point: any;
private current_geojson_point: any;
@ -140,7 +140,7 @@ export default class {
debugLog('mapElementFromConf: convert', pointKeyName)
debugLog('mapElementFromConf: mapping keys:', mappingKeys)
this.convertProperty(pointKeyName, mappingKeys, featurePoint, newProperties)
this.convertProperty(pointKeyName, mappingKeys, featurePoint, newProperties)
})
@ -182,6 +182,7 @@ export default class {
let remove_original_key = false;
if (this.mapping_config.tags_to_ignore_if_value_is.indexOf(originalValue) !== -1) {
debugLog('(x) => ignore', originalValue, ' in ', pointKeyName)
remove_original_key = true;
}
@ -204,6 +205,7 @@ export default class {
/**
* only use existing keys
*/
debugLog("only use existing keys,", pointKeyName)
if (mappingKeys.indexOf(pointKeyName) !== -1) {
let valueConvertedFromMapping = featurePoint.properties[pointKeyName]
let keyConvertedFromMapping = mappingKeys[mappingKeys.indexOf(pointKeyName)]
@ -394,7 +396,7 @@ export default class {
if (configObject.convert_to_phone) {
convertedValue = Formatters.convertToPhone(originalValue)
convertedValue = Formatters.convertToPhone(originalValue)
if (originalValue !== convertedValue) {
this.stats.phones_updated++
this.stats.phones_updated_list.push(convertedValue)
@ -417,73 +419,82 @@ export default class {
* config pour une clé
* nous pouvons renseigner une string ou un objet décrivant les transformations à réaliser
*/
if (!remove_original_key && configObject.conditional_values) {
if (configObject.conditional_values) {
let keysConditionnalValues: any = Object.keys(configObject.conditional_values)
let isFoundValue = keysConditionnalValues.indexOf(originalValue)
let conditionnalConfig: any = configObject.conditional_values[keysConditionnalValues[isFoundValue]]
debugLog('convertProperty: conditional_values__________',
configObject.conditional_values)
let keysConditionnalValues: any = Object.keys(configObject.conditional_values)
let isFoundValue = keysConditionnalValues.indexOf(originalValue)
debugLog('isFoundValue', isFoundValue, originalValue)
debugLog('keysConditionnalValues', keysConditionnalValues)
if (isFoundValue > -1) {
let conditionnalConfig: any = configObject.conditional_values[keysConditionnalValues[isFoundValue]]
/** ----------------------
* gestion des valeurs conditionnelles
* ---------------------- */
debugLog('conditionnalConfig', conditionnalConfig)
if (!remove_original_key) {
if (conditionnalConfig.ignore_this_data) {
debugLog(`on ignore cette clé car sa valeur "${originalValue}" est à exclure: `, pointKeyName, '=>', newKey)
remove_original_key = true;
}
if (conditionnalConfig.tags_to_add) {
// on peut définir un ensemble de tags à rajouter
let tagKeys = Object.keys(conditionnalConfig.tags_to_add)
debugLog('conditionnalConfig.tags_to_add', conditionnalConfig.tags_to_add)
conditionnalConfig.tags_to_add.forEach((object: any, pair: any) => {
debugLog('object', object)
debugLog('pair', pair)
let key: any = Object.keys(object)
key = key[0]
let value = object[key]
if (isFoundValue > -1) {
debugLog('key', key)
debugLog('value', value)
newProperties[key] = value
})
}
if (conditionnalConfig.truthy_value) {
// convertir la valeur, si elle est truthy, la transformer en ce que donne la propriété truthy_value
// exemple: le jeu de données dit que la colonne cable_t2_attache vaut "True", mais on veut le convertir en "1".
// on met donc truthy_value: '1'
if (this.truthyValues.indexOf(originalValue) !== -1) {
convertedValue = conditionnalConfig.truthy_value
/** ----------------------
* gestion des valeurs conditionnelles
* ---------------------- */
debugLog('conditionnalConfig', conditionnalConfig)
if (conditionnalConfig.ignore_this_data) {
debugLog(`on ignore cette clé car sa valeur "${originalValue}" est à exclure: `, pointKeyName, '=>', newKey)
remove_original_key = true;
}
}
if (conditionnalConfig.falsy_value) {
if (this.falsyValues.indexOf(originalValue) !== -1) {
convertedValue = conditionnalConfig.falsy_value
if (conditionnalConfig.truthy_value) {
// convertir la valeur, si elle est truthy, la transformer en ce que donne la propriété truthy_value
// exemple: le jeu de données dit que la colonne cable_t2_attache vaut "True", mais on veut le convertir en "1".
// on met donc truthy_value: '1'
if (this.truthyValues.indexOf(originalValue) !== -1) {
convertedValue = conditionnalConfig.truthy_value
}
}
if (conditionnalConfig.falsy_value) {
if (this.falsyValues.indexOf(originalValue) !== -1) {
convertedValue = conditionnalConfig.falsy_value
}
}
// use the value converted
else if (conditionnalConfig.value_converted) {
convertedValue = conditionnalConfig.value_converted
}
}
// use the value converted
else if (conditionnalConfig.value_converted) {
convertedValue = conditionnalConfig.value_converted
}
}
if (conditionnalConfig?.tags_to_add) {
debugLog('on ajoute des tags', conditionnalConfig.tags_to_add)
// on peut définir un ensemble de tags à rajouter
let tagKeys = Object.keys(conditionnalConfig.tags_to_add)
debugLog('conditionnalConfig.tags_to_add', conditionnalConfig.tags_to_add)
conditionnalConfig.tags_to_add.forEach((object: any, pair: any) => {
debugLog('object', object)
debugLog('pair', pair)
let key: any = Object.keys(object)
key = key[0]
let value = object[key]
debugLog('key', key)
debugLog('value', value)
newProperties[key] = value
})
}
}
debugLog('convertProperty: convertedValue ==========> {', newKey, ':', convertedValue, '}')
debugLog(' =============== remove_original_key', newKey, remove_original_key)
if (!remove_original_key && newKey && convertedValue && !configObject.ignore_this_data) {
debugLog('convertProperty: added', newKey, convertedValue)
debugLog('convertProperty: added', newKey, (`${convertedValue}`).trim())
newProperties[newKey] = (`${convertedValue}`).trim()
}
@ -493,6 +504,7 @@ export default class {
}
}
debugLog('newProperties', newProperties)
return newProperties;
}

18899
output/___mappingRouenParkingVelos.json Normal file
View File

File diff suppressed because it is too large Load Diff

2
output/my_converted_data_set__mappingRouenParkingVelos.json
View File

@ -14,6 +14,8 @@
"amenity": "bicycle_parking",
"ref:FR:rouen_veloparking_id": 775,
"capacity": 8,
"bicycle_parking": "stands",
"mobilier": "ARCEAU",
"check_date": "10/08/2021"
}
}

49
tests/data/mappings_to_test.ts
View File

@ -9,7 +9,10 @@ export const mappingRemoveAll: MappingConfigType = {
default_properties_of_point: {},
tags: {},
add_not_mapped_tags_too: false,
source: {},
source: {
geojson_path: "",
url: ""
},
filters: {},
}
export const mappingSame: MappingConfigType = {
@ -17,10 +20,13 @@ export const mappingSame: MappingConfigType = {
config_author: 'tykayn <contact@cipherbliss.com>',
default_properties_of_point: {},
tags: {
equal : 'equal'
equal: 'equal'
},
add_not_mapped_tags_too: false,
source: {},
source: {
geojson_path: "",
url: ""
},
filters: {},
}
export const mappingName: MappingConfigType = {
@ -29,9 +35,12 @@ export const mappingName: MappingConfigType = {
default_properties_of_point: {},
add_not_mapped_tags_too: false,
tags: {
nom_amenageur : 'name'
nom_amenageur: 'name'
},
source: {
geojson_path: "",
url: ""
},
source: {},
filters: {},
}
export const mappingPhone: MappingConfigType = {
@ -46,7 +55,10 @@ export const mappingPhone: MappingConfigType = {
},
},
add_not_mapped_tags_too: false,
source: {},
source: {
geojson_path: "",
url: ""
},
filters: {},
}
export const mappingBoolean: MappingConfigType = {
@ -60,7 +72,10 @@ export const mappingBoolean: MappingConfigType = {
},
},
add_not_mapped_tags_too: false,
source: {},
source: {
geojson_path: "",
url: ""
},
filters: {},
}
export const mappingTruthy: MappingConfigType = {
@ -74,7 +89,10 @@ export const mappingTruthy: MappingConfigType = {
},
},
add_not_mapped_tags_too: false,
source: {},
source: {
geojson_path: "",
url: ""
},
filters: {},
}
export const mappingFalsy: MappingConfigType = {
@ -88,7 +106,10 @@ export const mappingFalsy: MappingConfigType = {
},
},
add_not_mapped_tags_too: false,
source: {},
source: {
geojson_path: "",
url: ""
},
filters: {},
}
export const mappingIgnoreFalsy: MappingConfigType = {
@ -102,7 +123,10 @@ export const mappingIgnoreFalsy: MappingConfigType = {
},
},
add_not_mapped_tags_too: false,
source: {},
source: {
geojson_path: "",
url: ""
},
filters: {},
}
export const mappingIgnoreTruthy: MappingConfigType = {
@ -116,6 +140,9 @@ export const mappingIgnoreTruthy: MappingConfigType = {
},
},
add_not_mapped_tags_too: false,
source: {},
source: {
geojson_path: "",
url: ""
},
filters: {},
}