From 32d3834792589f53409e6ab169d4b4d4344c6c47 Mon Sep 17 00:00:00 2001 From: Tykayn Date: Sat, 5 Aug 2023 16:06:44 +0200 Subject: [PATCH] improve mapping --- .../mappings/converters/configIRVE.ts | 55 ++-- .../mappings/engine.ts | 266 ++++++++++-------- .../mappings/mapping-config.type.ts | 3 +- .../output/my_converted_data_set.json | 21 +- ...ed_data_set_filtered_zipcode_91_small.json | 56 +++- 5 files changed, 253 insertions(+), 148 deletions(-) diff --git a/mapping_geojson_to_osm_tags/mappings/converters/configIRVE.ts b/mapping_geojson_to_osm_tags/mappings/converters/configIRVE.ts index 046a8e42..cc3b9c99 100644 --- a/mapping_geojson_to_osm_tags/mappings/converters/configIRVE.ts +++ b/mapping_geojson_to_osm_tags/mappings/converters/configIRVE.ts @@ -11,6 +11,7 @@ const MappingIRVE: MappingConfigType = { default_properties_of_point: { 'amenity': 'charging_station' }, + add_not_mapped_tags_too: false, tags: { // ******* nombres @@ -31,9 +32,9 @@ const MappingIRVE: MappingConfigType = { paiement_acte: 'authentication:none', reservation: { - boolean_value_conversion: true, // convertit en yes ou no + convert_to_boolean_value: true, // convertit en yes ou no }, - observations: 'note', + // observations: 'note', nom_station: 'name', nom_enseigne: 'network', @@ -46,15 +47,30 @@ const MappingIRVE: MappingConfigType = { cable_t2_attache: { key_converted: 'socket:type2_cable', - // cable_t2_attache + // socket:type2_cable vaut 1 dans OSM si vrai truthy_value: '1', } , - prise_type_ef: 'socket:typee', - prise_type_2: 'socket:type2', - prise_type_combo_ccs: 'socket:type2_combo', - prise_type_chademo: 'socket:chademo', - + prise_type_ef: { + key_converted: 'socket:typee', + ignore_if_falsy: true, + convert_to_boolean_value: true, + }, + prise_type_2: { + key_converted: 'socket:type2', + ignore_if_falsy: true, + convert_to_boolean_value: true, + }, + prise_type_combo_ccs: { + key_converted: 'socket:type2_combo', + ignore_if_falsy: true, + convert_to_boolean_value: true, + }, + prise_type_chademo: { + key_converted: 'socket:chademo', + ignore_if_falsy: true, + convert_to_boolean_value: true, + }, // ******** champs plus complexes horaires: 'opening_hours', // déjà au bon format @@ -116,15 +132,20 @@ const MappingIRVE: MappingConfigType = { } -// "station_deux_roues": "yes", -// "amenity": "charging_station", -// "capacity": 12, -// "reservation": "False", -// "nom_amenageur": "Bob Lenon", -// "siren_amenageur": "12345678", -// "socket:typee": "False", -// "socket:type2": "true", -// "cable_t2_attache": "no", +// "cable_t2_attache": "True" +// "goal": "jeu de données pour tester la mapping engine", +// "prise_type_2": "yes", +// "station_deux_roues": "yes", +// "accessibilite_pmr": "Non accessible", +// "amenity": "charging_station", +// "capacity": 12, +// "reservation": "False", +// "nom_amenageur": "Bob Lenon", +// "siren_amenageur": "12345678", +// "socket:typee": "False", +// "prise_type_combo_ccs": "no", +// "fee": "no", +// "authentication:none": "yes" export default MappingIRVE; diff --git a/mapping_geojson_to_osm_tags/mappings/engine.ts b/mapping_geojson_to_osm_tags/mappings/engine.ts index 3f0fa57d..e3f69c38 100644 --- a/mapping_geojson_to_osm_tags/mappings/engine.ts +++ b/mapping_geojson_to_osm_tags/mappings/engine.ts @@ -135,161 +135,185 @@ export default class { debugLog(' ------ convertProperty: pointKeyName', pointKeyName) // debugLog('convertProperty: mappingKeys', mappingKeys) - /** - * only use existing keys - */ - if (mappingKeys.indexOf(pointKeyName) > 0) { - let valueConvertedFromMapping = featurePoint.properties[pointKeyName] - let keyConvertedFromMapping = mappingKeys[mappingKeys.indexOf(pointKeyName)] - let mappingConfigOfTag = this.mapping_config.tags[pointKeyName] - debugLog('========== mappingConfigOfTag', mappingConfigOfTag) - debugLog('convertProperty: found element', pointKeyName, '=>', keyConvertedFromMapping, 'value : ', valueConvertedFromMapping) - let convertedValue = '' + if (this.mapping_config.add_not_mapped_tags_too && (mappingKeys.indexOf(pointKeyName) === -1)) { + /** + * add all unmapped tags is enabled + */ + newProperties[pointKeyName] = originalValue; - let typeOfConfigForKey = typeof mappingConfigOfTag - let isStringValue = typeOfConfigForKey === 'string' - let isConfigMappingObject = typeOfConfigForKey === 'object' + } else { + /** + * only use existing keys + */ + if (mappingKeys.indexOf(pointKeyName) > 0) { + let valueConvertedFromMapping = featurePoint.properties[pointKeyName] + let keyConvertedFromMapping = mappingKeys[mappingKeys.indexOf(pointKeyName)] + let mappingConfigOfTag = this.mapping_config.tags[pointKeyName] - debugLog('convertProperty: - typeofValue', typeOfConfigForKey) - debugLog('convertProperty: - pointKeyName', pointKeyName) - debugLog('convertProperty: - valueConvertedFromMapping', valueConvertedFromMapping) - debugLog('typeof valueConvertedFromMapping === \'string\'', typeOfConfigForKey) + debugLog('========== mappingConfigOfTag', mappingConfigOfTag) + debugLog('convertProperty: found element', pointKeyName, '=>', keyConvertedFromMapping, 'value : ', valueConvertedFromMapping) + let convertedValue = originalValue + + let typeOfConfigForKey = typeof mappingConfigOfTag + let isStringValue = typeOfConfigForKey === 'string' + let isConfigMappingObject = typeOfConfigForKey === 'object' + + debugLog('convertProperty: - typeofValue', typeOfConfigForKey) + debugLog('convertProperty: - pointKeyName', pointKeyName) + debugLog('convertProperty: - valueConvertedFromMapping', valueConvertedFromMapping) + debugLog('typeof valueConvertedFromMapping === \'string\'', typeOfConfigForKey) - debugLog('convertProperty: isStringValue?', valueConvertedFromMapping, isStringValue) - debugLog('convertProperty: isStringValue?', valueConvertedFromMapping, isStringValue) + debugLog('convertProperty: isStringValue?', valueConvertedFromMapping, isStringValue) + debugLog('convertProperty: isStringValue?', valueConvertedFromMapping, isStringValue) - debugLog('mappingConfigOfTag', mappingConfigOfTag) - debugLog('typeOfConfigForKey', typeOfConfigForKey) + debugLog('mappingConfigOfTag', mappingConfigOfTag) + debugLog('typeOfConfigForKey', typeOfConfigForKey) - if (isStringValue) { - debugLog('convertProperty: -- string value') - debugLog('convertProperty: -- string value') - if (this.isBooleanKey(pointKeyName)) { - let lowerValue = (valueConvertedFromMapping + '').toLowerCase() - debugLog('convertProperty: isBooleanKey: lowerValue', lowerValue) - convertedValue = this.truthyValues.indexOf(lowerValue) ? 'yes' : 'no' + /** + * conversion si la clé à une config d'une string, on ne change que la clé, pas la valeur + */ + if (isStringValue) { + debugLog('convertProperty: -- string value') + debugLog('convertProperty: -- string value') + console.log('convertProperty: -- simple conversion : ', pointKeyName, '=> ', mappingConfigOfTag, '_', originalValue, '=>', valueConvertedFromMapping) + debugLog('convertProperty: -- convertedValue', convertedValue) - } else { - debugLog('convertProperty: -- simple conversion : ', pointKeyName, '_', originalValue, '=>', valueConvertedFromMapping) convertedValue = valueConvertedFromMapping - } - debugLog('convertProperty: -- convertedValue', convertedValue) - if (convertedValue) { - newProperties[keyConvertedFromMapping] = convertedValue - } - } else { - debugLog('convertProperty: no string value') - } - if (isConfigMappingObject) { - let configObject = mappingConfigOfTag - - debugLog('convertProperty: is config object', configObject) - let newKey: any = '' + pointKeyName - let remove_original_key = false; - - if (configObject.key_converted) { - newKey = configObject.key_converted - } - // else { - // newKey = pointKeyName - // } - - /** - * conversion booléenne - */ - if (mappingValueObject.boolean_value_conversion) { - debugLog('convertProperty: is boolean_value_conversion') - debugLog('convertProperty: ==========> original value', originalValue) - if (this.truthyValues.indexOf(originalValue) !== -1) { - convertedValue = 'yes' - } - if (this.falsyValues.indexOf(originalValue) !== -1) { - convertedValue = 'no' + if (convertedValue) { + newProperties[mappingConfigOfTag] = convertedValue } } else { - debugLog('convertProperty: is NOT having boolean_value_conversion', mappingValueObject) + debugLog('convertProperty: no string value') } - /** - * gestion des valeurs conditionnelles - * nous pouvons renseigner une string ou un objet décrivant les transformations à réaliser - */ - if (configObject.conditional_values) { - debugLog('convertProperty: conditional_values__________', - configObject.conditional_values) + if (isConfigMappingObject) { + let configObject = mappingConfigOfTag - let keysConditionnalValues: any = Object.keys(configObject.conditional_values) + debugLog('convertProperty: is config object', configObject) + let newKey: any = '' + pointKeyName + let remove_original_key = false; - let isFoundValue = keysConditionnalValues.indexOf(originalValue) - debugLog('isFoundValue', isFoundValue) - debugLog('keysConditionnalValues', keysConditionnalValues) + if (configObject.key_converted) { + newKey = configObject.key_converted - /** ---------------------- - * gestion des valeurs conditionnelles - * ---------------------- */ - if (isFoundValue !== -1) { - let conditionnalConfig: any = configObject.conditional_values[keysConditionnalValues[isFoundValue]] + } - if (conditionnalConfig.tags_to_add) { - // on peut définir un ensemble de tags à rajouter - let tagKeys = Object.keys(conditionnalConfig.tags_to_add) - debugLog('conditionnalConfig.tags_to_add', conditionnalConfig.tags_to_add) - conditionnalConfig.tags_to_add.forEach((object: any, pair: any) => { - debugLog('object', object) - debugLog('pair', pair) - let key: any = Object.keys(object) - key = key[0] - let value = object[key] + if (configObject.truthy_value) { + // convertir la valeur, si elle est truthy, la transformer en ce que donne la propriété truthy_value + // exemple: le jeu de données dit que la colonne cable_t2_attache vaut "True", mais on veut le convertir en "1". + // on met donc truthy_value: '1' - debugLog('key', key) - debugLog('value', value) - newProperties[key] = value - }) + console.log('truthy_value', originalValue) + if (this.truthyValues.indexOf(originalValue) !== -1) { + convertedValue = configObject.truthy_value } - if (conditionnalConfig.truthy_value) { - // convertir la valeur, si elle est truthy, la transformer en ce que donne la propriété truthy_value - // exemple: le jeu de données dit que la colonne cable_t2_attache vaut "True", mais on veut le convertir en "1". - // on met donc truthy_value: '1' - if (this.truthyValues.indexOf(valueConvertedFromMapping) !== -1) { - convertedValue = conditionnalConfig.truthy_value - } - } - if (conditionnalConfig.falsy_value) { - if (this.falsyValues.indexOf(valueConvertedFromMapping) !== -1) { - convertedValue = conditionnalConfig.falsy_value - } + } + if (configObject.falsy_value) { + if (this.falsyValues.indexOf(originalValue) !== -1) { + convertedValue = configObject.falsy_value } + } - if (conditionnalConfig.transform_function) { - // une transformation de la valeur - // apply transformation to value - convertedValue = conditionnalConfig.transform_function(valueConvertedFromMapping) + /** + * conversion booléenne + */ + if (mappingValueObject.convert_to_boolean_value) { + debugLog('convertProperty: is boolean_value_conversion') + debugLog('convertProperty: ==========> original value', originalValue) + if (this.truthyValues.indexOf(originalValue) !== -1) { + convertedValue = 'yes' } - // use the value converted - else if (conditionnalConfig.value_converted) { - convertedValue = conditionnalConfig.value_converted + if (this.falsyValues.indexOf(originalValue) !== -1) { + convertedValue = 'no' } - - + } else { + debugLog('convertProperty: is NOT having boolean_value_conversion', mappingValueObject) } if (configObject.remove_original_key) { remove_original_key = true } - } + if (configObject.ignore_if_falsy && this.falsyValues.indexOf(originalValue) !== -1) { + remove_original_key = true + } + /** + * config pour une clé + * nous pouvons renseigner une string ou un objet décrivant les transformations à réaliser + */ + if (!remove_original_key && configObject.conditional_values) { + debugLog('convertProperty: conditional_values__________', + configObject.conditional_values) - debugLog('convertProperty: convertedValue ==========> {', newKey, ':', convertedValue, '}') - debugLog(' =============== remove_original_key', remove_original_key) - if (!remove_original_key && newKey && !configObject.ignore_this_data) { + let keysConditionnalValues: any = Object.keys(configObject.conditional_values) - debugLog('convertProperty: added') - newProperties[newKey] = convertedValue + let isFoundValue = keysConditionnalValues.indexOf(originalValue) + debugLog('isFoundValue', isFoundValue) + debugLog('keysConditionnalValues', keysConditionnalValues) + + + if (isFoundValue !== -1) { + let conditionnalConfig: any = configObject.conditional_values[keysConditionnalValues[isFoundValue]] + /** ---------------------- + * gestion des valeurs conditionnelles + * ---------------------- */ + if (conditionnalConfig.tags_to_add) { + // on peut définir un ensemble de tags à rajouter + let tagKeys = Object.keys(conditionnalConfig.tags_to_add) + debugLog('conditionnalConfig.tags_to_add', conditionnalConfig.tags_to_add) + conditionnalConfig.tags_to_add.forEach((object: any, pair: any) => { + debugLog('object', object) + debugLog('pair', pair) + let key: any = Object.keys(object) + key = key[0] + let value = object[key] + + debugLog('key', key) + debugLog('value', value) + newProperties[key] = value + }) + } + if (conditionnalConfig.truthy_value) { + // convertir la valeur, si elle est truthy, la transformer en ce que donne la propriété truthy_value + // exemple: le jeu de données dit que la colonne cable_t2_attache vaut "True", mais on veut le convertir en "1". + // on met donc truthy_value: '1' + if (this.truthyValues.indexOf(originalValue) !== -1) { + convertedValue = conditionnalConfig.truthy_value + } + } + if (conditionnalConfig.falsy_value) { + if (this.falsyValues.indexOf(originalValue) !== -1) { + convertedValue = conditionnalConfig.falsy_value + } + } + if (conditionnalConfig.transform_function) { + // une transformation de la valeur + // apply transformation to value + convertedValue = conditionnalConfig.transform_function(originalValue) + } + // use the value converted + else if (conditionnalConfig.value_converted) { + convertedValue = conditionnalConfig.value_converted + } + + + } + + } + + debugLog('convertProperty: convertedValue ==========> {', newKey, ':', convertedValue, '}') + debugLog(' =============== remove_original_key', remove_original_key) + if (!remove_original_key && newKey && convertedValue && !configObject.ignore_this_data) { + + debugLog('convertProperty: added') + newProperties[newKey] = convertedValue + } } } } + } } \ No newline at end of file diff --git a/mapping_geojson_to_osm_tags/mappings/mapping-config.type.ts b/mapping_geojson_to_osm_tags/mappings/mapping-config.type.ts index 0962eaf6..c7a39b4b 100644 --- a/mapping_geojson_to_osm_tags/mappings/mapping-config.type.ts +++ b/mapping_geojson_to_osm_tags/mappings/mapping-config.type.ts @@ -27,6 +27,7 @@ export interface BoundingBoxCoordinatesType{ export default interface MappingConfigType{ config_name: string, config_author: string, + add_not_mapped_tags_too: boolean, default_properties_of_point: object, tags: FeaturePropertyMappingConfigType } @@ -38,7 +39,7 @@ export default interface MappingConfigType{ */ export interface FeaturePropertyMappingConfigType{ [key:string]: any, - boolean_value_conversion?:boolean, + convert_to_boolean_value?:boolean, remove_original_key?:boolean, conditionnal_values?:ConditionnalValuesType, transform_function?:Function, diff --git a/mapping_geojson_to_osm_tags/output/my_converted_data_set.json b/mapping_geojson_to_osm_tags/output/my_converted_data_set.json index fec7fee3..9d1a6107 100644 --- a/mapping_geojson_to_osm_tags/output/my_converted_data_set.json +++ b/mapping_geojson_to_osm_tags/output/my_converted_data_set.json @@ -6,14 +6,25 @@ "geometry": { "type": "Point", "coordinates": [ - 2.198339, - 48.678843 + 4.822159, + 45.635079 ] }, "properties": { - "bicycle": "yes", - "scooter": "yes", - "motorcar": "no" + "operator": "ELECTRA", + "owner:ref:FR:SIREN": "891624884", + "email": "help@electra.com", + "network": "ELECTRA", + "ref:EU:EVSE": "FRELCPSDRPG", + "name": "Sérézin-du-Rhône - Peugeot", + "socket:type2_combo": "yes", + "fee": "false", + "authentication:none": "true", + "reservation": "yes", + "opening_hours": "24/7", + "wheelchair": "Accessibilité inconnue", + "start_date": "2022-10-27", + "source:date": "2023-07-29" } } ] diff --git a/mapping_geojson_to_osm_tags/output/my_converted_data_set_filtered_zipcode_91_small.json b/mapping_geojson_to_osm_tags/output/my_converted_data_set_filtered_zipcode_91_small.json index fdeff40d..2a7c2a83 100644 --- a/mapping_geojson_to_osm_tags/output/my_converted_data_set_filtered_zipcode_91_small.json +++ b/mapping_geojson_to_osm_tags/output/my_converted_data_set_filtered_zipcode_91_small.json @@ -6,13 +6,61 @@ "geometry": { "type": "Point", "coordinates": [ - 2.198339, - 48.678843 + 4.822159, + 45.635079 ] }, "properties": { - "goal": "jeu de données pour tester la mapping engine", - "station_deux_roues": "yes" + "nom_amenageur": "ELECTRA", + "siren_amenageur": "891624884", + "contact_amenageur": "help@electra.com", + "nom_operateur": "ELECTRA", + "contact_operateur": "help@electra.com", + "telephone_operateur": "", + "nom_enseigne": "ELECTRA", + "id_station_itinerance": "FRELCPSDRPG", + "id_station_local": "", + "nom_station": "S\u00e9r\u00e9zin-du-Rh\u00f4ne - Peugeot", + "implantation_station": "Station d\u00e9di\u00e9e \u00e0 la recharge rapide", + "adresse_station": "12 chemin d\u00e9partemental 12, Peugeot 69360 S\u00e9r\u00e9zin-du-Rh\u00f4ne", + "code_insee_commune": "69294", + "coordonneesXY": "[4.82215900,45.63507900]", + "nbre_pdc": "8", + "id_pdc_itinerance": "FRELCECURS", + "id_pdc_local": "", + "puissance_nominale": "225", + "prise_type_ef": "false", + "prise_type_2": "false", + "prise_type_combo_ccs": "true", + "prise_type_chademo": "false", + "prise_type_autre": "false", + "gratuit": "false", + "paiement_acte": "true", + "paiement_cb": "true", + "paiement_autre": "true", + "tarification": "", + "condition_acces": "Acc\u00e8s libre", + "reservation": "true", + "horaires": "24/7", + "accessibilite_pmr": "Accessibilit\u00e9 inconnue", + "restriction_gabarit": "Inconnu", + "station_deux_roues": "false", + "raccordement": "Direct", + "num_pdl": "N/A", + "date_mise_en_service": "2022-10-27", + "observations": "T\u00e9l\u00e9charger l'application ELECTRA pour r\u00e9server et payer sur go-electra.com", + "date_maj": "2023-07-29", + "cable_t2_attache": "", + "last_modified": "2023-07-29T03:05:24.360000+00:00", + "datagouv_dataset_id": "623ca46c13130c3228abd018", + "datagouv_resource_id": "e9bb3424-77cd-40ba-8bbd-5a19362d0365", + "datagouv_organization_or_owner": "electra", + "consolidated_longitude": 4.822159, + "consolidated_latitude": 45.635079, + "consolidated_code_postal": "69360", + "consolidated_commune": "S\u00e9r\u00e9zin-du-Rh\u00f4ne", + "consolidated_is_lon_lat_correct": true, + "consolidated_is_code_insee_verified": true } } ]