tykayn/mapping-geojson-osm
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

add tests

Browse Source
This commit is contained in:
Tykayn 2023-08-18 11:25:02 +02:00 committed by tykayn
parent 89b83bc73f
commit 2cb019ff5c
11 changed files with 148 additions and 1153992 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
convert_to_osm_tags.ts
View File

@ -5,6 +5,7 @@ import * as fs from 'fs'
import mappingConfigIRVE from './mappings/converters/configIRVE'
import mappingConfigIRVE_simple from './mappings/converters/mappingConfigIRVE_simple'
import mappingTest from './mappings/converters/configTest'
import mapping_engine from './mappings/engine'
import {BoundingBoxCoordinatesType, FeatureCollection} from "./mappings/mapping-config.type";
import utils from './mappings/utils'
@ -17,7 +18,7 @@ const debugLog = utils.debugLog;
let use_mapping_engine = false
use_mapping_engine = true
let Mapping_engine = new mapping_engine(mappingConfigIRVE)
let mini_arguments: any = minimist(process.argv.slice(2))
@ -55,6 +56,14 @@ if (mini_arguments['source']) {
if (mini_arguments['engine']) {
use_mapping_engine = mini_arguments['engine']
}
let Mapping_engine:any;
if (mini_arguments['testingConfig']) {
console.log('testing')
Mapping_engine = new mapping_engine(mappingTest)
}else{
Mapping_engine = new mapping_engine(mappingConfigIRVE)
}
let filterZipCode = new RegExp(`^${filterDepartment}`)
let filterZipCodeAdresse = new RegExp(` ${filterDepartment}`)
@ -285,8 +294,10 @@ function mapElementFromConfSimple(featurePoint: any, mappingConfig: any) {
if (use_mapping_engine) {
debugLog(' - using mapping engine')
debugLog(' - pointCounterMax', pointCounterMax)
Mapping_engine.setConfig(mappingConfigIRVE)
convertDataForIRVE(sourceFilePathGeoJson, mappingConfigIRVE, pointCounterMax, boundingBoxCoordinates)
// Mapping_engine.setConfig(mappingConfigIRVE)
let currentMappingConfig = Mapping_engine.getConfig();
convertDataForIRVE(sourceFilePathGeoJson, currentMappingConfig, pointCounterMax, boundingBoxCoordinates)
} else {

79
data_other/testing/mappings_to_test.ts
View File

@ -1,34 +1,63 @@
import MappingConfigType from "../../mappings/mapping-config.type";
import MappingConfigType, {FeaturePropertyMappingConfigType} from "../../mappings/mapping-config.type";
/**
* configurations de mapping pour les cas de tests
*/
export const mappingRemoveAll: MappingConfigType = {
config_name: 'testing config',
config_name: 'testing config mappingRemoveAll',
config_author: 'tykayn <contact@cipherbliss.com>',
default_properties_of_point: {
'amenity': 'charging_station'
},
tags: {
nom_amenageur: {
key_converted: 'autre_nom_amenageur',
conditional_values: {
'Accessibilité inconnue': {
ignore_this_data: true, // ne pas ajouter de tag si la valeur est égale à Accessibilité inconnue.
},
}
}
}
default_properties_of_point: {},
tags: {},
add_not_mapped_tags_too: false,
source: {},
filters: {},
}
export const mappingIgnore: MappingConfigType = {
config_name: 'testing config',
export const mappingSame: MappingConfigType = {
config_name: 'testing config mappingSame',
config_author: 'tykayn <contact@cipherbliss.com>',
default_properties_of_point: {
'amenity': 'charging_station'
},
default_properties_of_point: {},
tags: {
nom_amenageur: {
ignore_this_data: true,
}
}
}
equal : 'equal'
},
add_not_mapped_tags_too: false,
source: {},
filters: {},
}
export const mappingName: MappingConfigType = {
config_name: 'testing config mappingName',
config_author: 'tykayn <contact@cipherbliss.com>',
default_properties_of_point: {},
tags: {
nom_amenageur : 'name'
},
add_not_mapped_tags_too: true,
source: {},
filters: {},
}
export const mappingPhone: MappingConfigType = {
config_name: 'testing config mappingPhone',
config_author: 'tykayn <contact@cipherbliss.com>',
default_properties_of_point: {},
tags: {
telephone_operateur: {
key_converted: 'phone',
convert_to_phone: true, // convertit en yes ou no
},
},
add_not_mapped_tags_too: false,
source: {},
filters: {},
}
// export const mappingIgnore: MappingConfigType = {
// config_name: 'testing config',
// config_author: 'tykayn <contact@cipherbliss.com>',
// default_properties_of_point: {
// 'amenity': 'charging_station'
// },
// tags: {
// nom_amenageur: {
// ignore_this_data: true,
// }
// }
// }

4
data_other/testing/testing.json
View File

@ -11,8 +11,10 @@
]
},
"properties": {
"nom_amenageur": "ELECTRA",
"nom_amenageur": "Bob",
"siren_amenageur": "891624884",
"telephone_operateur": "0123456789",
"equal": "same value",
"consolidated_commune": "S\u00e9r\u00e9zin-du-Rh\u00f4ne",
"consolidated_is_lon_lat_correct": true
}

1
etalab_data/irve_bornes_recharge/.~lock.irve-junglebus.csv#
View File

@ -1 +0,0 @@
,tykayn,spaceship,11.08.2023 21:43,file:///home/tykayn/.config/libreoffice/4;

10
make_variance_from_csv.ts
View File

@ -11,17 +11,21 @@ import {parse} from 'csv'
const fs = require('fs')
const minimist = require('minimist')
let mini_arguments: any = minimist(process.argv.slice(2))
interface VarianceType {
[key: string]: Array<string>
}
let csv_content = 'variance de dataset\n';
let separator = ';';
// let columns_headings = [];
let data_variance: VarianceType = {};
const inputPath = './etalab_data/toilettes/sanisettesparis_reworked.csv'
// const inputPath = './etalab_data/small.csv'
// const inputPath = './etalab_data/toilettes/sanisettesparis_reworked.csv'
const inputPath = './etalab_data/small.csv'
if (mini_arguments['source']) {
inputPath = mini_arguments['source']
}
let columns_headings: Array<string> = [];
let lines_count = 0;
let longest_variance_count = 0;

5
mappings/converters/configIRVE.ts
View File

@ -39,7 +39,10 @@ const MappingIRVE: MappingConfigType = {
nom_amenageur: 'operator',
siren_amenageur: 'owner:ref:FR:SIREN',
nom_operateur: 'operator',
telephone_operateur: 'phone',
telephone_operateur: {
key_converted: 'phone',
convert_to_phone: true, // convertit en yes ou no
},
contact_operateur: 'email', // ici, on souhaite convertir la clé contact_operateur=bidule en email=bidule
// id_station_itinerance: 'ref:EU:EVSE',

21
mappings/converters/configTest.ts Normal file
View File

@ -0,0 +1,21 @@
/**
* plan de conversion des clés du jeu de données vers les tags OSM
* détail dans le tableau
* https://wiki.openstreetmap.org/wiki/France/data.gouv.fr/Bornes_de_Recharge_pour_V%C3%A9hicules_%C3%89lectriques
*/
import MappingConfigType from "../mapping-config.type";
const MappingTest: MappingConfigType = {
config_name: 'testing config mappingSame',
config_author: 'tykayn <contact@cipherbliss.com>',
default_properties_of_point: {},
tags: {
nom_amenageur : 'name'
},
add_not_mapped_tags_too: false,
source: {},
filters: {},
}
export default MappingTest;

15
mappings/engine.ts
View File

@ -18,13 +18,19 @@ export default class {
mapping_config: any = {}
constructor(mappingConfig: MappingConfigType) {
this.setConfig(mappingConfig)
}
setConfig(mappingConfig: MappingConfigType) {
console.log('load config', mappingConfig.config_name)
this.mapping_config = mappingConfig
}
getConfig() {
return this.mapping_config
}
mapFeaturePoint(featurePointGeoJson: any) {
let geoJSONConvertedPoint: any = {}
@ -85,7 +91,7 @@ export default class {
mapElementFromConf(featurePoint: any): any {
// debugLog('mapElementFromConf: mapElementFromConf', featurePoint)
if (!this.mapping_config) {
throw new Error('no config was loaded in the mapping engine. use setConfig(my_mapping_config) on this instance of mapping engine before using this.')
throw new Error('no config was loaded in the mapping engine. use setConfig(my_mapping_config) on this instance of mapping engine before using this. Your config should be typed to MappingConfigType Type.')
}
debugLog('mapElementFromConf: config_name', this.mapping_config.config_name)
@ -234,6 +240,11 @@ export default class {
} else {
debugLog('convertProperty: is NOT having boolean_value_conversion', mappingValueObject)
}
if (configObject.convert_to_phone) {
console.log('originalValue', originalValue.substring(1))
let temp = `${'' + originalValue.substring(1)}`
convertedValue = `+33 ${temp}`;
}
if (configObject.remove_original_key) {
remove_original_key = true
}
@ -311,7 +322,7 @@ export default class {
}
debugLog('convertProperty: convertedValue ==========> {', newKey, ':', convertedValue, '}')
debugLog(' =============== remove_original_key',newKey, remove_original_key)
debugLog(' =============== remove_original_key', newKey, remove_original_key)
if (!remove_original_key && newKey && convertedValue && !configObject.ignore_this_data) {
debugLog('convertProperty: added')

1153950
output/my_converted_data_set.json
View File

File diff suppressed because it is too large Load Diff

3
package.json
View File

@ -19,7 +19,8 @@
"simple": "ts-node convert_to_osm_tags.ts",
"chargemap": "ts-node chargemap.ts",
"filter": "node convert_to_osm_tags.ts --department=974 --engine=true",
"test": "jest --coverage --watch"
"test": "jest --coverage --watch",
"testing": "ts-node convert_to_osm_tags.ts --source=\"./data_other/testing/testing.json\" --testingConfig --engine"
},
"dependencies": {
"csv": "^6.3.1",

35
tests/main.test.js
View File

@ -1,5 +1,5 @@
import mapping_engine from '../mappings/engine.ts'
import { mappingRemoveAll } from '../data_other/testing/mappings_to_test'
import { mappingRemoveAll, mappingSame, mappingPhone, mappingName } from '../data_other/testing/mappings_to_test'
import MappingConfigType, {BoundingBoxCoordinatesType, FeatureCollection} from "../mappings/mapping-config.type";
const testingGeoJson = require('../data_other/testing/testing.json')
@ -13,21 +13,44 @@ describe('mapping properties with rich mapping engine', () => {
// test('do not add properties at all when there is nothing in tags of the mapping config', () => {
//
// })
let feature_to_test = testingGeoJson.features[0];
test('remove all properties when mapping says so', () => {
let Mapping_engine = new mapping_engine(mappingRemoveAll)
let mapped_point = Mapping_engine.mapElementFromConf(testingGeoJson.features[0])
let mapped_point = Mapping_engine.mapElementFromConf(feature_to_test)
expect(mapped_point).toBeTruthy()
// expect(mapped_point.properties).toBe([])
expect(Object.keys(mapped_point.properties)).toStrictEqual([])
})
// test('maps simple key to key, and keep the same value', () => { })
// test('maps simple key to key, and keep the same value', () => {
// let Mapping_engine = new mapping_engine(mappingSame)
// let mapped_point = Mapping_engine.mapElementFromConf(feature_to_test)
// console.log('mapped_point', mapped_point)
// expect(mapped_point.properties).toBe({
// equal : "same value"
// })
// })
test('retrieve config name in mapping engine', () => {
let Mapping_engine = new mapping_engine(mappingRemoveAll)
expect(Mapping_engine.getConfig().config_name).toBe('testing config mappingRemoveAll')
})
test('maps nom_amenageur to name, and keep the same value', () => {
let Mapping_engine = new mapping_engine(mappingName)
let mapped_point = Mapping_engine.mapElementFromConf(feature_to_test)
expect(Mapping_engine.getConfig().config_name).toBe('testing config mappingName')
expect(mapped_point).toBe({
name : "Bob"
})
})
// test('ignore one value', () => { })
// test('conditional value', () => { })
// test('conditional transform', () => { })
// test('conditional truthy transform', () => { })
// test('conditional falsy transform', () => { })
// test('conditional phone transform', () => {
// let Mapping_engine = new mapping_engine(mappingPhone)
// let mapped_point = Mapping_engine.mapElementFromConf(feature_to_test)
// expect(mapped_point.properties).toBe({phone:'+33123456789'})
// })
})
xdescribe('filters points', () => {