tykayn/scripts
1
0
Fork 0
Code Issues 1 Pull Requests Projects Releases Wiki Activity

add typings in engine, object to test

Browse Source
This commit is contained in:
Tykayn 2023-08-05 09:55:14 +02:00 committed by tykayn
parent 816435b005
commit 18696f6eec
9 changed files with 205040 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
mapping_geojson_to_osm_tags/convert_to_osm_tags.ts
View File

@ -7,17 +7,19 @@ import mappingConfigIRVE from './mappings/converters/configIRVE'
import mappingConfigIRVE_simple from './mappings/converters/mappingConfigIRVE_simple'
import mapping_engine from './mappings/engine'
import custom_utils from './mappings/utils'
import MappingConfigType from "./mappings/mapping-config.type";
import {debugLog} from './mappings/utils.ts'
import MappingConfigType, {BoundingBoxCoordinatesType} from "./mappings/mapping-config.type";
import utils from './mappings/utils.ts'
const debugLog = utils.debugLog;
let use_mappping_engine = false
// use_mappping_engine = true
use_mappping_engine = true
let Mapping_engine = new mapping_engine(mappingConfigIRVE)
let mini_arguments = minimist(process.argv.slice(2))
// let sourceFileChargemapJson = './chargemap_data/hurepoix.json'
let sourceFilePathGeoJson = './etalab_data/small.json'
// wip filter
@ -60,6 +62,7 @@ let pointCounterMax = 1000000
let limitConversionToFirstPoint = false
// limitConversionToFirstPoint = true
if (limitConversionToFirstPoint) {
pointCounterMax = 1
}
@ -162,7 +165,7 @@ function convertDataForIRVE (sourceFilePath, mapping: MappingConfigType, pointCo
// write file on disk
let fileNameToWrite = 'my_converted_data_set' + filteredName + '.json'
console.log('write file ', fileNameToWrite)
writeFile(fileNameToWrite, JSON.stringify(converted_geo_json, null, 2))
utils.writeFile(fileNameToWrite, JSON.stringify(converted_geo_json, null, 2))
debugLog('mapped output:', converted_geo_json.features)
@ -194,9 +197,9 @@ function mapElementFromConfSimple (featurePoint, mappingConfig) {
if (mappingKeys.indexOf(pointKeyName) !== -1) {
debugLog('found element', pointKeyName, '=>', mappingConfig[pointKeyName], 'value : ', featurePoint.properties[pointKeyName])
let convertedValue = ''
if (isBooleanKey(pointKeyName)) {
if ( utils.isBooleanKey(pointKeyName)) {
let copyOfValue = '' + featurePoint.properties[pointKeyName]
let copyOfValue :any = '' + featurePoint.properties[pointKeyName]
if (typeof copyOfValue === typeof Object && copyOfValue.key_converted) {
copyOfValue = copyOfValue.key_converted
}
@ -223,7 +226,8 @@ if (use_mappping_engine) {
} else {
convertDataForIRVE(sourceFilePathGeoJson, mappingConfigIRVE_simple, pointCounterMax, boundingBoxCoordinates)
mappingConfigIRVE = mappingConfigIRVE_simple
convertDataForIRVE(sourceFilePathGeoJson, mappingConfigIRVE, pointCounterMax, boundingBoxCoordinates)
}

21
mapping_geojson_to_osm_tags/data_other/testing/testing.json Normal file
View File

@ -0,0 +1,21 @@
{
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"geometry": {
"type": "Point",
"coordinates": [
4.822159,
45.635079
]
},
"properties": {
"nom_amenageur": "ELECTRA",
"siren_amenageur": "891624884",
"consolidated_commune": "S\u00e9r\u00e9zin-du-Rh\u00f4ne",
"consolidated_is_lon_lat_correct": true
}
}
]
}

141
mapping_geojson_to_osm_tags/mappings/engine.ts
View File

@ -2,19 +2,19 @@ import custom_utils from './utils'
const {debugLog} = custom_utils
let listOfBooleanKeys = Object.keys({
prise_type_ef: 'socket:typee',
prise_type_2: 'socket:type2',
prise_type_combo_ccs: 'socket:type2_combo',
prise_type_chademo: 'socket:chademo',
gratuit: 'fee',
paiement_acte: 'authentication:none',
paiement_cb: 'payment:credit_cards',
cable_t2_attache: 'socket:type2_cable',
})
let listOfBooleanKeys = [
"prise_type_ef",
"prise_type_2",
"prise_type_combo_ccs",
"prise_type_chademo",
"gratuit",
"paiement_acte",
"paiement_cb",
"cable_t2_attache"
]
export default class {
mapping_config = {}
mapping_config: any = {}
constructor(mappingConfig) {
this.setConfig(mappingConfig)
@ -26,7 +26,7 @@ export default class {
mapFeaturePoint(featurePointGeoJson) {
let geoJSONConvertedPoint = {}
let geoJSONConvertedPoint: any = {}
geoJSONConvertedPoint.properties = {...this.mapping_config.default_properties_of_point}
geoJSONConvertedPoint.type = featurePointGeoJson.type
geoJSONConvertedPoint.geometry = featurePointGeoJson.geometry
@ -53,18 +53,23 @@ export default class {
truthyValues = ['true', 'True', 'TRUE', '1', 1]
falsyValues = ['false', 'False', 'FALSE', '0', 0]
/**
* reduce number of features
* @param offsetCount
* @param listOfFeatures
*/
filterFeaturesByOffset(offsetCount: number, listOfFeatures): Array<any> {
let filteredList = listOfFeatures
// TODO
return filteredList
}
filterFeaturesByPropertyRegex(bboxConfig, listOfFeatures) {
console.log('bboxConfig', bboxConfig)
let filteredList = listOfFeatures
// TODO
return filteredList
}
// filterFeaturesByPropertyRegex(bboxConfig, listOfFeatures) {
// console.log('bboxConfig', bboxConfig)
// let filteredList = listOfFeatures
// // TODO
// return filteredList
// }
filterFeaturesByPropertyRegex(propertyName, criteriaRegex, listOfFeatures) {
let filteredList = listOfFeatures.filter(feature => {
@ -76,7 +81,7 @@ export default class {
/**
* retuns the converted element from mapping config if present, null otherwise
*/
mapElementFromConf(featurePoint) {
mapElementFromConf(featurePoint: any) {
if (!this.mapping_config) {
throw new Error('no config was loaded in the mapping engine. use setConfig(my_mapping_config) on this instance of mapping engine before using this.')
@ -142,55 +147,55 @@ export default class {
newProperties[this.mapping_config[pointKeyName]] = convertedValue
}
} else if (isConfigMappingObject) {
// let newKey = ''
// let configObject = valueConvertedFromMapping
//
// if (configObject.key_converted) {
// newKey = configObject.key_converted
// }
//
// /**
// * gestion des valeurs conditionnelles
// * nous pouvons renseigner une string ou un objet décrivant les transformations à réaliser
// */
// if (configObject.conditional_values) {
// let keysConditionnalValues = Object.keys(configObject.conditional_values)
// let isFoundValue = keysConditionnalValues.indexOf(valueConvertedFromMapping)
// if (isFoundValue !== -1) {
// let conditionnalConfig = keysConditionnalValues[isFoundValue]
//
// if (conditionnalConfig.tags_to_add) {
// // on peut définir un ensemble de tags à rajouter
// newProperties.push(...conditionnalConfig.tags_to_add)
// }
// if (conditionnalConfig.truthy_value) {
// // convertir la valeur, si elle est truthy, la transformer en ce que donne la propriété truthy_value
// // exemple: le jeu de données dit que la colonne cable_t2_attache vaut "True", mais on veut le convertir en "1".
// // on met donc truthy_value: '1'
// if (this.truthyValues.indexOf(valueConvertedFromMapping) !== -1) {
// convertedValue = conditionnalConfig.truthy_value
// }
// }
// if (conditionnalConfig.falsy_value) {
// if (this.falsyValues.indexOf(valueConvertedFromMapping) !== -1) {
// convertedValue = conditionnalConfig.falsy_value
// }
// }
// if (conditionnalConfig.transform_function) {
// // une transformation de la valeur
// // apply transformation to value
// convertedValue = conditionnalConfig.transform_function(valueConvertedFromMapping)
// }
// // use the value converted
// else if (conditionnalConfig.value_converted) {
// convertedValue = conditionnalConfig.value_converted
// }
// }
// }
//
// if (newKey && !configObject.ignore_this_data) {
// newProperties[newKey] = convertedValue
// }
let newKey = ''
let configObject = valueConvertedFromMapping
if (configObject.key_converted) {
newKey = configObject.key_converted
}
/**
* gestion des valeurs conditionnelles
* nous pouvons renseigner une string ou un objet décrivant les transformations à réaliser
*/
if (configObject.conditional_values) {
let keysConditionnalValues = Object.keys(configObject.conditional_values)
let isFoundValue = keysConditionnalValues.indexOf(valueConvertedFromMapping)
if (isFoundValue !== -1) {
let conditionnalConfig :any = keysConditionnalValues[isFoundValue]
if (conditionnalConfig.tags_to_add) {
// on peut définir un ensemble de tags à rajouter
newProperties.push(...conditionnalConfig.tags_to_add)
}
if (conditionnalConfig.truthy_value) {
// convertir la valeur, si elle est truthy, la transformer en ce que donne la propriété truthy_value
// exemple: le jeu de données dit que la colonne cable_t2_attache vaut "True", mais on veut le convertir en "1".
// on met donc truthy_value: '1'
if (this.truthyValues.indexOf(valueConvertedFromMapping) !== -1) {
convertedValue = conditionnalConfig.truthy_value
}
}
if (conditionnalConfig.falsy_value) {
if (this.falsyValues.indexOf(valueConvertedFromMapping) !== -1) {
convertedValue = conditionnalConfig.falsy_value
}
}
if (conditionnalConfig.transform_function) {
// une transformation de la valeur
// apply transformation to value
convertedValue = conditionnalConfig.transform_function(valueConvertedFromMapping)
}
// use the value converted
else if (conditionnalConfig.value_converted) {
convertedValue = conditionnalConfig.value_converted
}
}
}
if (newKey && !configObject.ignore_this_data) {
newProperties[newKey] = convertedValue
}
}
}
}

3
mapping_geojson_to_osm_tags/mappings/mapping-config.type.ts
View File

@ -1,4 +1,5 @@
export interface FeaturePropertyMappingConfigType{
[key:string]: any,
key_converted?:string,
truthy_value?:any,
}
@ -12,5 +13,5 @@ export default interface MappingConfigType{
config_name: string,
config_author: string,
default_properties_of_point: object,
tags: object
tags: FeaturePropertyMappingConfigType
}

41
mapping_geojson_to_osm_tags/mappings/utils.ts
View File

@ -1,13 +1,14 @@
import fs from 'node-fs'
import fs from 'node:fs'
let show_debug = 0
show_debug = 1
let output_folder = 'output';
/**
* faire un log
* @param message
* wrapper de log qui se montre uniquemnt si show_debug a été activé
* @param args
*/
function debugLog(...args : any) {
function debugLog(...args: any[]) {
if (!show_debug) {
return
}
@ -15,16 +16,16 @@ function debugLog(...args : any) {
}
let listOfBooleanKeys = Object.keys({
prise_type_ef: 'socket:typee',
prise_type_2: 'socket:type2',
prise_type_combo_ccs: 'socket:type2_combo',
prise_type_chademo: 'socket:chademo',
gratuit: 'fee',
paiement_acte: 'authentication:none',
paiement_cb: 'payment:credit_cards',
cable_t2_attache: 'socket:type2_cable',
})
let listOfBooleanKeys = [
"prise_type_ef",
"prise_type_2",
"prise_type_combo_ccs",
"prise_type_chademo",
"gratuit",
"paiement_acte",
"paiement_cb",
"cable_t2_attache"
]
/**
@ -32,16 +33,22 @@ let listOfBooleanKeys = Object.keys({
* @param pointKeyName
* @returns {boolean}
*/
function isBooleanKey(pointKeyName) {
function isBooleanKey(pointKeyName: string): boolean {
return listOfBooleanKeys.indexOf(pointKeyName) !== -1
}
function writeFile(fileName, fileContent) {
/**
* crée un fichier dans le dossier par défaut, output
* @param fileName
* @param fileContent
*/
function writeFile(fileName: string, fileContent: any) {
debugLog('write file ', fileName)
return fs.writeFile(
`./output/${fileName}`,
`./${output_folder}/${fileName}`,
fileContent,
'utf8',
(err) => {

1
mapping_geojson_to_osm_tags/osm_output/borne-présentes-dans-osm_france_entière.osm Normal file
View File

@ -0,0 +1 @@
<osm version="0.6" generator="geojsontoosm"/>

204851
mapping_geojson_to_osm_tags/osm_output/bornes_irve_existantes_présentes_dans_osm_2023-03-08.json Normal file
View File

File diff suppressed because it is too large Load Diff

74
mapping_geojson_to_osm_tags/tests/main.test.js
View File

@ -1,36 +1,68 @@
import finder from "../utils/finder.ts";
import { Jest as mockUpload } from '@jest/environment'
import mapping_engine from '../mappings/engine'
xdescribe('mapping properties with rich mapping engine', () => {
test('remove all properties when mapping says so', () => { });
test('maps simple key to key, and keep the same value', () => { });
test('ignore one value', () => { });
test('conditional value', () => { });
test('conditional transform', () => { });
test('conditional truthy transform', () => { });
test('conditional falsy transform', () => { });
describe('mapping properties with rich mapping engine', () => {
test('do not add properties at all when there is nothing in tags of the mapping config', () => {
})
test('remove all properties when mapping says so', () => {
let mappingRemoveAll = {
config_name: 'testing config',
config_author: 'tykayn <contact@cipherbliss.com>',
default_properties_of_point: {
'amenity': 'charging_station'
},
tags: {
nom_amenageur: {
key_converted: 'wheelchair',
conditional_values: {
'Accessibilité inconnue': {
ignore_this_data: true, // ne pas ajouter de tag si la valeur est égale à Accessibilité inconnue.
},
}
}
}
}
let Mapping_engine = new mapping_engine(mappingRemoveAll)
let mapped_point = {}
list_of_points.forEach(feature_point => {
mapped_point = Mapping_engine.mapElementFromConf(feature_point)
})
expect(mapped_point.properties).toBeFalsy();
})
test('maps simple key to key, and keep the same value', () => { })
test('ignore one value', () => { })
test('conditional value', () => { })
test('conditional transform', () => { })
test('conditional truthy transform', () => { })
test('conditional falsy transform', () => { })
})
xdescribe('filters points', () => {
test('filter no points', () => { });
test('filter only one point', () => { });
test('filter X number of points', () => { });
test('filter city points', () => { });
test('filter bounding box', () => { });
test('filter combo, city', () => { });
test('filter combo, city + bbox', () => { });
test('filter combo, city + bbox + offset', () => { });
test('filter no points', () => { })
test('filter only one point', () => { })
test('filter X number of points', () => { })
test('filter city points', () => { })
test('filter bounding box', () => { })
test('filter combo, city', () => { })
test('filter combo, city + bbox', () => { })
test('filter combo, city + bbox + offset', () => { })
})
/**
* thats a roadmap, yes
*/
xdescribe('infer domain of values from csv file', () => {
test('gets the list of unique values in column', () => { });
test('gets the list of unique values in column', () => { })
})
xdescribe('infer domain of values from geojson file', () => {
test('gets the list of unique values in column', () => { });
test('gets the list of unique values in column', () => { })
})
xdescribe('build mapping engine config from unique values', () => {
test('builds a valid mapping config', () => { });
test('builds a valid mapping config', () => { })
})