313 lines
11 KiB
TypeScript
313 lines
11 KiB
TypeScript
/**
|
|
* convertisseur de données de bornes de recharge électrique à partir de données Chargemap et open data Etalab
|
|
*/
|
|
import * as fs from 'fs'
|
|
|
|
import mappingConfigIRVE from './mappings/converters/configIRVE'
|
|
import mappingConfigIRVE_simple from './mappings/converters/mappingConfigIRVE_simple'
|
|
import mappingTest from './mappings/converters/configTest'
|
|
import mapping_engine from './mappings/engine'
|
|
import {BoundingBoxCoordinatesType, FeatureCollection} from "./mappings/mapping-config.type";
|
|
import utils from './mappings/utils'
|
|
|
|
|
|
const minimist = require('minimist')
|
|
|
|
const debugLog = utils.debugLog;
|
|
|
|
let use_mapping_engine = false
|
|
use_mapping_engine = true
|
|
|
|
|
|
|
|
let mini_arguments: any = minimist(process.argv.slice(2))
|
|
|
|
|
|
// let sourceFilePathGeoJson = './etalab_data/small.json'
|
|
let sourceFilePathGeoJson = './data_other/irve_osm_2023-08-30.json'
|
|
// let sourceFilePathGeoJson = './output/my_converted_data_set_filtered_zipcode_91.json'
|
|
// let sourceFilePathGeoJson = './output/my_converted_data_set_filtered_zipcode_91_small.json'
|
|
|
|
// wip filter
|
|
let filterOnBoundingBox = true
|
|
filterOnBoundingBox = false
|
|
|
|
let boundingBoxCoordinates: BoundingBoxCoordinatesType = {
|
|
xMin: 1.91,
|
|
xMax: 2.38,
|
|
yMin: 48.7,
|
|
yMax: 48.4,
|
|
}
|
|
|
|
let filterCoordinates = true
|
|
filterCoordinates = false
|
|
|
|
let enable_filter_on_department = true
|
|
enable_filter_on_department = false
|
|
|
|
let filterDepartment = 91
|
|
if (mini_arguments['department']) {
|
|
filterDepartment = mini_arguments['department']
|
|
enable_filter_on_department = true
|
|
}
|
|
if (mini_arguments['source']) {
|
|
sourceFilePathGeoJson = mini_arguments['source']
|
|
}
|
|
if (mini_arguments['engine']) {
|
|
use_mapping_engine = mini_arguments['engine']
|
|
}
|
|
let Mapping_engine:any;
|
|
|
|
if (mini_arguments['testingConfig']) {
|
|
console.log('testing')
|
|
Mapping_engine = new mapping_engine(mappingTest)
|
|
}else{
|
|
Mapping_engine = new mapping_engine(mappingConfigIRVE)
|
|
}
|
|
|
|
let filterZipCode = new RegExp(`^${filterDepartment}`)
|
|
let filterZipCodeAdresse = new RegExp(` ${filterDepartment}`)
|
|
let filteredName = ''
|
|
|
|
if (enable_filter_on_department) {
|
|
filteredName = '_filtered_zipcode_' + filterDepartment
|
|
} else if (filterOnBoundingBox) {
|
|
filteredName = '_filtered_bbox_' + boundingBoxCoordinates.xMin + '-' + boundingBoxCoordinates.xMax + '_' + boundingBoxCoordinates.yMin + '-' + boundingBoxCoordinates.yMax
|
|
}
|
|
let pointCounterMax: number = 1000000
|
|
|
|
let limitConversionToFirstPoint: boolean = false
|
|
// limitConversionToFirstPoint = true
|
|
|
|
if (limitConversionToFirstPoint) {
|
|
pointCounterMax = 1
|
|
}
|
|
let defaultPropertiesOfPoint: any = {
|
|
'amenity': 'charging_station'
|
|
}
|
|
|
|
let converted_geo_json: any = {
|
|
type: 'FeatureCollection',
|
|
features: []
|
|
}
|
|
|
|
|
|
let output_folder = 'output';
|
|
|
|
/**
|
|
* crée un fichier dans le dossier par défaut, output
|
|
* @param fileName
|
|
* @param fileContent
|
|
*/
|
|
function writeFile(fileName: string, fileContent: any) {
|
|
let write_path = `./${output_folder}/${fileName}`
|
|
debugLog('write file ', fileName , write_path)
|
|
|
|
|
|
return fs.writeFile(
|
|
write_path,
|
|
fileContent,
|
|
'utf8',
|
|
(err) => {
|
|
if (err) {
|
|
debugLog(`Error writing file: ${err}`)
|
|
} else {
|
|
debugLog(`File ${fileName} is written successfully!`)
|
|
}
|
|
}
|
|
)
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param sourceFilePath
|
|
* @param mapping
|
|
* @param pointCounterMax
|
|
* @param boundingBoxCoordinates
|
|
*/
|
|
function convertDataForIRVE(sourceFilePath: string, mapping: any, pointCounterMax: number, boundingBoxCoordinates: any) {
|
|
debugLog('convert data: source file from :', sourceFilePath)
|
|
|
|
fs.readFile(sourceFilePath, 'utf8', function (err, data) {
|
|
let point_counter = 0
|
|
let feature_points_after_filter: any = []
|
|
|
|
if (err) {
|
|
return debugLog(err)
|
|
}
|
|
let data_transformed: FeatureCollection = JSON.parse(data)
|
|
// debug('data keys ', Object.keys(dataTransformed))
|
|
// debugLog('properties of point 0', data_transformed.features[0])
|
|
|
|
if (data_transformed.features) {
|
|
|
|
debugLog('data found, features:', data_transformed.features.length)
|
|
|
|
// find interesting list of points to use
|
|
let list_of_points: any = data_transformed.features
|
|
debugLog('listOfPoints.length', list_of_points.length)
|
|
|
|
/**
|
|
* filtering
|
|
* TODO: do it in the engine
|
|
*/
|
|
// for each point from the data source, filter if we take it or not
|
|
list_of_points.forEach((feature_point: any) => {
|
|
|
|
let regex_filter_test_result = true
|
|
|
|
if (enable_filter_on_department) {
|
|
debugLog('filtre sur les départements activé')
|
|
regex_filter_test_result = (
|
|
filterZipCode.test(feature_point.properties.consolidated_code_postal)
|
|
||
|
|
filterZipCodeAdresse.test(feature_point.properties.adresse_station)
|
|
)
|
|
} else {
|
|
debugLog('pas de filtre sur les départements')
|
|
}
|
|
|
|
if (filterOnBoundingBox) {
|
|
debugLog('filtre sur les coordonnées bounding box activé')
|
|
|
|
let x = feature_point.properties.coordonneesXY[0]
|
|
let xMin = boundingBoxCoordinates.xMin
|
|
let xMax = boundingBoxCoordinates.xMax
|
|
let yMin = boundingBoxCoordinates.yMin
|
|
let yMax = boundingBoxCoordinates.yMax
|
|
|
|
let y = feature_point.properties.coordonneesXY[1]
|
|
regex_filter_test_result = (
|
|
(x >= xMin && x <= xMax)
|
|
&&
|
|
(y >= yMin && y <= yMax)
|
|
)
|
|
|
|
} else {
|
|
debugLog('pas de filtre sur les coordonnées bounding box')
|
|
}
|
|
|
|
|
|
// TODO add filter offset max
|
|
// filter points depending on zipcode
|
|
if (regex_filter_test_result) {
|
|
feature_points_after_filter.push(feature_point)
|
|
debugLog(' +1 point', point_counter)
|
|
point_counter++
|
|
}
|
|
|
|
})
|
|
|
|
/**
|
|
* conversion
|
|
*/
|
|
debugLog(' after filtering, feature_points_after_filter number of points: ', feature_points_after_filter.length)
|
|
feature_points_after_filter.forEach((feature_point: any) => {
|
|
// debugLog('featurePoint.properties.consolidated_code_postal', feature_point.properties.consolidated_code_postal)
|
|
debugLog('convert : work on 1 point')
|
|
// debugLog('convert :featurePoint', feature_point)
|
|
let mapped_point: any = {}
|
|
|
|
if (use_mapping_engine) {
|
|
// debugLog('convert :using mapping engine on feature point'
|
|
// , feature_point
|
|
// )
|
|
mapped_point = Mapping_engine.mapElementFromConf(feature_point)
|
|
debugLog('mapped_point', mapped_point)
|
|
} else {
|
|
debugLog('convert :using simple converter on feature point', feature_point)
|
|
mapped_point = mapElementFromConfSimple(feature_point, mapping)
|
|
}
|
|
// debugLog('mapped_point one point', mapped_point)
|
|
if (mapped_point) {
|
|
converted_geo_json.features.push(mapped_point)
|
|
debugLog('convert : added one point to converted_geo_json')
|
|
} else {
|
|
debugLog('convert : !!! there is no map one point')
|
|
}
|
|
})
|
|
// output new geojson
|
|
|
|
debugLog('convert : convertedGeoJson.features.length', converted_geo_json.features.length)
|
|
// write file on disk
|
|
if (converted_geo_json.features.length) {
|
|
|
|
let fileNameToWrite = 'my_converted_data_set' + filteredName + '.json'
|
|
console.log('features: ', converted_geo_json.features.length)
|
|
debugLog('convert : write file ', fileNameToWrite)
|
|
|
|
console.log('mapping_engine.stats', Mapping_engine.stats)
|
|
writeFile(fileNameToWrite, JSON.stringify(converted_geo_json, null, 2))
|
|
|
|
} else {
|
|
console.log('convert : no writing of file, because there is no converted feature')
|
|
}
|
|
|
|
return converted_geo_json
|
|
}
|
|
})
|
|
}
|
|
|
|
/**
|
|
* retuns the converted element from mapping config if present, null otherwise
|
|
*/
|
|
function mapElementFromConfSimple(featurePoint: any, mappingConfig: any) {
|
|
let mappingKeys = Object.keys(mappingConfig)
|
|
let featurePointPropertiesKeys = Object.keys(featurePoint.properties)
|
|
|
|
debugLog('keys', mappingKeys, featurePointPropertiesKeys)
|
|
|
|
let newProperties: any = defaultPropertiesOfPoint
|
|
|
|
// reinit properties of current point
|
|
let basePoint = Object.create(featurePoint)
|
|
basePoint.type = featurePoint.type
|
|
basePoint.geometry = featurePoint.geometry
|
|
basePoint.properties = newProperties
|
|
|
|
// apply new properties if found in mapping config
|
|
featurePointPropertiesKeys.forEach((pointKeyName: string) => {
|
|
|
|
if (mappingKeys.indexOf(pointKeyName) !== -1) {
|
|
// debugLog('found element', pointKeyName, '=>', mappingConfig[pointKeyName], 'value : ', featurePoint.properties[pointKeyName])
|
|
let convertedValue: any = ''
|
|
if (utils.isBooleanKey(pointKeyName)) {
|
|
|
|
let copyOfValue: any = '' + featurePoint.properties[pointKeyName]
|
|
if (typeof copyOfValue === typeof Object && copyOfValue.key_converted) {
|
|
copyOfValue = copyOfValue.key_converted
|
|
}
|
|
convertedValue = copyOfValue.toLowerCase() == 'true' ? 'yes' : 'no'
|
|
} else {
|
|
convertedValue = featurePoint.properties[pointKeyName]
|
|
}
|
|
|
|
if (convertedValue) {
|
|
let convertedKey: any = mappingConfig[pointKeyName]
|
|
newProperties[convertedKey] = convertedValue
|
|
}
|
|
}
|
|
})
|
|
|
|
debugLog('basePoint', basePoint)
|
|
return basePoint
|
|
}
|
|
|
|
if (use_mapping_engine) {
|
|
debugLog(' - using mapping engine')
|
|
debugLog(' - pointCounterMax', pointCounterMax)
|
|
|
|
Mapping_engine.setConfig(mappingConfigIRVE)
|
|
let currentMappingConfig = Mapping_engine.getConfig();
|
|
convertDataForIRVE(sourceFilePathGeoJson, currentMappingConfig, pointCounterMax, boundingBoxCoordinates)
|
|
|
|
} else {
|
|
|
|
let mappingConfigIRVE = mappingConfigIRVE_simple
|
|
convertDataForIRVE(sourceFilePathGeoJson, mappingConfigIRVE, pointCounterMax, boundingBoxCoordinates)
|
|
}
|
|
|
|
|
|
|
|
|