refacto to use mapping engine class

This commit is contained in:
Tykayn 2023-07-30 18:37:07 +02:00 committed by tykayn
parent 7175a95ed1
commit 5cb3cc02b1
5 changed files with 357 additions and 244 deletions

View File

@ -4,226 +4,238 @@
import fs from 'node-fs'
import minimist from 'minimist'
import mappingConfigIRVE from 'mappings/configIRVE'
import mapping_engine from "./mappings/engine";
import {debugLog} from "./mappings/utils";
// let debugLog = custom_utils.debugLog
let mapping_engine = new mapping_engine(mappingConfigIRVE);
let use_mappping_engine = false;
let mini_arguments = minimist(process.argv.slice(2))
let show_debug = 0
// let sourceFileChargemapJson = './chargemap_data/hurepoix.json'
let sourceFilePathGeoJson = './etalab_data/latest.json'
// wip filter
let filterOnBoundingBox = true
filterOnBoundingBox = false
let boundingBoxCoordinates = {
xMin: 1.91,
xMax: 2.38,
yMin: 48.7,
yMax: 48.4,
xMin: 1.91,
xMax: 2.38,
yMin: 48.7,
yMax: 48.4,
}
let filterCoordinates = true
let enableFilterOnDepartment = true
let enable_filter_on_department = true
let filterDepartment = 91
if (mini_arguments['department']) {
filterDepartment = mini_arguments['department']
filterDepartment = mini_arguments['department']
}
if (mini_arguments['source']) {
sourceFilePathGeoJson = mini_arguments['source']
}
let filterZipCode = new RegExp(`^${filterDepartment}`)
let filterZipCodeAdresse = new RegExp(` ${filterDepartment}`)
let filteredName = ''
if (enableFilterOnDepartment) {
filteredName = '_filtered_zipcode_' + filterDepartment
if (enable_filter_on_department) {
filteredName = '_filtered_zipcode_' + filterDepartment
} else if (filterOnBoundingBox) {
filteredName = '_filtered_bbox_' + boundingBoxCoordinates.xMin + '-' + boundingBoxCoordinates.xMax + '_' + boundingBoxCoordinates.yMin + '-' + boundingBoxCoordinates.yMax
filteredName = '_filtered_bbox_' + boundingBoxCoordinates.xMin + '-' + boundingBoxCoordinates.xMax + '_' + boundingBoxCoordinates.yMin + '-' + boundingBoxCoordinates.yMax
}
let pointCounterMax = 1000000
let limitConversionToFirstPoint = false
// limitConversionToFirstPoint = true;
if (limitConversionToFirstPoint) {
pointCounterMax = 1
pointCounterMax = 1
}
// let sourceFileChargemapJson = './chargemap_data/hurepoix.json'
let sourceFileIRVEGeoJson = './etalab_data/latest.json'
let defaultPropertiesOfPoint = {
'amenity': 'charging_station'
'amenity': 'charging_station'
}
let listOfBooleanKeys = Object.keys({
prise_type_ef: 'socket:typee',
prise_type_2: 'socket:type2',
prise_type_combo_ccs: 'socket:type2_combo',
prise_type_chademo: 'socket:chademo',
gratuit: 'fee',
paiement_acte: 'authentication:none',
paiement_cb: 'payment:credit_cards',
cable_t2_attache: 'socket:type2_cable',
prise_type_ef: 'socket:typee',
prise_type_2: 'socket:type2',
prise_type_combo_ccs: 'socket:type2_combo',
prise_type_chademo: 'socket:chademo',
gratuit: 'fee',
paiement_acte: 'authentication:none',
paiement_cb: 'payment:credit_cards',
cable_t2_attache: 'socket:type2_cable',
})
function debugLog (message) {
if (!show_debug) {
return
}
console.log('debug: ', ...message)
let converted_geo_json = {
type: 'FeatureCollection',
features: []
}
/**
*
* @param sourceFilePath
* @param mapping
* @param pointCounterMax
* @param boundingBoxCoordinates
*/
function convertDataForIRVE(sourceFilePath, mapping, pointCounterMax, boundingBoxCoordinates) {
debugLog('convertDataFromChargemap from ', sourceFilePath)
fs.readFile(sourceFilePath, 'utf8', function (err, data) {
let point_counter = 0
if (err) {
return debugLog(err)
}
let data_transformed = JSON.parse(data)
// debug('data keys ', Object.keys(dataTransformed))
debugLog('debug: properties of point 0', data_transformed.features[0])
if (data_transformed.features) {
debugLog('data found, features:', data_transformed.features.length)
// find interesting list of points to use
let list_of_points = data_transformed.features
// for each point from the data source, convert with the mapping
console.log('listOfPoints.length', list_of_points.length)
list_of_points.forEach(feature_point => {
let regex_filter_test_result = true
if (enable_filter_on_department) {
console.log('filtre sur les départements activé')
regex_filter_test_result = (
filterZipCode.test(feature_point.properties.consolidated_code_postal)
||
filterZipCodeAdresse.test(feature_point.properties.adresse_station)
)
} else if (filterOnBoundingBox) {
console.log('filtre sur les coordonnées activé')
let x = feature_point.properties.coordonneesXY[0]
let xMin = boundingBoxCoordinates.xMin
let xMax = boundingBoxCoordinates.xMax
let yMin = boundingBoxCoordinates.yMin
let yMax = boundingBoxCoordinates.yMax
let y = feature_point.properties.coordonneesXY[1]
regex_filter_test_result = (
(x >= xMin && x <= xMax)
&&
(y >= yMin && y <= yMax)
)
}
// filter points depending on zipcode
if (filterCoordinates && regex_filter_test_result) {
debugLog('featurePoint.properties.consolidated_code_postal', feature_point.properties.consolidated_code_postal)
// limit results number of points
// if (pointcounter < pointCounterMax) {
debugLog('add point')
debugLog('featurePoint', feature_point)
let mapped_point = {};
if (use_mappping_engine) {
mapping_engine.setConfig(mapping)
mapped_point = mapping_engine.mapFeaturePoint(feature_point)
} else {
mapped_point = mapElementFromConfSimple(feature_point, mapping)
}
debugLog('map one point', feature_point, mapped_point)
if (mapped_point) {
converted_geo_json.features.push(mapped_point)
}
}
// }
point_counter++
})
// output new geojson
console.log('convertedGeoJson.features.length', converted_geo_json.features.length)
// write file on disk
let fileNameToWrite = 'my_converted_data_set' + filteredName + '.json'
console.log('write file ', fileNameToWrite)
writeJsonFile(fileNameToWrite, JSON.stringify(converted_geo_json, null, 2))
debugLog('mapped output:', converted_geo_json.features)
return converted_geo_json
}
})
}
function convertDataForIRVE (sourceFilePath, mapping, pointCounterMax, boundingBoxCoordinates) {
debugLog('convertDataFromChargemap from ', sourceFilePath)
let convertedGeoJson = {
type: 'FeatureCollection',
features: []
}
fs.readFile(sourceFilePath, 'utf8', function (err, data) {
let pointcounter = 0
if (err) {
return debugLog(err)
}
let dataTransformed = JSON.parse(data)
// debug('data keys ', Object.keys(dataTransformed))
debugLog('debug: properties of point 0', dataTransformed.features[0])
if (dataTransformed.features) {
debugLog('data found, features:', dataTransformed.features.length)
// find interesting list of points to use
let listOfPoints = dataTransformed.features
// for each point from the data source, convert with the mapping
console.log('listOfPoints.length', listOfPoints.length)
listOfPoints.forEach(featurePoint => {
let regextestresult = true
if (enableFilterOnDepartment) {
console.log('filtre sur les départements activé')
regextestresult = (
filterZipCode.test(featurePoint.properties.consolidated_code_postal)
||
filterZipCodeAdresse.test(featurePoint.properties.adresse_station)
)
} else if (filterOnBoundingBox) {
console.log('filtre sur les coordonnées activé')
let x = featurePoint.properties.coordonneesXY[0]
let xMin = boundingBoxCoordinates.xMin
let xMax = boundingBoxCoordinates.xMax
let yMin = boundingBoxCoordinates.yMin
let yMax = boundingBoxCoordinates.yMax
let y = featurePoint.properties.coordonneesXY[1]
regextestresult = (
(x >= xMin && x <= xMax)
&&
(y >= yMin && y <= yMax)
)
}
// filter points depending on zipcode
if (filterCoordinates && regextestresult) {
debugLog('featurePoint.properties.consolidated_code_postal', featurePoint.properties.consolidated_code_postal)
// limit results number of points
// if (pointcounter < pointCounterMax) {
debugLog('add point')
debugLog('featurePoint', featurePoint)
let mappedPoint = mapElementFromConf(featurePoint, mapping)
debugLog('map one point', featurePoint, mappedPoint)
if (mappedPoint) {
convertedGeoJson.features.push(mappedPoint)
}
}
// }
pointcounter++
})
// output new geojson
console.log('convertedGeoJson.features.length', convertedGeoJson.features.length)
// write file on disk
let fileNameToWrite = 'my_converted_data_set' + filteredName + '.json'
console.log('write file ', fileNameToWrite)
writeJsonFile(fileNameToWrite, JSON.stringify(convertedGeoJson, null, 2))
debugLog('mapped output:', convertedGeoJson.features)
return convertedGeoJson
}
})
}
function isBooleanKey (pointKeyName) {
return listOfBooleanKeys.indexOf(pointKeyName) !== -1
}
/**
* retuns the converted element from mapping config if present, null otherwise
*/
function mapElementFromConfSimple(featurePoint, mappingConfig)
{
let mappingKeys = Object.keys(mappingConfig)
let featurePointPropertiesKeys = Object.keys(featurePoint.properties)
function mapElementFromConf (featurePoint, mappingConfig) {
let mappingKeys = Object.keys(mappingConfig)
let featurePointPropertiesKeys = Object.keys(featurePoint.properties)
debugLog('keys', mappingKeys, featurePointPropertiesKeys)
debugLog('keys', mappingKeys, featurePointPropertiesKeys)
let newProperties = defaultPropertiesOfPoint
let newProperties = defaultPropertiesOfPoint
// reinit properties of current point
let basePoint = Object.create(featurePoint)
basePoint.type = featurePoint.type
basePoint.geometry = featurePoint.geometry
basePoint.properties = newProperties
// reinit properties of current point
let basePoint = Object.create(featurePoint)
basePoint.type = featurePoint.type
basePoint.geometry = featurePoint.geometry
basePoint.properties = newProperties
// apply new properties if found in mapping config
featurePointPropertiesKeys.forEach(pointKeyName => {
// apply new properties if found in mapping config
featurePointPropertiesKeys.forEach(pointKeyName => {
if (mappingKeys.indexOf(pointKeyName) !== -1) {
debugLog('found element', pointKeyName, '=>', mappingConfig[pointKeyName], 'value : ', featurePoint.properties[pointKeyName])
let convertedValue = ''
if (isBooleanKey(pointKeyName)) {
if (mappingKeys.indexOf(pointKeyName) !== -1) {
debugLog('found element', pointKeyName, '=>', mappingConfig[pointKeyName], 'value : ', featurePoint.properties[pointKeyName])
let convertedValue = ''
if (isBooleanKey(pointKeyName)) {
convertedValue = featurePoint.properties[pointKeyName].toLowerCase() == 'true' ? 'yes' : 'no'
} else {
convertedValue = featurePoint.properties[pointKeyName]
}
convertedValue = featurePoint.properties[pointKeyName].toLowerCase() == 'true' ? 'yes' : 'no'
} else {
convertedValue = featurePoint.properties[pointKeyName]
}
if (convertedValue) {
newProperties[mappingConfig[pointKeyName]] = convertedValue
}
}
})
if (convertedValue) {
newProperties[mappingConfig[pointKeyName]] = convertedValue
}
}
})
debugLog('basePoint', basePoint)
return basePoint
debugLog('basePoint', basePoint)
return basePoint
}
function writeJsonFile (fileName, fileContent) {
debugLog('write file ', fileName)
function writeJsonFile(fileName, fileContent) {
debugLog('write file ', fileName)
return fs.writeFile(
`./output/${fileName}`,
fileContent,
'utf8',
(err) => {
if (err) {
debugLog(`Error writing file: ${err}`)
} else {
debugLog(`File ${fileName} is written successfully!`)
}
}
)
return fs.writeFile(
`./output/${fileName}`,
fileContent,
'utf8',
(err) => {
if (err) {
debugLog(`Error writing file: ${err}`)
} else {
debugLog(`File ${fileName} is written successfully!`)
}
}
)
}
console.log('pointCounterMax', pointCounterMax)
convertDataForIRVE(sourceFileIRVEGeoJson, mappingConfigIRVE, pointCounterMax, boundingBoxCoordinates)
// convertDataFromChargemap(sourceFileChargemapJson, mappingConfigIRVE)
convertDataForIRVE(sourceFilePathGeoJson, mappingConfigIRVE, pointCounterMax, boundingBoxCoordinates)

View File

@ -2,7 +2,7 @@
D'après le jeu de données Etalab
## liste des opérateurs
## liste des 310 opérateurs
270 AGENCY
2Ed Coutances
2F Production

View File

@ -5,69 +5,82 @@
*/
export default {
// ******* nombres
nbre_pdc: 'capacity',
// ******* textes
amenity: 'amenity', // conserver le tag de base
capacity: 'capacity', // conserver le tag de base
nom_amenageur: 'operator',
siren_amenageur: 'owner:ref:FR:SIREN',
nom_operateur: 'operator',
telephone_operateur: 'phone',
contact_operateur: 'email', // ici, on souhaite convertir la clé contact_operateur=bidule en email=bidule
config_name: "IRVE config",
config_author: "tykayn <contact@cipherbliss.com>",
default_properties_of_point: {
'amenity': 'charging_station'
},
tags: {
id_station_itinerance: 'ref:EU:EVSE',
id_station_local: 'ref',
// ******* nombres
nbre_pdc: 'capacity',
// ******* textes
amenity: 'amenity', // conserver le tag de base
capacity: 'capacity', // conserver le tag de base
nom_amenageur: 'operator',
siren_amenageur: 'owner:ref:FR:SIREN',
nom_operateur: 'operator',
telephone_operateur: 'phone',
contact_operateur: 'email', // ici, on souhaite convertir la clé contact_operateur=bidule en email=bidule
gratuit: 'fee',
paiement_acte: 'authentication:none',
id_station_itinerance: 'ref:EU:EVSE',
id_station_local: 'ref',
reservation: 'reservation',
observations: 'note',
nom_station: 'name',
nom_enseigne: 'network',
gratuit: 'fee',
paiement_acte: 'authentication:none',
// ******* dates
date_mise_en_service: 'start_date',
date_maj: 'source:date',
reservation: 'reservation',
observations: 'note',
nom_station: 'name',
nom_enseigne: 'network',
// TODO gestion des types dont on doit convertir la valeur
// ******** champs booléens
cable_t2_attache: 'socket:type2_cable',
prise_type_ef: 'socket:typee',
prise_type_2: 'socket:type2',
prise_type_combo_ccs: 'socket:type2_combo',
prise_type_chademo: 'socket:chademo',
// ******* dates
date_mise_en_service: 'start_date',
date_maj: 'source:date',
// ******** champs plus complexes
horaires: 'opening_hours', // déjà au bon format
// accessibilite_pmr: 'wheelchair',
// paiement_cb: 'payment:credit_cards',
accessibilite_pmr: {
key_converted: "wheelchair",
conditional_values: {
"Accessibilité inconnue": {
value_converted: "",
ignore_this_data: true, // ne pas ajouter de tag si la valeur est égale à Accessibilité inconnue.
// transform_function : (original_value) => original_value.toLowerCase(),
},
"Accessible mais non réservé PMR": {
value_converted: ""
},
"Réservé PMR": {
value_converted: ""
},
"Non accessible": {
value_converted: "no"
},
"Mo-Fr 08:30-12:00,Mo-Fr 14:00-19:00,Sat 09:00-18:30": {
value_converted: "Mo-Fr 08:30-12:00,Mo-Fr 14:00-19:00,Sat 09:00-18:30"
}
"24/7": {
value_converted: ""
// TODO gestion des types dont on doit convertir la valeur
// ******** champs booléens
cable_t2_attache:
{
key_converted: 'socket:type2_cable',
// cable_t2_attache
truthy_value: '1'
}
,
prise_type_ef: 'socket:typee',
prise_type_2: 'socket:type2',
prise_type_combo_ccs: 'socket:type2_combo',
prise_type_chademo: 'socket:chademo',
// ******** champs plus complexes
horaires: 'opening_hours', // déjà au bon format
// accessibilite_pmr: 'wheelchair',
// paiement_cb: 'payment:credit_cards',
accessibilite_pmr: {
key_converted: "wheelchair",
conditional_values: {
"Accessibilité inconnue": {
// value_converted: "",
ignore_this_data: true, // ne pas ajouter de tag si la valeur est égale à Accessibilité inconnue.
// transform_function : (original_value) => original_value.toLowerCase(),
},
"Accessible mais non réservé PMR": {
value_converted: "yes"
},
"Réservé PMR": {
value_converted: "yes"
},
"Non accessible": {
value_converted: "no"
},
"Mo-Fr 08:30-12:00,Mo-Fr 14:00-19:00,Sat 09:00-18:30": {
value_converted: "Mo-Fr 08:30-12:00,Mo-Fr 14:00-19:00,Sat 09:00-18:30"
},
"24/7": {
value_converted: ""
}
// choix:
@ -81,25 +94,20 @@ export default {
// Mo-Fr 09:00-16:00
// Mo-Fr 08:00-12:00,Mo-Fr 14:00-18:00,Th 08:00-18:00
}
},
station_deux_roues: {
key_converted: null,
conditional_values: {
// ajout de trois tags si la valeur est yes
yes: {
tags_to_add: [
{bicycle: "yes"},
{scooter: "yes"},
{motorcar: "no"},
]
}
},
station_deux_roues: {
key_converted: null,
conditional_values: {
// ajout de trois tags si la valeur est yes
yes: {
tags_to_add: [
{bicycle: "yes"},
{scooter: "yes"},
{motorcar: "no"},
]
}
}
}
}
// station_deux_roues =>
// bicycle=yes
// scooter=yes
// motorcar=no
},
}

View File

@ -0,0 +1,75 @@
import debugLog from 'utils'
export default class {
config = {}
constructor(mappingConfig){
this.setConfig(mappingConfig)
}
setConfig(mappingConfig){
this.config = mappingConfig
}
mapFeaturePoint(featurePointGeoJson){
let geoJSONConvertedPoint = {}
geoJSONConvertedPoint.properties = {... this.config.default_properties_of_point};
geoJSONConvertedPoint.type = featurePointGeoJson.type;
geoJSONConvertedPoint.geometry = featurePointGeoJson.geometry
let props = featurePointGeoJson.properties
props.forEach((key,value)=>{
})
return geoJSONConvertedPoint;
}
isBooleanKey (pointKeyName) {
return listOfBooleanKeys.indexOf(pointKeyName) !== -1
}
/**
* retuns the converted element from mapping config if present, null otherwise
*/
mapElementFromConf (featurePoint, mappingConfig) {
let mappingKeys = Object.keys(mappingConfig)
let featurePointPropertiesKeys = Object.keys(featurePoint.properties)
debugLog('keys', mappingKeys, featurePointPropertiesKeys)
let newProperties = defaultPropertiesOfPoint
// reinit properties of current point
let basePoint = Object.create(featurePoint)
basePoint.type = featurePoint.type
basePoint.geometry = featurePoint.geometry
basePoint.properties = newProperties
// apply new properties if found in mapping config
featurePointPropertiesKeys.forEach(pointKeyName => {
if (mappingKeys.indexOf(pointKeyName) !== -1) {
debugLog('found element', pointKeyName, '=>', mappingConfig[pointKeyName], 'value : ', featurePoint.properties[pointKeyName])
let convertedValue = ''
if (isBooleanKey(pointKeyName)) {
convertedValue = featurePoint.properties[pointKeyName].toLowerCase() == 'true' ? 'yes' : 'no'
} else {
convertedValue = featurePoint.properties[pointKeyName]
}
if (convertedValue) {
newProperties[mappingConfig[pointKeyName]] = convertedValue
}
}
})
debugLog('basePoint', basePoint)
return basePoint
}
}

View File

@ -0,0 +1,18 @@
let show_debug = 0
/**
* faire un log
* @param message
*/
function debugLog (message) {
if (!show_debug) {
return
}
console.log('debug: ', ...message)
}
export default
{
debugLog
}