refacto to use mapping engine class

This commit is contained in:
Tykayn 2023-07-30 18:37:07 +02:00 committed by tykayn
parent 7175a95ed1
commit 5cb3cc02b1
5 changed files with 357 additions and 244 deletions

View File

@ -4,226 +4,238 @@
import fs from 'node-fs' import fs from 'node-fs'
import minimist from 'minimist' import minimist from 'minimist'
import mappingConfigIRVE from 'mappings/configIRVE' import mappingConfigIRVE from 'mappings/configIRVE'
import mapping_engine from "./mappings/engine";
import {debugLog} from "./mappings/utils";
// let debugLog = custom_utils.debugLog
let mapping_engine = new mapping_engine(mappingConfigIRVE);
let use_mappping_engine = false;
let mini_arguments = minimist(process.argv.slice(2)) let mini_arguments = minimist(process.argv.slice(2))
let show_debug = 0
// let sourceFileChargemapJson = './chargemap_data/hurepoix.json'
let sourceFilePathGeoJson = './etalab_data/latest.json'
// wip filter // wip filter
let filterOnBoundingBox = true let filterOnBoundingBox = true
filterOnBoundingBox = false filterOnBoundingBox = false
let boundingBoxCoordinates = { let boundingBoxCoordinates = {
xMin: 1.91, xMin: 1.91,
xMax: 2.38, xMax: 2.38,
yMin: 48.7, yMin: 48.7,
yMax: 48.4, yMax: 48.4,
} }
let filterCoordinates = true let filterCoordinates = true
let enableFilterOnDepartment = true let enable_filter_on_department = true
let filterDepartment = 91 let filterDepartment = 91
if (mini_arguments['department']) { if (mini_arguments['department']) {
filterDepartment = mini_arguments['department'] filterDepartment = mini_arguments['department']
}
if (mini_arguments['source']) {
sourceFilePathGeoJson = mini_arguments['source']
} }
let filterZipCode = new RegExp(`^${filterDepartment}`) let filterZipCode = new RegExp(`^${filterDepartment}`)
let filterZipCodeAdresse = new RegExp(` ${filterDepartment}`) let filterZipCodeAdresse = new RegExp(` ${filterDepartment}`)
let filteredName = '' let filteredName = ''
if (enableFilterOnDepartment) { if (enable_filter_on_department) {
filteredName = '_filtered_zipcode_' + filterDepartment filteredName = '_filtered_zipcode_' + filterDepartment
} else if (filterOnBoundingBox) { } else if (filterOnBoundingBox) {
filteredName = '_filtered_bbox_' + boundingBoxCoordinates.xMin + '-' + boundingBoxCoordinates.xMax + '_' + boundingBoxCoordinates.yMin + '-' + boundingBoxCoordinates.yMax filteredName = '_filtered_bbox_' + boundingBoxCoordinates.xMin + '-' + boundingBoxCoordinates.xMax + '_' + boundingBoxCoordinates.yMin + '-' + boundingBoxCoordinates.yMax
} }
let pointCounterMax = 1000000 let pointCounterMax = 1000000
let limitConversionToFirstPoint = false let limitConversionToFirstPoint = false
// limitConversionToFirstPoint = true; // limitConversionToFirstPoint = true;
if (limitConversionToFirstPoint) { if (limitConversionToFirstPoint) {
pointCounterMax = 1 pointCounterMax = 1
} }
// let sourceFileChargemapJson = './chargemap_data/hurepoix.json'
let sourceFileIRVEGeoJson = './etalab_data/latest.json'
let defaultPropertiesOfPoint = { let defaultPropertiesOfPoint = {
'amenity': 'charging_station' 'amenity': 'charging_station'
} }
let listOfBooleanKeys = Object.keys({ let listOfBooleanKeys = Object.keys({
prise_type_ef: 'socket:typee', prise_type_ef: 'socket:typee',
prise_type_2: 'socket:type2', prise_type_2: 'socket:type2',
prise_type_combo_ccs: 'socket:type2_combo', prise_type_combo_ccs: 'socket:type2_combo',
prise_type_chademo: 'socket:chademo', prise_type_chademo: 'socket:chademo',
gratuit: 'fee', gratuit: 'fee',
paiement_acte: 'authentication:none', paiement_acte: 'authentication:none',
paiement_cb: 'payment:credit_cards', paiement_cb: 'payment:credit_cards',
cable_t2_attache: 'socket:type2_cable', cable_t2_attache: 'socket:type2_cable',
}) })
function debugLog (message) {
if (!show_debug) { let converted_geo_json = {
return type: 'FeatureCollection',
} features: []
console.log('debug: ', ...message) }
/**
*
* @param sourceFilePath
* @param mapping
* @param pointCounterMax
* @param boundingBoxCoordinates
*/
function convertDataForIRVE(sourceFilePath, mapping, pointCounterMax, boundingBoxCoordinates) {
debugLog('convertDataFromChargemap from ', sourceFilePath)
fs.readFile(sourceFilePath, 'utf8', function (err, data) {
let point_counter = 0
if (err) {
return debugLog(err)
}
let data_transformed = JSON.parse(data)
// debug('data keys ', Object.keys(dataTransformed))
debugLog('debug: properties of point 0', data_transformed.features[0])
if (data_transformed.features) {
debugLog('data found, features:', data_transformed.features.length)
// find interesting list of points to use
let list_of_points = data_transformed.features
// for each point from the data source, convert with the mapping
console.log('listOfPoints.length', list_of_points.length)
list_of_points.forEach(feature_point => {
let regex_filter_test_result = true
if (enable_filter_on_department) {
console.log('filtre sur les départements activé')
regex_filter_test_result = (
filterZipCode.test(feature_point.properties.consolidated_code_postal)
||
filterZipCodeAdresse.test(feature_point.properties.adresse_station)
)
} else if (filterOnBoundingBox) {
console.log('filtre sur les coordonnées activé')
let x = feature_point.properties.coordonneesXY[0]
let xMin = boundingBoxCoordinates.xMin
let xMax = boundingBoxCoordinates.xMax
let yMin = boundingBoxCoordinates.yMin
let yMax = boundingBoxCoordinates.yMax
let y = feature_point.properties.coordonneesXY[1]
regex_filter_test_result = (
(x >= xMin && x <= xMax)
&&
(y >= yMin && y <= yMax)
)
}
// filter points depending on zipcode
if (filterCoordinates && regex_filter_test_result) {
debugLog('featurePoint.properties.consolidated_code_postal', feature_point.properties.consolidated_code_postal)
// limit results number of points
// if (pointcounter < pointCounterMax) {
debugLog('add point')
debugLog('featurePoint', feature_point)
let mapped_point = {};
if (use_mappping_engine) {
mapping_engine.setConfig(mapping)
mapped_point = mapping_engine.mapFeaturePoint(feature_point)
} else {
mapped_point = mapElementFromConfSimple(feature_point, mapping)
}
debugLog('map one point', feature_point, mapped_point)
if (mapped_point) {
converted_geo_json.features.push(mapped_point)
}
}
// }
point_counter++
})
// output new geojson
console.log('convertedGeoJson.features.length', converted_geo_json.features.length)
// write file on disk
let fileNameToWrite = 'my_converted_data_set' + filteredName + '.json'
console.log('write file ', fileNameToWrite)
writeJsonFile(fileNameToWrite, JSON.stringify(converted_geo_json, null, 2))
debugLog('mapped output:', converted_geo_json.features)
return converted_geo_json
}
})
} }
function convertDataForIRVE (sourceFilePath, mapping, pointCounterMax, boundingBoxCoordinates) {
debugLog('convertDataFromChargemap from ', sourceFilePath)
let convertedGeoJson = {
type: 'FeatureCollection',
features: []
}
fs.readFile(sourceFilePath, 'utf8', function (err, data) {
let pointcounter = 0
if (err) {
return debugLog(err)
}
let dataTransformed = JSON.parse(data)
// debug('data keys ', Object.keys(dataTransformed))
debugLog('debug: properties of point 0', dataTransformed.features[0])
if (dataTransformed.features) {
debugLog('data found, features:', dataTransformed.features.length)
// find interesting list of points to use
let listOfPoints = dataTransformed.features
// for each point from the data source, convert with the mapping
console.log('listOfPoints.length', listOfPoints.length)
listOfPoints.forEach(featurePoint => {
let regextestresult = true
if (enableFilterOnDepartment) {
console.log('filtre sur les départements activé')
regextestresult = (
filterZipCode.test(featurePoint.properties.consolidated_code_postal)
||
filterZipCodeAdresse.test(featurePoint.properties.adresse_station)
)
} else if (filterOnBoundingBox) {
console.log('filtre sur les coordonnées activé')
let x = featurePoint.properties.coordonneesXY[0]
let xMin = boundingBoxCoordinates.xMin
let xMax = boundingBoxCoordinates.xMax
let yMin = boundingBoxCoordinates.yMin
let yMax = boundingBoxCoordinates.yMax
let y = featurePoint.properties.coordonneesXY[1]
regextestresult = (
(x >= xMin && x <= xMax)
&&
(y >= yMin && y <= yMax)
)
}
// filter points depending on zipcode
if (filterCoordinates && regextestresult) {
debugLog('featurePoint.properties.consolidated_code_postal', featurePoint.properties.consolidated_code_postal)
// limit results number of points
// if (pointcounter < pointCounterMax) {
debugLog('add point')
debugLog('featurePoint', featurePoint)
let mappedPoint = mapElementFromConf(featurePoint, mapping)
debugLog('map one point', featurePoint, mappedPoint)
if (mappedPoint) {
convertedGeoJson.features.push(mappedPoint)
}
}
// }
pointcounter++
})
// output new geojson
console.log('convertedGeoJson.features.length', convertedGeoJson.features.length)
// write file on disk
let fileNameToWrite = 'my_converted_data_set' + filteredName + '.json'
console.log('write file ', fileNameToWrite)
writeJsonFile(fileNameToWrite, JSON.stringify(convertedGeoJson, null, 2))
debugLog('mapped output:', convertedGeoJson.features)
return convertedGeoJson
}
})
}
function isBooleanKey (pointKeyName) {
return listOfBooleanKeys.indexOf(pointKeyName) !== -1
}
/** /**
* retuns the converted element from mapping config if present, null otherwise * retuns the converted element from mapping config if present, null otherwise
*/ */
function mapElementFromConfSimple(featurePoint, mappingConfig)
{
let mappingKeys = Object.keys(mappingConfig)
let featurePointPropertiesKeys = Object.keys(featurePoint.properties)
function mapElementFromConf (featurePoint, mappingConfig) { debugLog('keys', mappingKeys, featurePointPropertiesKeys)
let mappingKeys = Object.keys(mappingConfig)
let featurePointPropertiesKeys = Object.keys(featurePoint.properties)
debugLog('keys', mappingKeys, featurePointPropertiesKeys) let newProperties = defaultPropertiesOfPoint
let newProperties = defaultPropertiesOfPoint // reinit properties of current point
let basePoint = Object.create(featurePoint)
basePoint.type = featurePoint.type
basePoint.geometry = featurePoint.geometry
basePoint.properties = newProperties
// reinit properties of current point // apply new properties if found in mapping config
let basePoint = Object.create(featurePoint) featurePointPropertiesKeys.forEach(pointKeyName => {
basePoint.type = featurePoint.type
basePoint.geometry = featurePoint.geometry
basePoint.properties = newProperties
// apply new properties if found in mapping config if (mappingKeys.indexOf(pointKeyName) !== -1) {
featurePointPropertiesKeys.forEach(pointKeyName => { debugLog('found element', pointKeyName, '=>', mappingConfig[pointKeyName], 'value : ', featurePoint.properties[pointKeyName])
let convertedValue = ''
if (isBooleanKey(pointKeyName)) {
if (mappingKeys.indexOf(pointKeyName) !== -1) { convertedValue = featurePoint.properties[pointKeyName].toLowerCase() == 'true' ? 'yes' : 'no'
debugLog('found element', pointKeyName, '=>', mappingConfig[pointKeyName], 'value : ', featurePoint.properties[pointKeyName]) } else {
let convertedValue = '' convertedValue = featurePoint.properties[pointKeyName]
if (isBooleanKey(pointKeyName)) { }
convertedValue = featurePoint.properties[pointKeyName].toLowerCase() == 'true' ? 'yes' : 'no' if (convertedValue) {
} else { newProperties[mappingConfig[pointKeyName]] = convertedValue
convertedValue = featurePoint.properties[pointKeyName] }
} }
})
if (convertedValue) { debugLog('basePoint', basePoint)
newProperties[mappingConfig[pointKeyName]] = convertedValue return basePoint
}
}
})
debugLog('basePoint', basePoint)
return basePoint
} }
function writeJsonFile (fileName, fileContent) { function writeJsonFile(fileName, fileContent) {
debugLog('write file ', fileName) debugLog('write file ', fileName)
return fs.writeFile( return fs.writeFile(
`./output/${fileName}`, `./output/${fileName}`,
fileContent, fileContent,
'utf8', 'utf8',
(err) => { (err) => {
if (err) { if (err) {
debugLog(`Error writing file: ${err}`) debugLog(`Error writing file: ${err}`)
} else { } else {
debugLog(`File ${fileName} is written successfully!`) debugLog(`File ${fileName} is written successfully!`)
} }
} }
) )
} }
console.log('pointCounterMax', pointCounterMax) console.log('pointCounterMax', pointCounterMax)
convertDataForIRVE(sourceFileIRVEGeoJson, mappingConfigIRVE, pointCounterMax, boundingBoxCoordinates) convertDataForIRVE(sourceFilePathGeoJson, mappingConfigIRVE, pointCounterMax, boundingBoxCoordinates)
// convertDataFromChargemap(sourceFileChargemapJson, mappingConfigIRVE)

View File

@ -2,7 +2,7 @@
D'après le jeu de données Etalab D'après le jeu de données Etalab
## liste des opérateurs ## liste des 310 opérateurs
270 AGENCY 270 AGENCY
2Ed Coutances 2Ed Coutances
2F Production 2F Production

View File

@ -5,69 +5,82 @@
*/ */
export default { export default {
// ******* nombres config_name: "IRVE config",
nbre_pdc: 'capacity', config_author: "tykayn <contact@cipherbliss.com>",
// ******* textes default_properties_of_point: {
amenity: 'amenity', // conserver le tag de base 'amenity': 'charging_station'
capacity: 'capacity', // conserver le tag de base },
nom_amenageur: 'operator', tags: {
siren_amenageur: 'owner:ref:FR:SIREN',
nom_operateur: 'operator',
telephone_operateur: 'phone',
contact_operateur: 'email', // ici, on souhaite convertir la clé contact_operateur=bidule en email=bidule
id_station_itinerance: 'ref:EU:EVSE', // ******* nombres
id_station_local: 'ref', nbre_pdc: 'capacity',
// ******* textes
amenity: 'amenity', // conserver le tag de base
capacity: 'capacity', // conserver le tag de base
nom_amenageur: 'operator',
siren_amenageur: 'owner:ref:FR:SIREN',
nom_operateur: 'operator',
telephone_operateur: 'phone',
contact_operateur: 'email', // ici, on souhaite convertir la clé contact_operateur=bidule en email=bidule
gratuit: 'fee', id_station_itinerance: 'ref:EU:EVSE',
paiement_acte: 'authentication:none', id_station_local: 'ref',
reservation: 'reservation', gratuit: 'fee',
observations: 'note', paiement_acte: 'authentication:none',
nom_station: 'name',
nom_enseigne: 'network',
// ******* dates reservation: 'reservation',
date_mise_en_service: 'start_date', observations: 'note',
date_maj: 'source:date', nom_station: 'name',
nom_enseigne: 'network',
// TODO gestion des types dont on doit convertir la valeur // ******* dates
// ******** champs booléens date_mise_en_service: 'start_date',
cable_t2_attache: 'socket:type2_cable', date_maj: 'source:date',
prise_type_ef: 'socket:typee',
prise_type_2: 'socket:type2',
prise_type_combo_ccs: 'socket:type2_combo',
prise_type_chademo: 'socket:chademo',
// ******** champs plus complexes // TODO gestion des types dont on doit convertir la valeur
horaires: 'opening_hours', // déjà au bon format // ******** champs booléens
cable_t2_attache:
// accessibilite_pmr: 'wheelchair', {
// paiement_cb: 'payment:credit_cards', key_converted: 'socket:type2_cable',
// cable_t2_attache
accessibilite_pmr: { truthy_value: '1'
key_converted: "wheelchair",
conditional_values: {
"Accessibilité inconnue": {
value_converted: "",
ignore_this_data: true, // ne pas ajouter de tag si la valeur est égale à Accessibilité inconnue.
// transform_function : (original_value) => original_value.toLowerCase(),
},
"Accessible mais non réservé PMR": {
value_converted: ""
},
"Réservé PMR": {
value_converted: ""
},
"Non accessible": {
value_converted: "no"
},
"Mo-Fr 08:30-12:00,Mo-Fr 14:00-19:00,Sat 09:00-18:30": {
value_converted: "Mo-Fr 08:30-12:00,Mo-Fr 14:00-19:00,Sat 09:00-18:30"
}
"24/7": {
value_converted: ""
} }
,
prise_type_ef: 'socket:typee',
prise_type_2: 'socket:type2',
prise_type_combo_ccs: 'socket:type2_combo',
prise_type_chademo: 'socket:chademo',
// ******** champs plus complexes
horaires: 'opening_hours', // déjà au bon format
// accessibilite_pmr: 'wheelchair',
// paiement_cb: 'payment:credit_cards',
accessibilite_pmr: {
key_converted: "wheelchair",
conditional_values: {
"Accessibilité inconnue": {
// value_converted: "",
ignore_this_data: true, // ne pas ajouter de tag si la valeur est égale à Accessibilité inconnue.
// transform_function : (original_value) => original_value.toLowerCase(),
},
"Accessible mais non réservé PMR": {
value_converted: "yes"
},
"Réservé PMR": {
value_converted: "yes"
},
"Non accessible": {
value_converted: "no"
},
"Mo-Fr 08:30-12:00,Mo-Fr 14:00-19:00,Sat 09:00-18:30": {
value_converted: "Mo-Fr 08:30-12:00,Mo-Fr 14:00-19:00,Sat 09:00-18:30"
},
"24/7": {
value_converted: ""
}
// choix: // choix:
@ -81,25 +94,20 @@ export default {
// Mo-Fr 09:00-16:00 // Mo-Fr 09:00-16:00
// Mo-Fr 08:00-12:00,Mo-Fr 14:00-18:00,Th 08:00-18:00 // Mo-Fr 08:00-12:00,Mo-Fr 14:00-18:00,Th 08:00-18:00
} }
}, },
station_deux_roues: { station_deux_roues: {
key_converted: null, key_converted: null,
conditional_values: { conditional_values: {
// ajout de trois tags si la valeur est yes // ajout de trois tags si la valeur est yes
yes: { yes: {
tags_to_add: [ tags_to_add: [
{bicycle: "yes"}, {bicycle: "yes"},
{scooter: "yes"}, {scooter: "yes"},
{motorcar: "no"}, {motorcar: "no"},
] ]
}
} }
} }
},
}
// station_deux_roues =>
// bicycle=yes
// scooter=yes
// motorcar=no
} }

View File

@ -0,0 +1,75 @@
import debugLog from 'utils'
export default class {
config = {}
constructor(mappingConfig){
this.setConfig(mappingConfig)
}
setConfig(mappingConfig){
this.config = mappingConfig
}
mapFeaturePoint(featurePointGeoJson){
let geoJSONConvertedPoint = {}
geoJSONConvertedPoint.properties = {... this.config.default_properties_of_point};
geoJSONConvertedPoint.type = featurePointGeoJson.type;
geoJSONConvertedPoint.geometry = featurePointGeoJson.geometry
let props = featurePointGeoJson.properties
props.forEach((key,value)=>{
})
return geoJSONConvertedPoint;
}
isBooleanKey (pointKeyName) {
return listOfBooleanKeys.indexOf(pointKeyName) !== -1
}
/**
* retuns the converted element from mapping config if present, null otherwise
*/
mapElementFromConf (featurePoint, mappingConfig) {
let mappingKeys = Object.keys(mappingConfig)
let featurePointPropertiesKeys = Object.keys(featurePoint.properties)
debugLog('keys', mappingKeys, featurePointPropertiesKeys)
let newProperties = defaultPropertiesOfPoint
// reinit properties of current point
let basePoint = Object.create(featurePoint)
basePoint.type = featurePoint.type
basePoint.geometry = featurePoint.geometry
basePoint.properties = newProperties
// apply new properties if found in mapping config
featurePointPropertiesKeys.forEach(pointKeyName => {
if (mappingKeys.indexOf(pointKeyName) !== -1) {
debugLog('found element', pointKeyName, '=>', mappingConfig[pointKeyName], 'value : ', featurePoint.properties[pointKeyName])
let convertedValue = ''
if (isBooleanKey(pointKeyName)) {
convertedValue = featurePoint.properties[pointKeyName].toLowerCase() == 'true' ? 'yes' : 'no'
} else {
convertedValue = featurePoint.properties[pointKeyName]
}
if (convertedValue) {
newProperties[mappingConfig[pointKeyName]] = convertedValue
}
}
})
debugLog('basePoint', basePoint)
return basePoint
}
}

View File

@ -0,0 +1,18 @@
let show_debug = 0
/**
* faire un log
* @param message
*/
function debugLog (message) {
if (!show_debug) {
return
}
console.log('debug: ', ...message)
}
export default
{
debugLog
}