tykayn/scripts
1
0
Fork 0
Code Issues 1 Pull Requests Projects Releases Wiki Activity

start bbox filtering

Browse Source
This commit is contained in:
Tykayn 2023-07-29 22:16:36 +02:00 committed by tykayn
parent 72c335d270
commit bfb59df09c
2 changed files with 1518839 additions and 168 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

329
mapping_geojson_to_osm_tags/convert_to_osm_tags.mjs
View File

@ -1,44 +1,42 @@
/** /**
* convertisseur de données de bornes de recharge électrique à partir de données Chargemap et open data Etalab * convertisseur de données de bornes de recharge électrique à partir de données Chargemap et open data Etalab
*/ */
import fs from 'node-fs'; import fs from 'node-fs'
let show_debug = 0;
let show_debug = 0
// wip filter // wip filter
let filterOnBoundingBox = true let filterOnBoundingBox = true
filterOnBoundingBox = false
let boundingBoxCoordinates ={ let boundingBoxCoordinates = {
xMin: 1.91, xMin: 1.91,
yMin: 48.7, xMax: 2.38,
xMax: 2.38, yMin: 48.7,
yMax: 48.4, yMax: 48.4,
} }
let filterCoordinates = true; let filterCoordinates = true
let enableFilterOnDepartment = false; let enableFilterOnDepartment = false
let filterDepartment = 91; let filterDepartment = 91
let filterZipCode = new RegExp(`^${filterDepartment}`); let filterZipCode = new RegExp(`^${filterDepartment}`)
let filterZipCodeAdresse = new RegExp(` ${filterDepartment}`); let filterZipCodeAdresse = new RegExp(` ${filterDepartment}`)
let filteredName ='' let filteredName = ''
if(enableFilterOnDepartment){ if (enableFilterOnDepartment) {
filteredName = '_filtered_zipcode_'+filterDepartment filteredName = '_filtered_zipcode_' + filterDepartment
} }
if(filterOnBoundingBox){ if (filterOnBoundingBox) {
filteredName = '_filtered_bbox_'+boundingBoxCoordinates.xMin+'-'+boundingBoxCoordinates.xMax+'_'+boundingBoxCoordinates.yMin+'-'+boundingBoxCoordinates.yMax filteredName = '_filtered_bbox_' + boundingBoxCoordinates.xMin + '-' + boundingBoxCoordinates.xMax + '_' + boundingBoxCoordinates.yMin + '-' + boundingBoxCoordinates.yMax
} }
let pointCounterMax = 1000000; let pointCounterMax = 1000000
let limitConversionToFirstPoint = false; let limitConversionToFirstPoint = false
// limitConversionToFirstPoint = true; // limitConversionToFirstPoint = true;
if (limitConversionToFirstPoint) { if (limitConversionToFirstPoint) {
pointCounterMax = 1 pointCounterMax = 1
} }
// let sourceFileChargemapJson = './chargemap_data/hurepoix.json' // let sourceFileChargemapJson = './chargemap_data/hurepoix.json'
let sourceFileIRVEGeoJson = './etalab_data/latest.json' let sourceFileIRVEGeoJson = './etalab_data/latest.json'
/** /**
@ -47,194 +45,189 @@ let sourceFileIRVEGeoJson = './etalab_data/latest.json'
* https://wiki.openstreetmap.org/wiki/France/data.gouv.fr/Bornes_de_Recharge_pour_V%C3%A9hicules_%C3%89lectriques * https://wiki.openstreetmap.org/wiki/France/data.gouv.fr/Bornes_de_Recharge_pour_V%C3%A9hicules_%C3%89lectriques
*/ */
let mappingConfigIRVE = { let mappingConfigIRVE = {
amenity: "amenity", // conserver le tag de base amenity: 'amenity', // conserver le tag de base
capacity: "capacity", // conserver le tag de base capacity: 'capacity', // conserver le tag de base
nom_amenageur: "operator", nom_amenageur: 'operator',
siren_amenageur: "owner:ref:FR:SIREN", siren_amenageur: 'owner:ref:FR:SIREN',
nom_operateur: "operator", nom_operateur: 'operator',
telephone_operateur: "phone", telephone_operateur: 'phone',
contact_operateur: "email", // ici, on souhaite convertir la clé contact_operateur=bidule en email=bidule contact_operateur: 'email', // ici, on souhaite convertir la clé contact_operateur=bidule en email=bidule
nbre_pdc: "capacity", nbre_pdc: 'capacity',
id_station_itinerance: "ref:EU:EVSE", id_station_itinerance: 'ref:EU:EVSE',
id_station_local: "ref", id_station_local: 'ref',
prise_type_ef: "socket:typee", prise_type_ef: 'socket:typee',
prise_type_2: "socket:type2", prise_type_2: 'socket:type2',
prise_type_combo_ccs: "socket:type2_combo", prise_type_combo_ccs: 'socket:type2_combo',
prise_type_chademo: "socket:chademo", prise_type_chademo: 'socket:chademo',
gratuit: "fee", gratuit: 'fee',
paiement_acte: "authentication:none", paiement_acte: 'authentication:none',
paiement_cb: "payment:credit_cards", paiement_cb: 'payment:credit_cards',
reservation: "reservation", reservation: 'reservation',
accessibilite_pmr: "wheelchair", accessibilite_pmr: 'wheelchair',
date_mise_en_service: "start_date", date_mise_en_service: 'start_date',
observations: "note", observations: 'note',
date_maj: "source:date", date_maj: 'source:date',
nom_station: "name", nom_station: 'name',
nom_enseigne: "network", nom_enseigne: 'network',
cable_t2_attache: "socket:type2_cable", cable_t2_attache: 'socket:type2_cable',
// TODO gestion des types dont on doit convertir la valeur // TODO gestion des types dont on doit convertir la valeur
// station_deux_roues => // station_deux_roues =>
// ajout de trois tags: // ajout de trois tags:
// bicycle=yes // bicycle=yes
// scooter=yes // scooter=yes
// motorcar=no // motorcar=no
// consolidated_code_postal: "zipcode", // consolidated_code_postal: "zipcode",
} }
function debugLog (message) {
function debugLog(message) { if (!show_debug) {
if (!show_debug) { return
return }
} console.log('debug: ', ...message)
console.log('debug: ', ...message)
} }
function convertDataForIRVE(sourceFilePath, mapping, pointCounterMax) { function convertDataForIRVE (sourceFilePath, mapping, pointCounterMax, boundingBoxCoordinates) {
debugLog('convertDataFromChargemap from ', sourceFilePath); debugLog('convertDataFromChargemap from ', sourceFilePath)
let convertedGeoJson = { let convertedGeoJson = {
type: 'FeatureCollection', type: 'FeatureCollection',
features: [] features: []
} }
fs.readFile(sourceFilePath, 'utf8', function (err, data) { fs.readFile(sourceFilePath, 'utf8', function (err, data) {
let pointcounter = 0; let pointcounter = 0
if (err) { if (err) {
return debugLog(err); return debugLog(err)
} }
let dataTransformed = JSON.parse(data); let dataTransformed = JSON.parse(data)
// debug('data keys ', Object.keys(dataTransformed)) // debug('data keys ', Object.keys(dataTransformed))
debugLog('debug: properties of point 0', dataTransformed.features[0]) debugLog('debug: properties of point 0', dataTransformed.features[0])
if (dataTransformed.features) { if (dataTransformed.features) {
debugLog('data found, features:', dataTransformed.features.length) debugLog('data found, features:', dataTransformed.features.length)
// find interesting list of points to use // find interesting list of points to use
let listOfPoints = dataTransformed.features let listOfPoints = dataTransformed.features
// for each point from the data source, convert with the mapping // for each point from the data source, convert with the mapping
console.log('listOfPoints.length', listOfPoints.length) console.log('listOfPoints.length', listOfPoints.length)
listOfPoints.forEach(featurePoint => { listOfPoints.forEach(featurePoint => {
let regextestresult = true; let regextestresult = true
if(enableFilterOnDepartment){ if (enableFilterOnDepartment) {
console.log('filtre sur les départements activé') console.log('filtre sur les départements activé')
regextestresult = ( regextestresult = (
filterZipCode.test(featurePoint.properties.consolidated_code_postal) filterZipCode.test(featurePoint.properties.consolidated_code_postal)
|| ||
filterZipCodeAdresse.test(featurePoint.properties.adresse_station) filterZipCodeAdresse.test(featurePoint.properties.adresse_station)
) )
}else if(filterOnBoundingBox){ } else if (filterOnBoundingBox) {
console.log('filtre sur les coordonnées activé') console.log('filtre sur les coordonnées activé')
let x = coordonneesXY[0] let x = featurePoint.properties.coordonneesXY[0]
let xMin = boundingBoxCoordinates.xMin let xMin = boundingBoxCoordinates.xMin
let xMax = boundingBoxCoordinates.xMax let xMax = boundingBoxCoordinates.xMax
let yMin = boundingBoyCoordinates.yMin let yMin = boundingBoxCoordinates.yMin
let yMax = boundingBoyCoordinates.yMay let yMax = boundingBoxCoordinates.yMax
let y = coordonneesXY[1] let y = featurePoint.properties.coordonneesXY[1]
regextestresult = ( regextestresult = (
filterZipCode.test(x >= xMin && x <= xMax) (x >= xMin && x <= xMax)
&& &&
filterZipCode.test(y >= yMin && y <= yMax) (y >= yMin && y <= yMax)
) )
} }
// filter points depending on zipcode
if (filterCoordinates && regextestresult) {
debugLog('featurePoint.properties.consolidated_code_postal', featurePoint.properties.consolidated_code_postal)
// filter points depending on zipcode // limit results number of points
if (filterCoordinates && regextestresult) { // if (pointcounter < pointCounterMax) {
debugLog('featurePoint.properties.consolidated_code_postal', featurePoint.properties.consolidated_code_postal) debugLog('add point')
// limit results number of points debugLog('featurePoint', featurePoint)
// if (pointcounter < pointCounterMax) {
debugLog('add point') let mappedPoint = mapElementFromConf(featurePoint, mapping)
debugLog('map one point', featurePoint, mappedPoint)
if (mappedPoint) {
convertedGeoJson.features.push(mappedPoint)
}
}
// }
pointcounter++
debugLog('featurePoint', featurePoint) })
// output new geojson
console.log('convertedGeoJson.features.length', convertedGeoJson.features.length)
// write file on disk
let fileNameToWrite = 'my_converted_data_set' + filteredName + '.json'
console.log('write file ', fileNameToWrite)
writeJsonFile(fileNameToWrite, JSON.stringify(convertedGeoJson, null, 2))
let mappedPoint = mapElementFromConf(featurePoint, mapping) debugLog('mapped output:', convertedGeoJson.features)
debugLog('map one point', featurePoint, mappedPoint)
if (mappedPoint) {
convertedGeoJson.features.push(mappedPoint)
}
}
// }
pointcounter++;
}) return convertedGeoJson
// output new geojson }
console.log('convertedGeoJson.features.length', convertedGeoJson.features.length) })
// write file on disk
let fileNameToWrite = 'my_converted_data_set'+filteredName+'.json';
console.log('write file ',fileNameToWrite )
writeJsonFile( fileNameToWrite, JSON.stringify(convertedGeoJson, null, 2))
debugLog('mapped output:', convertedGeoJson.features)
return convertedGeoJson;
}
})
} }
/** /**
* retuns the converted element from mapping config if present, null otherwise * retuns the converted element from mapping config if present, null otherwise
*/ */
function mapElementFromConf(featurePoint, mappingConfig) { function mapElementFromConf (featurePoint, mappingConfig) {
let mappingKeys = Object.keys(mappingConfig) let mappingKeys = Object.keys(mappingConfig)
let featurePointPropertiesKeys = Object.keys(featurePoint.properties) let featurePointPropertiesKeys = Object.keys(featurePoint.properties)
debugLog('keys', mappingKeys, featurePointPropertiesKeys) debugLog('keys', mappingKeys, featurePointPropertiesKeys)
let newProperties = {} let newProperties = {}
// reinit properties of current point // reinit properties of current point
let basePoint = Object.create(featurePoint) let basePoint = Object.create(featurePoint)
basePoint.type = featurePoint.type basePoint.type = featurePoint.type
basePoint.geometry = featurePoint.geometry basePoint.geometry = featurePoint.geometry
basePoint.properties = newProperties basePoint.properties = newProperties
// apply new properties if found in mapping config // apply new properties if found in mapping config
featurePointPropertiesKeys.forEach(pointKeyName => { featurePointPropertiesKeys.forEach(pointKeyName => {
if (mappingKeys.indexOf(pointKeyName) !== -1) { if (mappingKeys.indexOf(pointKeyName) !== -1) {
debugLog('found element', pointKeyName, '=>', mappingConfig[pointKeyName], 'value : ', featurePoint.properties[pointKeyName]) debugLog('found element', pointKeyName, '=>', mappingConfig[pointKeyName], 'value : ', featurePoint.properties[pointKeyName])
newProperties[mappingConfig[pointKeyName]] = featurePoint.properties[pointKeyName] newProperties[mappingConfig[pointKeyName]] = featurePoint.properties[pointKeyName]
} }
}) })
debugLog('basePoint', basePoint) debugLog('basePoint', basePoint)
return basePoint return basePoint
} }
function writeJsonFile (fileName, fileContent) {
debugLog('write file ', fileName)
function writeJsonFile(fileName, fileContent) { return fs.writeFile(
debugLog('write file ', fileName); `./output/${fileName}`,
fileContent,
return fs.writeFile( 'utf8',
`./output/${fileName}`, (err) => {
fileContent, if (err) {
"utf8", debugLog(`Error writing file: ${err}`)
(err) => { } else {
if (err) { debugLog(`File ${fileName} is written successfully!`)
debugLog(`Error writing file: ${err}`); }
} else { }
debugLog(`File ${fileName} is written successfully!`); )
}
}
);
} }
console.log('pointCounterMax', pointCounterMax) console.log('pointCounterMax', pointCounterMax)
convertDataForIRVE(sourceFileIRVEGeoJson, mappingConfigIRVE, pointCounterMax) convertDataForIRVE(sourceFileIRVEGeoJson, mappingConfigIRVE, pointCounterMax, boundingBoxCoordinates)
// convertDataFromChargemap(sourceFileChargemapJson, mappingConfigIRVE) // convertDataFromChargemap(sourceFileChargemapJson, mappingConfigIRVE)

1518678
mapping_geojson_to_osm_tags/output/my_converted_data_set.json
View File

File diff suppressed because it is too large Load Diff