tykayn/scripts
1
0
Fork 0
Code Issues 1 Pull Requests Projects Releases Wiki Activity

start migration to Ts

Browse Source
This commit is contained in:
Tykayn 2023-08-02 13:20:50 +02:00 committed by tykayn
parent 6fc08cc6b3
commit abc0916e7b
13 changed files with 6949 additions and 583324 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
mapping_geojson_to_osm_tags/README.md
View File

@ -12,11 +12,11 @@ Pour récupérer le jeu de données pour les bornes elec et convertir le geojson
bash refresh_data.sh bash refresh_data.sh
``` ```
## config ## config
Configurer le filtre de département et les autres options dans convert_to_osm_tags.mjs. Configurer le filtre de département et les autres options dans convert_to_osm_tags.ts.
## utilisation ## utilisation
```bash ```bash
node convert_to_osm_tags.mjs node convert_to_osm_tags.ts
``` ```
résultat en json dans le dossier output. résultat en json dans le dossier output.

12
mapping_geojson_to_osm_tags/convert_to_osm_tags.mjs → mapping_geojson_to_osm_tags/convert_to_osm_tags.ts
View File

@ -1,11 +1,11 @@
/** /**
* convertisseur de données de bornes de recharge électrique à partir de données Chargemap et open data Etalab * convertisseur de données de bornes de recharge électrique à partir de données Chargemap et open data Etalab
*/ */
import fs from 'node-fs' import fs from 'node'
import minimist from 'minimist' import * as minimist from 'minimist'
import mappingConfigIRVE from './mappings/converters/configIRVE.mjs' import mappingConfigIRVE from './mappings/converters/configIRVE'
import mappingConfigIRVE_simple from './mappings/converters/mappingConfigIRVE_simple.mjs' import mappingConfigIRVE_simple from './mappings/converters/mappingConfigIRVE_simple'
import mapping_engine from './mappings/engine.mjs' import mapping_engine from './mappings/engine'
import custom_utils from './mappings/utils.mjs' import custom_utils from './mappings/utils.mjs'
const { debugLog } = custom_utils const { debugLog } = custom_utils
@ -20,7 +20,7 @@ let Mapping_engine = new mapping_engine(mappingConfigIRVE)
let mini_arguments = minimist(process.argv.slice(2)) let mini_arguments = minimist(process.argv.slice(2))
// let sourceFileChargemapJson = './chargemap_data/hurepoix.json' // let sourceFileChargemapJson = './chargemap_data/hurepoix.json'
let sourceFilePathGeoJson = './etalab_data/latest.json' let sourceFilePathGeoJson = './etalab_data/small.json'
// wip filter // wip filter
let filterOnBoundingBox = true let filterOnBoundingBox = true

48
mapping_geojson_to_osm_tags/etalab_data/small.json
View File

@ -13,54 +13,8 @@
"properties": { "properties": {
"nom_amenageur": "ELECTRA", "nom_amenageur": "ELECTRA",
"siren_amenageur": "891624884", "siren_amenageur": "891624884",
"contact_amenageur": "help@electra.com",
"nom_operateur": "ELECTRA",
"contact_operateur": "help@electra.com",
"telephone_operateur": "",
"nom_enseigne": "ELECTRA",
"id_station_itinerance": "FRELCPSDRPG",
"id_station_local": "",
"nom_station": "S\u00e9r\u00e9zin-du-Rh\u00f4ne - Peugeot",
"implantation_station": "Station d\u00e9di\u00e9e \u00e0 la recharge rapide",
"adresse_station": "12 chemin d\u00e9partemental 12, Peugeot 69360 S\u00e9r\u00e9zin-du-Rh\u00f4ne",
"code_insee_commune": "69294",
"coordonneesXY": "[4.82215900,45.63507900]",
"nbre_pdc": "8",
"id_pdc_itinerance": "FRELCECURS",
"id_pdc_local": "",
"puissance_nominale": "225",
"prise_type_ef": "false",
"prise_type_2": "false",
"prise_type_combo_ccs": "true",
"prise_type_chademo": "false",
"prise_type_autre": "false",
"gratuit": "false",
"paiement_acte": "true",
"paiement_cb": "true",
"paiement_autre": "true",
"tarification": "",
"condition_acces": "Acc\u00e8s libre",
"reservation": "true",
"horaires": "24/7",
"accessibilite_pmr": "Accessibilit\u00e9 inconnue",
"restriction_gabarit": "Inconnu",
"station_deux_roues": "false",
"raccordement": "Direct",
"num_pdl": "N/A",
"date_mise_en_service": "2022-10-27",
"observations": "T\u00e9l\u00e9charger l'application ELECTRA pour r\u00e9server et payer sur go-electra.com",
"date_maj": "2023-07-29",
"cable_t2_attache": "",
"last_modified": "2023-07-29T03:05:24.360000+00:00",
"datagouv_dataset_id": "623ca46c13130c3228abd018",
"datagouv_resource_id": "e9bb3424-77cd-40ba-8bbd-5a19362d0365",
"datagouv_organization_or_owner": "electra",
"consolidated_longitude": 4.822159,
"consolidated_latitude": 45.635079,
"consolidated_code_postal": "69360",
"consolidated_commune": "S\u00e9r\u00e9zin-du-Rh\u00f4ne", "consolidated_commune": "S\u00e9r\u00e9zin-du-Rh\u00f4ne",
"consolidated_is_lon_lat_correct": true, "consolidated_is_lon_lat_correct": true
"consolidated_is_code_insee_verified": true
} }
} }
] ]

0
mapping_geojson_to_osm_tags/mappings/converters/configIRVE.mjs → mapping_geojson_to_osm_tags/mappings/converters/configIRVE.ts
View File

0
mapping_geojson_to_osm_tags/mappings/converters/mappingConfigIRVE_simple.mjs → mapping_geojson_to_osm_tags/mappings/converters/mappingConfigIRVE_simple.ts
View File

184
mapping_geojson_to_osm_tags/mappings/engine.mjs
View File

@ -1,184 +0,0 @@
import custom_utils from './utils.mjs'
const { debugLog } = custom_utils
let listOfBooleanKeys = Object.keys({
prise_type_ef: 'socket:typee',
prise_type_2: 'socket:type2',
prise_type_combo_ccs: 'socket:type2_combo',
prise_type_chademo: 'socket:chademo',
gratuit: 'fee',
paiement_acte: 'authentication:none',
paiement_cb: 'payment:credit_cards',
cable_t2_attache: 'socket:type2_cable',
})
export default class {
mapping_config = {}
constructor (mappingConfig) {
this.setConfig(mappingConfig)
}
setConfig (mappingConfig) {
this.mapping_config = mappingConfig
}
mapFeaturePoint (featurePointGeoJson) {
let geoJSONConvertedPoint = {}
geoJSONConvertedPoint.properties = { ...this.mapping_config.default_properties_of_point }
geoJSONConvertedPoint.type = featurePointGeoJson.type
geoJSONConvertedPoint.geometry = featurePointGeoJson.geometry
let props = featurePointGeoJson.properties
props.forEach((key, value) => {
})
return geoJSONConvertedPoint
}
/**
* TODO convert to mapping config property to transform_truthy
* @param pointKeyName
* @returns {boolean}
*/
isBooleanKey (pointKeyName) {
return listOfBooleanKeys.indexOf(pointKeyName) !== -1
}
truthyValues = ['true', 'True', 'TRUE', '1', 1]
falsyValues = ['false', 'False', 'FALSE', '0', 0]
filterFeaturesByOffset (offsetCount, listOfFeatures) {
let filteredList = listOfFeatures
// TODO
return filteredList
}
filterFeaturesByPropertyRegex (bboxConfig, listOfFeatures) {
console.log('bboxConfig', bboxConfig)
let filteredList = listOfFeatures
// TODO
return filteredList
}
filterFeaturesByPropertyRegex (propertyName, criteriaRegex, listOfFeatures) {
let filteredList = listOfFeatures.filter(feature => {
return criteriaRegex.test(feature?.properties[propertyName])
})
return filteredList
}
/**
* retuns the converted element from mapping config if present, null otherwise
*/
mapElementFromConf(featurePoint) {
if (!this.mapping_config) {
throw new Error('no config was loaded in the mapping engine. use setconfig(my_mapping_config) on this instance of mapping engine before using this.')
}
debugLog('config_name', this.mapping_config.config_name)
let mappingKeys = Object.keys(this.mapping_config.tags)
let featurePointPropertiesKeys = Object.keys(featurePoint.properties)
debugLog('keys', mappingKeys, featurePointPropertiesKeys)
let newProperties = Object.create(this.mapping_config.default_properties_of_point)
// reinit properties of current point
let basePoint = Object.create(featurePoint)
basePoint.type = featurePoint.type
basePoint.geometry = featurePoint.geometry
// apply new properties if found in mapping config
featurePointPropertiesKeys.forEach(pointKeyName => {
if (!mappingKeys.indexOf(pointKeyName) !== -1) {
debugLog('found element', pointKeyName, '=>', this.mapping_config[pointKeyName], 'value : ', featurePoint.properties[pointKeyName])
let convertedValue = ''
let valueConvertedFromMapping = featurePoint.properties[pointKeyName]
let typeofValue = typeof valueConvertedFromMapping
let isStringValue = typeofValue === 'string'
debugLog('typeof featurePoint.properties[pointKeyName] === \'string\'', typeofValue)
let isConfigMappingObject = typeofValue === 'string'
if (isStringValue) {
debugLog('string value')
if (this.isBooleanKey(pointKeyName)) {
let lowerValue = (valueConvertedFromMapping + '').toLowerCase()
convertedValue = this.truthyValues.indexOf(lowerValue) ? 'yes' : 'no'
} else {
convertedValue = valueConvertedFromMapping
}
if (convertedValue) {
newProperties[this.mapping_config[pointKeyName]] = convertedValue
}
} else if (isConfigMappingObject) {
let newKey = ''
let configObject = valueConvertedFromMapping
if (configObject.key_converted) {
newKey = configObject.key_converted
}
/**
* gestion des valeurs conditionnelles
* nous pouvons renseigner une string ou un objet décrivant les transformations à réaliser
*/
if (configObject.conditional_values) {
let keysConditionnalValues = Object.keys(configObject.conditional_values)
let isFoundValue = keysConditionnalValues.indexOf(valueConvertedFromMapping)
if (isFoundValue !== -1) {
let conditionnalConfig = keysConditionnalValues[isFoundValue]
if (conditionnalConfig.tags_to_add) {
// on peut définir un ensemble de tags à rajouter
newProperties.push(...conditionnalConfig.tags_to_add)
}
if (conditionnalConfig.truthy_value) {
// convertir la valeur, si elle est truthy, la transformer en ce que donne la propriété truthy_value
// exemple: le jeu de données dit que la colonne cable_t2_attache vaut "True", mais on veut le convertir en "1".
// on met donc truthy_value: '1'
if (this.truthyValues.indexOf(valueConvertedFromMapping) !== -1) {
convertedValue = conditionnalConfig.truthy_value
}
}
if (conditionnalConfig.falsy_value) {
if (this.falsyValues.indexOf(valueConvertedFromMapping) !== -1) {
convertedValue = conditionnalConfig.falsy_value
}
}
if (conditionnalConfig.transform_function) {
// une transformation de la valeur
// apply transformation to value
convertedValue = conditionnalConfig.transform_function(valueConvertedFromMapping)
}
// use the value converted
else if (conditionnalConfig.value_converted) {
convertedValue = conditionnalConfig.value_converted
}
}
}
if (newKey && !configObject.ignore_this_data) {
newProperties[newKey] = convertedValue
}
}
}
})
basePoint.properties = newProperties
debugLog('basePoint', basePoint)
return basePoint
}
}

198
mapping_geojson_to_osm_tags/mappings/engine.ts Normal file
View File

@ -0,0 +1,198 @@
import custom_utils from './utils.mjs'
const {debugLog} = custom_utils
let listOfBooleanKeys = Object.keys({
prise_type_ef: 'socket:typee',
prise_type_2: 'socket:type2',
prise_type_combo_ccs: 'socket:type2_combo',
prise_type_chademo: 'socket:chademo',
gratuit: 'fee',
paiement_acte: 'authentication:none',
paiement_cb: 'payment:credit_cards',
cable_t2_attache: 'socket:type2_cable',
})
export default class {
mapping_config = {}
constructor(mappingConfig) {
this.setConfig(mappingConfig)
}
setConfig(mappingConfig) {
this.mapping_config = mappingConfig
}
mapFeaturePoint(featurePointGeoJson) {
let geoJSONConvertedPoint = {}
geoJSONConvertedPoint.properties = {...this.mapping_config.default_properties_of_point}
geoJSONConvertedPoint.type = featurePointGeoJson.type
geoJSONConvertedPoint.geometry = featurePointGeoJson.geometry
let props = featurePointGeoJson.properties
props.forEach((key, value) => {
})
return geoJSONConvertedPoint
}
/**
* TODO convert to mapping config property to transform_truthy
* @param pointKeyName
* @returns {boolean}
*/
isBooleanKey(pointKeyName) {
return listOfBooleanKeys.indexOf(pointKeyName) !== -1
}
truthyValues = ['true', 'True', 'TRUE', '1', 1]
falsyValues = ['false', 'False', 'FALSE', '0', 0]
filterFeaturesByOffset(offsetCount, listOfFeatures) {
let filteredList = listOfFeatures
// TODO
return filteredList
}
filterFeaturesByPropertyRegex(bboxConfig, listOfFeatures) {
console.log('bboxConfig', bboxConfig)
let filteredList = listOfFeatures
// TODO
return filteredList
}
filterFeaturesByPropertyRegex(propertyName, criteriaRegex, listOfFeatures) {
let filteredList = listOfFeatures.filter(feature => {
return criteriaRegex.test(feature?.properties[propertyName])
})
return filteredList
}
/**
* retuns the converted element from mapping config if present, null otherwise
*/
mapElementFromConf(featurePoint) {
if (!this.mapping_config) {
throw new Error('no config was loaded in the mapping engine. use setConfig(my_mapping_config) on this instance of mapping engine before using this.')
}
console.log('mapping_config tags', this.mapping_config.tags)
debugLog('config_name', this.mapping_config.config_name)
let mappingKeys = Object.keys(this.mapping_config.tags)
// let mappingKeys = (this.mapping_config.tags)
let featurePointPropertiesKeys = Object.keys(featurePoint.properties)
debugLog('============= keys mappingKeys:', this.mapping_config.tags.length, mappingKeys.length)
debugLog('============= keys featurePointPropertiesKeys :', featurePoint.properties.length, featurePointPropertiesKeys.length)
let newProperties = Object.create(this.mapping_config.default_properties_of_point)
return;
// reinit properties of current point
let basePoint = Object.create(featurePoint)
basePoint.type = featurePoint.type
basePoint.geometry = featurePoint.geometry
// apply new properties if found in mapping config
featurePointPropertiesKeys.forEach(pointKeyName => {
this.convertProperty(pointKeyName, mappingKeys, featurePoint, newProperties)
})
basePoint.properties = newProperties
debugLog('basePoint', basePoint)
return basePoint
}
convertProperty(pointKeyName, mappingKeys, featurePoint, newProperties) {
console.log('pointKeyName', pointKeyName)
if (!mappingKeys.indexOf(pointKeyName) !== -1) {
debugLog('found element', pointKeyName, '=>', this.mapping_config[pointKeyName], 'value : ', featurePoint.properties[pointKeyName])
let convertedValue = ''
let valueConvertedFromMapping = featurePoint.properties[pointKeyName]
let typeofValue = typeof valueConvertedFromMapping
let isStringValue = typeofValue === 'string'
debugLog('- pointKeyName', pointKeyName)
debugLog('- valueConvertedFromMapping', valueConvertedFromMapping)
// debugLog('typeof featurePoint.properties[pointKeyName] === \'string\'', typeofValue)
let isConfigMappingObject = typeofValue === 'string'
if (isStringValue) {
debugLog('-- string value')
if (this.isBooleanKey(pointKeyName)) {
let lowerValue = (valueConvertedFromMapping + '').toLowerCase()
debugLog('isBooleanKey: lowerValue', lowerValue)
convertedValue = this.truthyValues.indexOf(lowerValue) ? 'yes' : 'no'
} else {
convertedValue = valueConvertedFromMapping
}
debugLog('-- convertedValue', convertedValue)
if (convertedValue) {
newProperties[this.mapping_config[pointKeyName]] = convertedValue
}
} else if (isConfigMappingObject) {
// let newKey = ''
// let configObject = valueConvertedFromMapping
//
// if (configObject.key_converted) {
// newKey = configObject.key_converted
// }
//
// /**
// * gestion des valeurs conditionnelles
// * nous pouvons renseigner une string ou un objet décrivant les transformations à réaliser
// */
// if (configObject.conditional_values) {
// let keysConditionnalValues = Object.keys(configObject.conditional_values)
// let isFoundValue = keysConditionnalValues.indexOf(valueConvertedFromMapping)
// if (isFoundValue !== -1) {
// let conditionnalConfig = keysConditionnalValues[isFoundValue]
//
// if (conditionnalConfig.tags_to_add) {
// // on peut définir un ensemble de tags à rajouter
// newProperties.push(...conditionnalConfig.tags_to_add)
// }
// if (conditionnalConfig.truthy_value) {
// // convertir la valeur, si elle est truthy, la transformer en ce que donne la propriété truthy_value
// // exemple: le jeu de données dit que la colonne cable_t2_attache vaut "True", mais on veut le convertir en "1".
// // on met donc truthy_value: '1'
// if (this.truthyValues.indexOf(valueConvertedFromMapping) !== -1) {
// convertedValue = conditionnalConfig.truthy_value
// }
// }
// if (conditionnalConfig.falsy_value) {
// if (this.falsyValues.indexOf(valueConvertedFromMapping) !== -1) {
// convertedValue = conditionnalConfig.falsy_value
// }
// }
// if (conditionnalConfig.transform_function) {
// // une transformation de la valeur
// // apply transformation to value
// convertedValue = conditionnalConfig.transform_function(valueConvertedFromMapping)
// }
// // use the value converted
// else if (conditionnalConfig.value_converted) {
// convertedValue = conditionnalConfig.value_converted
// }
// }
// }
//
// if (newKey && !configObject.ignore_this_data) {
// newProperties[newKey] = convertedValue
// }
}
}
}
}

0
mapping_geojson_to_osm_tags/mappings/infer_domain_from_csv.mjs → mapping_geojson_to_osm_tags/mappings/infer_domain_from_csv.ts
View File

32
mapping_geojson_to_osm_tags/mappings/utils.mjs
View File

@ -1,16 +1,17 @@
import fs from 'node-fs' import fs from 'node-fs'
let show_debug = 0 let show_debug = 0
show_debug = 1
/** /**
* faire un log * faire un log
* @param message * @param message
*/ */
function debugLog (message) { function debugLog(message) {
if (!show_debug) { if (!show_debug) {
return return
} }
console.log('debug: ', message) console.log('debug: ', message)
} }
@ -26,7 +27,6 @@ let listOfBooleanKeys = Object.keys({
}) })
/** /**
* *
* @param pointKeyName * @param pointKeyName
@ -37,24 +37,24 @@ function isBooleanKey(pointKeyName) {
return listOfBooleanKeys.indexOf(pointKeyName) !== -1 return listOfBooleanKeys.indexOf(pointKeyName) !== -1
} }
function writeFile (fileName, fileContent) { function writeFile(fileName, fileContent) {
debugLog('write file ', fileName) debugLog('write file ', fileName)
return fs.writeFile( return fs.writeFile(
`./output/${fileName}`, `./output/${fileName}`,
fileContent, fileContent,
'utf8', 'utf8',
(err) => { (err) => {
if (err) { if (err) {
debugLog(`Error writing file: ${err}`) debugLog(`Error writing file: ${err}`)
} else { } else {
debugLog(`File ${fileName} is written successfully!`) debugLog(`File ${fileName} is written successfully!`)
} }
} }
) )
} }
export default
{ export default {
debugLog, debugLog,
isBooleanKey, isBooleanKey,
writeFile writeFile

583066
mapping_geojson_to_osm_tags/output/my_converted_data_set.json
View File

File diff suppressed because it is too large Load Diff

6720
mapping_geojson_to_osm_tags/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

6
mapping_geojson_to_osm_tags/package.json
View File

@ -10,8 +10,8 @@
"url": "https://forge.chapril.org/tykayn/scripts.git" "url": "https://forge.chapril.org/tykayn/scripts.git"
}, },
"scripts": { "scripts": {
"start": "ts-node main.ts", "start": "ts-node convert_to_osm_tags.ts --engine",
"filter": "node convert_to_osm_tags.mjs --department=76 --engine=true", "filter": "node convert_to_osm_tags.ts --department=76 --engine=true",
"test": "jest --coverage --watch", "test": "jest --coverage --watch",
"tests": "jest --coverage --watch" "tests": "jest --coverage --watch"
}, },
@ -25,7 +25,7 @@
"@babel/preset-typescript": "^7.22.5", "@babel/preset-typescript": "^7.22.5",
"@jest/globals": "^29.5.0", "@jest/globals": "^29.5.0",
"@types/minimist": "^1.2.2", "@types/minimist": "^1.2.2",
"@types/node": "^20.4.2", "@types/node": "^20.4.5",
"babel-jest": "^29.5.0", "babel-jest": "^29.5.0",
"jest": "^29.5.0", "jest": "^29.5.0",
"loglevel": "^1.8.1", "loglevel": "^1.8.1",

3
mapping_geojson_to_osm_tags/tsconfig.js Normal file
View File

@ -0,0 +1,3 @@
module.exports= {
noImplicitAny: false,
}