batch ignore points if a property is found with a certain critera of exclusion

convert_to_osm_tags.ts

@@ -162,7 +162,7 @@ function convertDataFromSource(sourceFilePath: string, mapping: any, pointCounte
 
         if (data_transformed.features) {
 
-            console.log('data found in source, features:', data_transformed.features.length)
+            console.log('------ data found in source, features:', data_transformed.features.length)
 
             // find interesting list of points to use
             let list_of_points: any = data_transformed.features
@@ -176,6 +176,16 @@ function convertDataFromSource(sourceFilePath: string, mapping: any, pointCounte
             list_of_points.forEach((feature_point: any) => {
 
                 let regex_filter_test_result = true
+                let remove_original_key = false;
+                feature_point.properties.tags.forEach((tagKey:string,  tagValue:string)=>{
+                    if (this.mapping_config.filters.exclude_point_if_tag_not_empty.indexOf(tagKey) !== -1 && tagValue.length && tagValue !== 'null') {
+                        remove_original_key = true;
+                    }
+                })
+                if(remove_original_key){
+                    return ;
+                }
+
 
                 if (enable_filter_on_department) {
                     debugLog('filtre sur les départements activé')

docs/parking_vélo.md

@@ -22,3 +22,7 @@ Conversion en fichier OSM avec le paquet python pip geojson2osm:
 ```shell 
 geojsontoosm output/my_converted_data_set__mappingIssy2Roues.json > osm_output/issy_parkings_2r.osm
 ```
+## Parking cyclables de la métropole de Rouen
+```bash 
+ts-node convert_to_osm_tags.ts --source=etalab_data/cyclabilité/rouen_parking_velos.json --output-file=rouen_parking_velos.json --engine-config=mappingRouenParkingVelos
+```

mappings/converters/configRouen_OpenData_velo_parkings.ts

@@ -16,46 +16,77 @@ const mappingRouenParkingVelos: MappingConfigType = {
         url: 'https://data.metropole-rouen-normandie.fr/api/explore/v2.1/catalog/datasets/liste-des-stationnements-cyclables-metropole-rouen-normandie/exports/geojson?lang=fr&timezone=Europe%2FBerlin'
     },
     filters: {
+        exclude_point_if_tag_not_empty: ['osm_id']
     },
     add_not_mapped_tags_too: false,
     boolean_keys: [
         "acces_reglement",
     ],
+    tags_to_ignore_if_value_is: ['Non renseigne'],
     tags: {
         // ******* nombres
-        nb_places: 'capacity',
+        capacite: 'capacity',
-        nb_pl_velo: 'capacity:bike',
+        capacite_cargo: 'capacity:cargo_bike',
-        nb_pl_moto: 'capacity:motorcycle',
+        gestionnaire: {
-        annee_crea: 'start_date',
+            key_converted: 'operator',
+            conditionnal_values: {
+                "Non renseigne": {
+                    ignore_this_data: true, // ne pas ajouter de tag si la valeur est égale à
+                }
+            }
+        },
+        proprietaire:  {
+            key_converted: 'owner',
+            conditionnal_values: {
+                "Non renseigne": {
+                    ignore_this_data: true, // ne pas ajouter de tag si la valeur est égale à
+                }
+            }
+        },
+        date_maj: 'check_date',
         // ******* textes
-        type_aire: {
+        commentaire: 'note',
-            "remove_original_key": true,
+        surveillance: {
-            "conditional_values": {
+            key_converted: 'surveillance',
-                "Poteaux": {
+            ignore_if_falsy: true,
+            convert_to_boolean_value: true,
+        },
+        mobilier: {
+            key_converted: 'mobilier',
+            remove_original_key: true,
+            conditional_values: {
+                "POTELET": {
                     "tags_to_add": [
                         {
                             "bicycle_parking": "bollard"
                         },
                     ]
                 },
-                "Arceaux": {
+                "ARCEAU": {
-                    "tags_to_add": [
-                        {
-                            "bicycle_parking": "bollard"
-                        },
-                        ]
-                },
-                "Anneaux": {
                     "tags_to_add": [
                         {
                             "bicycle_parking": "stands"
                         },
                     ]
                 },
+                "RATELIER": {
+                    "tags_to_add": [
+                        {
+                            "bicycle_parking": "rack"
+                        },
+                    ]
+                },
+                // "Anneaux": {
+                //     "tags_to_add": [
+                //         {
+                //             "bicycle_parking": "stands"
+                //         },
+                //         ]
+                // },
             }
         },
         // type_aire: 'nb_pl_moto', // Poteaux => bicycle_parking=bollard, Arceaux, Anneaux => bicycle_parking=stands, Marquage, null, Non renseigné, Range-vélos, Sans, Trottoir
-        id_r2roues: 'ref:FR:rouen_veloparking_id',
+        id_local: 'ref:FR:rouen_veloparking_id',
         // acces_reglement: 'access',
 
     }

mappings/converters/configTest.ts

@@ -13,7 +13,10 @@ const MappingTest: MappingConfigType = {
         nom_amenageur : 'name'
     },
     add_not_mapped_tags_too: false,
-    source: {},
+    source: {
+        geojson_path: "",
+        url: ""
+    },
     filters: {},
 }
 

mappings/engine.ts

@@ -1,6 +1,5 @@
 import custom_utils from './utils'
 import MappingConfigType from "./mapping-config.type";
-import {debuglog} from "util";
 
 const {debugLog} = custom_utils
 
@@ -139,8 +138,10 @@ export default class {
 
             debugLog('mapElementFromConf: convert', pointKeyName)
             debugLog('mapElementFromConf: mapping keys:', mappingKeys)
+
                 this.convertProperty(pointKeyName, mappingKeys, featurePoint, newProperties)
 
+
         })
 
         basePoint.properties = newProperties
@@ -178,6 +179,12 @@ export default class {
         debugLog(' ------ convertProperty: pointKeyName', pointKeyName)
         // debugLog('convertProperty: mappingKeys', mappingKeys)
 
+        let remove_original_key = false;
+        if (this.mapping_config.tags_to_ignore_if_value_is.indexOf(originalValue) !== -1) {
+            remove_original_key = true;
+        }
+
+
         if (this.jardinage) {
             debugLog(' ------ on fait du jardinage')
             debugLog(' ------ mode mise en qualité activé')
@@ -240,12 +247,14 @@ export default class {
                     debugLog('convertProperty: no string value')
                 }
 
-                if (isConfigMappingObject) {
                 let configObject = mappingConfigOfTag
 
+                if (isConfigMappingObject) {
+
+
                     debugLog('convertProperty: is config object', configObject)
                     let newKey: any = '' + pointKeyName
-                    let remove_original_key = false;
+
 
                     if (configObject.key_converted) {
                         newKey = configObject.key_converted

mappings/mapping-config.type.ts

@@ -1,75 +1,96 @@
 interface GeoJsonGeometry {
-    type:string,
+    type: string,
-    coordinates:Array<number>,
+    coordinates: Array<number>,
 }
 
 interface GeoJsonProperties {
-    [key:string]: any,
+    [key: string]: any,
 }
 
 interface GeoJsonFeature {
-    type:string,
+    type: string,
-    geometry:GeoJsonGeometry,
+    geometry: GeoJsonGeometry,
-    properties:GeoJsonProperties,
+    properties: GeoJsonProperties,
 }
 
-export interface FeatureCollection{
+export interface FeatureCollection {
-    type:string,
+    type: string,
-    features:Array<GeoJsonFeature>,
+    features: Array<GeoJsonFeature>,
 }
 
-export interface BoundingBoxCoordinatesType{
+export interface BoundingBoxCoordinatesType {
     xMin: number,
     xMax: number,
     yMin: number,
     yMax: number,
 }
-export default interface MappingConfigType{
+
+/**
+ * configuration to choose what point to exclude or include from geographic or properties hints
+ */
+export interface filteringConfig {
+    enable_coordinates_filter?: boolean;
+    enable_properties_filter?: boolean;
+    properties?: object;
+    bounding_box?: object;
+    exclude_point_if_tag_not_empty?: Array<string>;
+    exclude_point_if_tag_truthy?: Array<string>;
+    exclude_point_if_tag_falsy?: Array<string>;
+}
+
+interface sourceConfig {
+    geojson_path: string; // the relative path to the geojson source file to analyse, from the root of this repository
+    url: string; // URL from where the geojson comes online, on a data platform. This URL should be fetchable to get the most recent data of the concerned dataset to convert.
+}
+
+export default interface MappingConfigType {
     config_name: string,
     config_author: string,
     osmose?: boolean,
     boolean_keys?: Array<string>,
+    tags_to_ignore_if_value_is?: Array<string>,
     add_not_mapped_tags_too: boolean,
-    default_properties_of_point: object,
+    default_properties_of_point?: object, // tag to add to every converted point by default
-    source: object,
+    source: sourceConfig,
-    filters: object,
+    filters?: filteringConfig,
     tags: FeaturePropertyMappingConfigType
 }
 
 
-
 /**
  * configuration concernant toutes les valeurs
  */
-export interface FeaturePropertyMappingConfigType{
+export interface FeaturePropertyMappingConfigType {
-    [key:string]: any,
+    [key: string]: any,
-    convert_to_boolean_value?:boolean,
+
-    remove_original_key?:boolean,
+    convert_to_boolean_value?: boolean,
-    conditionnal_values?:ConditionnalValuesType,
+    remove_original_key?: boolean,
-    transform_function?:Function,
+    conditionnal_values?: ConditionnalValuesType,
+    transform_function?: Function,
 }
 
 /**
  * choix de conversion de la valeur originale selon des critères donnés
  */
-export interface ConditionnalValuesConfigType{
+export interface ConditionnalValuesConfigType {
-    key_converted?:string,
+    key_converted?: string,
-    value_converted?:string,
+    value_converted?: string,
-    truthy_value?:any,
+    truthy_value?: any,
-    falsy_value?:any, // si la valeur originale est falsy, la convertir en la valeur donnée ici
+    falsy_value?: any, // si la valeur originale est falsy, la convertir en la valeur donnée ici
-    ignore_this_data?:boolean,
+    ignore_this_data?: boolean,
-    tags_to_add?:TagsToAddConfig,
+    tags_to_add?: TagsToAddConfig,
-    transform_function?:Function,
+    transform_function?: Function,
 }
 
-export interface ConditionnalValuesType{
+export interface ConditionnalValuesType {
-    [key:string]: ConditionnalValuesConfigType,
+    [key: string]: ConditionnalValuesConfigType,
 }
+
 interface OneOSMTag {
-    [key:string]: string,
+    [key: string]: string,
 }
 
-export interface TagsToAddConfig{
+export interface TagsToAddConfig {
     tags_to_add: Array<OneOSMTag>
 }