configMobilizon.ts

@@ -0,0 +1,8 @@
+exports.config = {
+  feeder_mobilizon_user_id: "",
+  db_name: "",
+  db_user: "",
+  db_pass: "",
+};
+
+export {};

index.js

@@ -1,69 +1,122 @@
 // @author tykayn contact@cipherbliss.com www.cipherbliss.com
 // imports
-import utils from './utils.ts';
+import utils from "./utils.ts";
-import parserConfig from './config.ts';
+import parserConfig from "./config.ts";
-import { htmlEscape } from 'escape-goat';
+import { v4 as uuidv4 } from "uuid";
-import Parser from 'rss-parser';
+import { htmlEscape } from "escape-goat";
-
+import Parser from "rss-parser";
-const fetch = require('node-fetch');
+import { Client } from "pg";
+const fetch = require("node-fetch");
 let parser = new Parser();
 
 // @ts-ignore
-console.log(' ==============================================='.blue);
+console.log(" ===============================================".blue);
 // @ts-ignore
-console.log(' ====  Rss Feeder for mobilizon - by tykayn ===='.blue);
+console.log(" ====  Rss Feeder for mobilizon - by tykayn ====".blue);
 // @ts-ignore
-console.log(' ==============================================='.blue);
+console.log(" ===============================================".blue);
 if (parserConfig.debug) {
   // @ts-ignore
-  console.log('configMobilizon'.blue, config);
+  console.log("configMobilizon".blue, config);
 }
 const utilsTools = new utils();
 
+let localMobilizonEventsByTitle = [];
 
 // trouver si un évènement existe déjà par son titre
+const doesThisEventAlreadyExistInLocalEvents = (rssEvent) => {
+  const eventAlreadyExists =
+    -1 !== localMobilizonEventsByTitle.indexOf(rssEvent.title);
+  if (!eventAlreadyExists) {
+    if (parserConfig.debug) {
+      console.log("ajouter l event ", htmlEscape(rssEvent.title));
+    }
+    addEventQuery(rssEvent);
+  }
+};
 
-let createEventQueries = '';
+let createEventQueries = "";
 let counterOfEventsToAdd = 0;
-const addEventQuery = utilsTools.agendadulibre.addQuery;
+const addEventQuery = (rssEvent) => {
-const runCreationQuery = utilsTools.runCreationQuery;
+  if (!createEventQueries) {
+    createEventQueries =
+      "INSERT INTO events(title, description, organizer_actor_id,inserted_at,updated_at, uuid, url, status, category, options,participants_stats, begins_on, ends_on) VALUES ";
+  }
+  if (counterOfEventsToAdd) {
+    createEventQueries += ` , `;
+  }
+
+  let title = "'" + htmlEscape(rssEvent.title) + "'";
+  let content = "'" + htmlEscape(rssEvent.content) + "'";
+  let uuid = uuidv4();
+  let uuidString = "'" + uuid + "'";
+  let eventUrl =
+    "'" + parserConfig.mobilizon_public_url + "/events/" + uuid + "'";
+  let begins_on = "'" + utilsTools.convertRssDate(rssEvent.date) + "'";
+  let ends_on = "'" + utilsTools.convertRssDate(rssEvent.date) + "'";
+  let baseOptions =
+    '{"offers": [], "program": null, "attendees": [], "show_end_time": true, "show_start_time": true, "comment_moderation": "allow_all", "anonymous_participation": true, "participation_condition": [], "show_participation_price": false, "maximum_attendee_capacity": 0, "remaining_attendee_capacity": 0, "hide_organizer_when_group_event": false, "show_remaining_attendee_capacity": false}';
+  let baseStats =
+    '{"creator": 1, "rejected": 0, "moderator": 0, "participant": 0, "not_approved": 0, "administrator": 0, "not_confirmed": 0}';
+  // TODO complete with date conversion
+  //  begins_on , ends_on expecting date format like this: "2020-12-17 23:00:00"
+
+  createEventQueries += `( ${title}, ${content}, ${parserConfig.feeder_mobilizon_user_id}, 'now()','now()', ${uuidString}, ${eventUrl}, 'confirmed' , 'meeting', ${baseOptions}, ${baseStats}, ${begins_on} , ${ends_on} )`;
+
+  counterOfEventsToAdd++;
+};
+const runCreationQuery = async () => {
+  createEventQueries = createEventQueries + ";";
+  utilsTools.writeFile("event_creation_query.psql", createEventQueries);
+
+  if (createEventQueries) {
+    console.log("     ");
+    console.log(" ⚙️⚙️⚙️ ");
+    console.log("     ");
+    console.log(" createEventQueries");
+    console.log(createEventQueries);
+
+    const res = await client.query(createEventQueries);
+    console.log("res", res);
+    return res;
+  }
+};
+
+const client = new Client({
+  host: "localhost",
+  user: parserConfig.db_user,
+  password: parserConfig.db_pass,
+  database: parserConfig.db_name,
+});
 
 (async () => {
-  console.log('⌛ interroger la BDD mobilizon postgresql');
+  console.log("⌛ interroger la BDD mobilizon postgresql");
 
-  utilsTools.setupClientPostgresql();
+  await client.connect();
+  console.log("✅ OK connecté à postgresql");
+  console.log(" ");
+  const res = await client.query("SELECT * from events");
+  console.log("💾 évènements enregistrés dans mobilizon : ", res.rows.length); // Hello world!
 
-  await utilsTools.client.connect();
+  utilsTools.postgresEventsExisting = res.rows;
-  console.log('✅ OK connecté à postgresql');
-  console.log(' ');
-  const res = await utilsTools.client.query('SELECT * from events');
-  console.log('💾 évènements enregistrés dans mobilizon : ', res.rows.length); // Hello world!
-
-  utilsTools.localMobilizonEventsByTitle = res.rows;
 
   if (parserConfig.askAgendaDuLibre) {
     // @ts-ignore
-    console.log(' ===================================================='.blue);
+    console.log(" ====================================================".blue);
     // @ts-ignore
-    console.log(' ====  demander à l\'agenda du libre son flux RSS ===='.blue);
+    console.log(" ====  demander à l'agenda du libre son flux RSS ====".blue);
     // @ts-ignore
-    console.log(' ===================================================='.blue);
+    console.log(" ====================================================".blue);
-
-    console.log('⌛ lecture du flux rss : ' + parserConfig.rss_feed_url);
-    fetch(parserConfig.rss_feed_url, { method: 'GET' })
-
-      .then((arrayOfEvents) => {
-        console.log('xml rss_feed_url count', arrayOfEvents.length);
-
-      });
 
+    console.log("⌛ lecture du flux rss : " + parserConfig.rss_feed_url);
     let feed = await parser.parseURL(parserConfig.rss_feed_url);
-    console.log('✅ flux rss obtenu');
+    console.log("✅ flux rss obtenu");
+    utilsTools.writeFile("agenda_du_libre_feed.xml", feed);
     console.log(feed.title);
 
-    console.log('⚙️ interprétation des évènements');
+    console.log("⚙️ interprétation des évènements");
     console.log(
-      '⌛ trouver les évènements qui ne font pas partie de la BDD mobilizon postgresql',
+      "⌛ trouver les évènements qui ne font pas partie de la BDD mobilizon postgresql"
     );
     feed.items.forEach((eventFound) => {
       console.log(eventFound.title);
@@ -73,50 +126,46 @@ const runCreationQuery = utilsTools.runCreationQuery;
   }
   if (parserConfig.askOsmCal) {
     // @ts-ignore
-    console.log(' ===================================================='.blue);
+    console.log(" ====================================================".blue);
     // @ts-ignore
-    console.log(' ====  demander à OSM cal ===='.blue);
+    console.log(" ====  demander à OSM cal ====".blue);
     // @ts-ignore
-    console.log(' ===================================================='.blue);
+    console.log(" ====================================================".blue);
 
-    fetch(parserConfig.osmcal_url, { method: 'GET' })
+    fetch(parserConfig.osmcal_url, { method: "GET" })
       .then((res) => res.json())
       .then((arrayOfEvents) => {
-        console.log('json osmcal count', arrayOfEvents.length);
+        console.log("json osmcal count", arrayOfEvents.length);
 
-        utilsTools.writeFile('osmcal.json', arrayOfEvents, 'json');
+        utilsTools.writeFile("osmcal.json", arrayOfEvents);
 
-        console.log('arrayOfEvents[0]', arrayOfEvents[1])
+        arrayOfEvents.forEach((eventFound) => {
-        utilsTools.osmcal.doesEventExists(arrayOfEvents[1])
+          console.log(eventFound.name);
-
+          // console.log('item', item)
-        // arrayOfEvents.forEach((eventFound) => {
+          utilsTools.osmCal.doesEventExists(eventFound);
-        //   utilsTools.osmcal.doesEventExists(eventFound);
+        });
-        // });
       });
   }
 
   if (parserConfig.runAddQueriesToMobilizon) {
     // @ts-ignore
     console.log(
-      '➕ rajouter les évènements manquants à la base mobilizon'.blue,
+      "➕ rajouter les évènements manquants à la base mobilizon".blue
     );
 
-    utilsTools.makeQuery();
+    await runCreationQuery();
-    utilsTools.runCreationQuery();
     // @ts-ignore
-    console.log(' ---------- '.green);
+    console.log(" ---------- ".green);
     // @ts-ignore
-    console.log(counterOfEventsToAdd, ' évènements ajoutés '.green);
+    console.log(counterOfEventsToAdd, " évènements ajoutés ".green);
     // @ts-ignore
-    console.log(' ---------- '.green);
+    console.log(" ---------- ".green);
     // @ts-ignore
-    console.log('✅ fermer la connec postgresql');
+    console.log("✅ ça c'est fait ".green);
-    await utilsTools.client.end();
-    console.log('✅ ça c\'est fait '.green);
   }
-
+  await client.end();
 })();
 
 exports = () => {
-  console.log('✅ hey ho', this);
+  console.log("hey ho", this);
 };

sources_examples/nominatim_reverse.json

@@ -1,22 +0,0 @@
-{
-  "place_id": 257545391,
-  "licence": "Data © OpenStreetMap contributors, ODbL 1.0. https://osm.org/copyright",
-  "osm_type": "relation",
-  "osm_id": 5753651,
-  "lat": "11.8447153",
-  "lon": "1.8991527451060923",
-  "display_name": "Tansarga, Tapoa, Est, Burkina Faso",
-  "address": {
-    "department": "Tansarga",
-    "province": "Tapoa",
-    "region": "Est",
-    "country": "Burkina Faso",
-    "country_code": "bf"
-  },
-  "boundingbox": [
-    "11.6508914",
-    "12.0385474",
-    "1.7610148",
-    "2.0502629"
-  ]
-}

utils.ts

@@ -1,216 +1,50 @@
-// @ts-ignore
-import parserConfig from "./config.ts";
-import {v4 as uuidv4} from "uuid";
-import {Client} from "pg";
-import {htmlEscape} from "escape-goat";
-
 const moment = require("moment");
 
 const fs = require("fs");
 
 class utils {
+  convertRssDate(rssDate) {
+    let converted = moment(rssDate)
+      .format("YYYY-MM-DD LTS")
+      .slice(0, -3)
+      .concat(".000000"); // in js format like 2021-03-12T19:00:00Z
 
-    /**
+    console.log("converted", converted);
-     * postgres functions
+    // like 2021-01-03 15:31:02.918940
-     */
+    return converted;
-    client;
+  }
-    makeQuery = () => {
-        this.createEventQueries = `INSERT INTO events(title, description, organizer_actor_id,inserted_at,updated_at, uuid, url, status, category, options,participants_stats, begins_on, ends_on) VALUES ${this.agendadulibre.queryToAdd} ${this.osmcal.queryToAdd};`;
-        this.writeFile("event_creation_query.psql", this.createEventQueries, "psql");
-    }
-    runCreationQuery = async () => {
-        if (this.createEventQueries) {
-            console.log("     ");
-            console.log(" ⚙️⚙️⚙️ ");
-            console.log("     ");
-            console.log(" createEventQueries");
-            console.log(this.createEventQueries);
 
-            const res = await this.client.query(this.createEventQueries);
+  testdateconvert() {
-            console.log("res", res);
+    let converted = this.convertRssDate("2021-03-12T19:00:00Z");
-            return res;
+    console.log("converted", converted);
+  }
+
+  postgresEventsExisting = [];
+  writeFile = (fileName, data) => {
+    // write file to disk
+    fs.writeFile(
+      `./sources_examples/${fileName}`,
+      JSON.stringify(data),
+      "utf8",
+      (err) => {
+        if (err) {
+          console.log(`Error writing file: ${err}`);
         } else {
-            console.log(" DISABLED createEventQueries");
+          console.log(`File ${fileName} is written successfully!`);
         }
-    };
+      }
-
+    );
-    /**
+  };
-     * memorizing properties
+  osmCal = {
-     */
+    queryToAdd: "",
-    createEventQueries = "";
+    doesEventExists: (event) => {},
-    counterOfEventsToAdd = 0;
+    addQuery: (event) => {},
-    localMobilizonEventsByTitle = [];
+  };
-
+  agendadulibre = {
-
+    queryToAdd: "",
-    /**
+    doesEventExists: (event) => {},
-     * converters
+    addQuery: (event) => {},
-     */
+  };
-
-    geocoderNominatim(coords){
-        // https://nominatim.openstreetmap.org/reverse?lat=<value>&lon=<value>&<params>
-        console.log('https://nominatim.openstreetmap.org/reverse?lat='+coords[0]+'&lon='+coords[0])+'&format=json'
-    }
-    convertRssDate(rssDate) {
-        let converted = moment(rssDate)
-            .format("YYYY-MM-DD LTS")
-            .slice(0, -3)
-            .concat(".000000"); // in js format like 2021-03-12T19:00:00Z
-
-        console.log("converted", converted);
-        // like 2021-01-03 15:31:02.918940
-        return converted;
-    }
-
-    convertCoordinateLinkOsmCal(coords) {
-
-
-        this.geocoderNominatim(coords);
-
-        return `  <div class='coordinates'>
-            <a href='https://www.openstreetmap.org/directions?from=&to=${coords[0]}%2C${coords[1]}'>voir le lieu sur une carte 
-            </a>
-            </div>  `
-    }
-
-    testdateconvert() {
-        let converted = this.convertRssDate("2021-03-12T19:00:00Z");
-        console.log("converted", converted);
-    }
-
-    /**
-     * file management
-     */
-    writeFile = (fileName, data, formatData) => {
-        let dataToSave = data;
-        if (formatData == 'json') {
-            dataToSave = JSON.stringify(data, null, 4)
-        }
-        // write file to disk
-        fs.writeFile(
-            `./sources_examples/${fileName}`,
-            dataToSave,
-            "utf8",
-            (err) => {
-                if (err) {
-                    console.log(`Error writing file: ${err}`);
-                } else {
-                    console.log(`File ${fileName} is written successfully!`);
-                }
-            }
-        );
-    };
-
-    /** ==============================
-     * importation sources
-     */
-
-    osmcal = {
-        queryToAdd: "",
-        counterOfEventsToAdd: 0,
-        getTitle: (event) => {
-            return event.name;
-            //+ ' '+ event.location.short
-        },
-        doesEventExists: (event) => {
-            const eventAlreadyExists =
-                -1 !== this.localMobilizonEventsByTitle.indexOf(this.osmcal.getTitle(event));
-            if (!eventAlreadyExists) {
-                if (parserConfig.debug) {
-                    console.log('ajouter l event ', htmlEscape(this.osmcal.getTitle(event)));
-                }
-                this.osmcal.addQuery(event);
-            }
-
-        },
-        addQuery: (event) => {
-
-            if (this.osmcal.queryToAdd) {
-                this.osmcal.queryToAdd += ` , `;
-            }
-            let title = "'" + htmlEscape(this.osmcal.getTitle(event)) + "'";
-            let content = "'" + htmlEscape(`${event.date.human} <br/>${+event.url} <br/> `) + "'";
-
-            if (event.location) {
-                content += `${event.location.detailed} `
-                if (event.location.venue) {
-                    content += ` <br/>
-                    ${event.location.venue}`
-                }
-                if (event.location.coords) {
-                    content += ` <br/> ` + this.convertCoordinateLinkOsmCal(event.location.coords)
-                }
-            }
-            console.log(' ')
-            console.log('   title', title)
-            console.log('   content', content)
-
-            let uuid = uuidv4();
-            let uuidString = "'" + uuid + "'";
-            let eventUrl =
-                "'" + parserConfig.mobilizon_public_url + "/events/" + uuid + "'";
-            let begins_on = "'" + this.convertRssDate(event.date.start) + "'";
-            let ends_on = "'" + this.convertRssDate(event.date.end) + "'";
-            let baseOptions =
-                '{"offers": [], "program": null, "attendees": [], "show_end_time": true, "show_start_time": true, "comment_moderation": "allow_all", "anonymous_participation": true, "participation_condition": [], "show_participation_price": false, "maximum_attendee_capacity": 0, "remaining_attendee_capacity": 0, "hide_organizer_when_group_event": false, "show_remaining_attendee_capacity": false}';
-            let baseStats =
-                '{"creator": 1, "rejected": 0, "moderator": 0, "participant": 0, "not_approved": 0, "administrator": 0, "not_confirmed": 0}';
-            //  begins_on , ends_on expecting date format like this: "2020-12-17 23:00:00"
-
-            this.osmcal.queryToAdd += `( ${title}, ${content}, ${parserConfig.feeder_mobilizon_user_id}, 'now()','now()', ${uuidString}, ${eventUrl}, 'confirmed' , 'meeting', ${baseOptions}, ${baseStats}, ${begins_on} , ${ends_on} )`;
-
-
-            this.osmcal.counterOfEventsToAdd++;
-            this.counterOfEventsToAdd++;
-
-        },
-    };
-    agendadulibre = {
-        queryToAdd: "",
-        counterOfEventsToAdd: 0,
-        doesEventExists: (event) => {
-            const eventAlreadyExists =
-                -1 !== this.localMobilizonEventsByTitle.indexOf(event.title);
-            if (!eventAlreadyExists) {
-                if (parserConfig.debug) {
-                    console.log('ajouter l event ', htmlEscape(event.title));
-                }
-                this.agendadulibre.addQuery(event);
-            }
-        },
-        addQuery: (event) => {
-            if (this.agendadulibre.queryToAdd) {
-                this.agendadulibre.queryToAdd += ` , `;
-            }
-
-            let title = "'" + htmlEscape(event.title) + "'";
-            let content = "'" + htmlEscape(event.content) + "'";
-            let uuid = uuidv4();
-            let uuidString = "'" + uuid + "'";
-            let eventUrl =
-                "'" + parserConfig.mobilizon_public_url + "/events/" + uuid + "'";
-            let begins_on = "'" + this.convertRssDate(event.date) + "'";
-            let ends_on = "'" + this.convertRssDate(event.date) + "'";
-            let baseOptions =
-                '{"offers": [], "program": null, "attendees": [], "show_end_time": true, "show_start_time": true, "comment_moderation": "allow_all", "anonymous_participation": true, "participation_condition": [], "show_participation_price": false, "maximum_attendee_capacity": 0, "remaining_attendee_capacity": 0, "hide_organizer_when_group_event": false, "show_remaining_attendee_capacity": false}';
-            let baseStats =
-                '{"creator": 1, "rejected": 0, "moderator": 0, "participant": 0, "not_approved": 0, "administrator": 0, "not_confirmed": 0}';
-            //  begins_on , ends_on expecting date format like this: "2020-12-17 23:00:00"
-
-            this.agendadulibre.queryToAdd += `( ${title}, ${content}, ${parserConfig.feeder_mobilizon_user_id}, 'now()','now()', ${uuidString}, ${eventUrl}, 'confirmed' , 'meeting', ${baseOptions}, ${baseStats}, ${begins_on} , ${ends_on} )`;
-
-            this.agendadulibre.counterOfEventsToAdd++;
-            this.counterOfEventsToAdd++;
-        }
-    };
-
-    setupClientPostgresql = () => {
-        this.client = new Client({
-            host: "localhost",
-            user: parserConfig.db_user,
-            password: parserConfig.db_pass,
-            database: parserConfig.db_name,
-        });
-    }
 }
 
 export default utils;