add conversion of osmcal
This commit is contained in:
parent
e5eea3f803
commit
0bb0d27ead
@ -12,8 +12,8 @@ const parserConfig = {
|
||||
db_user: "mobilizon",
|
||||
db_pass: "mobilizon",
|
||||
// other configs
|
||||
askAgendaDuLibre: true, // should we fetch the Agenda du libre feed ?
|
||||
askOsmCal: false,
|
||||
askAgendaDuLibre: false, // should we fetch the Agenda du libre feed ?
|
||||
askOsmCal: true,
|
||||
runAddQueriesToMobilizon: false,
|
||||
debug: false,
|
||||
};
|
||||
|
||||
107
index.js
107
index.js
@ -1,75 +1,74 @@
|
||||
// @author tykayn contact@cipherbliss.com www.cipherbliss.com
|
||||
// imports
|
||||
import utils from "./utils.ts";
|
||||
import parserConfig from "./config.ts";
|
||||
import { htmlEscape } from "escape-goat";
|
||||
import Parser from "rss-parser";
|
||||
const fetch = require("node-fetch");
|
||||
import utils from './utils.ts';
|
||||
import parserConfig from './config.ts';
|
||||
import { htmlEscape } from 'escape-goat';
|
||||
import Parser from 'rss-parser';
|
||||
|
||||
const fetch = require('node-fetch');
|
||||
let parser = new Parser();
|
||||
|
||||
// @ts-ignore
|
||||
console.log(" ===============================================".blue);
|
||||
console.log(' ==============================================='.blue);
|
||||
// @ts-ignore
|
||||
console.log(" ==== Rss Feeder for mobilizon - by tykayn ====".blue);
|
||||
console.log(' ==== Rss Feeder for mobilizon - by tykayn ===='.blue);
|
||||
// @ts-ignore
|
||||
console.log(" ===============================================".blue);
|
||||
console.log(' ==============================================='.blue);
|
||||
if (parserConfig.debug) {
|
||||
// @ts-ignore
|
||||
console.log("configMobilizon".blue, config);
|
||||
console.log('configMobilizon'.blue, config);
|
||||
}
|
||||
const utilsTools = new utils();
|
||||
|
||||
let localMobilizonEventsByTitle = [];
|
||||
|
||||
// trouver si un évènement existe déjà par son titre
|
||||
const doesThisEventAlreadyExistInLocalEvents = (rssEvent) => {
|
||||
const eventAlreadyExists =
|
||||
-1 !== localMobilizonEventsByTitle.indexOf(rssEvent.title);
|
||||
if (!eventAlreadyExists) {
|
||||
if (parserConfig.debug) {
|
||||
console.log("ajouter l event ", htmlEscape(rssEvent.title));
|
||||
}
|
||||
addEventQuery(rssEvent);
|
||||
}
|
||||
};
|
||||
|
||||
let createEventQueries = "";
|
||||
let createEventQueries = '';
|
||||
let counterOfEventsToAdd = 0;
|
||||
const addEventQuery = utilsTools.agendadulibre.addQuery;
|
||||
const runCreationQuery = utilsTools.runCreationQuery;
|
||||
|
||||
|
||||
|
||||
(async () => {
|
||||
console.log("⌛ interroger la BDD mobilizon postgresql");
|
||||
console.log('⌛ interroger la BDD mobilizon postgresql');
|
||||
|
||||
utilsTools.setupClientPostgresql()
|
||||
utilsTools.setupClientPostgresql();
|
||||
|
||||
await utilsTools.client.connect();
|
||||
console.log("✅ OK connecté à postgresql");
|
||||
console.log(" ");
|
||||
const res = await utilsTools.client.query("SELECT * from events");
|
||||
console.log("💾 évènements enregistrés dans mobilizon : ", res.rows.length); // Hello world!
|
||||
console.log('✅ OK connecté à postgresql');
|
||||
console.log(' ');
|
||||
const res = await utilsTools.client.query('SELECT * from events');
|
||||
console.log('💾 évènements enregistrés dans mobilizon : ', res.rows.length); // Hello world!
|
||||
|
||||
utilsTools.postgresEventsExisting = res.rows;
|
||||
utilsTools.localMobilizonEventsByTitle = res.rows;
|
||||
|
||||
if (parserConfig.askAgendaDuLibre) {
|
||||
// @ts-ignore
|
||||
console.log(" ====================================================".blue);
|
||||
console.log(' ===================================================='.blue);
|
||||
// @ts-ignore
|
||||
console.log(" ==== demander à l'agenda du libre son flux RSS ====".blue);
|
||||
console.log(' ==== demander à l\'agenda du libre son flux RSS ===='.blue);
|
||||
// @ts-ignore
|
||||
console.log(" ====================================================".blue);
|
||||
console.log(' ===================================================='.blue);
|
||||
|
||||
console.log('⌛ lecture du flux rss : ' + parserConfig.rss_feed_url);
|
||||
fetch(parserConfig.rss_feed_url, { method: 'GET' })
|
||||
|
||||
.then((arrayOfEvents) => {
|
||||
console.log('xml rss_feed_url count', arrayOfEvents.length);
|
||||
|
||||
arrayOfEvents.forEach((eventFound) => {
|
||||
console.log(eventFound.name);
|
||||
// console.log('item', item)
|
||||
utilsTools.osmcal.doesEventExists(eventFound);
|
||||
});
|
||||
});
|
||||
|
||||
console.log("⌛ lecture du flux rss : " + parserConfig.rss_feed_url);
|
||||
let feed = await parser.parseURL(parserConfig.rss_feed_url);
|
||||
console.log("✅ flux rss obtenu");
|
||||
utilsTools.writeFile("agenda_du_libre_feed.xml", feed);
|
||||
console.log('✅ flux rss obtenu');
|
||||
console.log(feed.title);
|
||||
|
||||
console.log("⚙️ interprétation des évènements");
|
||||
console.log('⚙️ interprétation des évènements');
|
||||
console.log(
|
||||
"⌛ trouver les évènements qui ne font pas partie de la BDD mobilizon postgresql"
|
||||
'⌛ trouver les évènements qui ne font pas partie de la BDD mobilizon postgresql',
|
||||
);
|
||||
feed.items.forEach((eventFound) => {
|
||||
console.log(eventFound.title);
|
||||
@ -79,18 +78,18 @@ const runCreationQuery = utilsTools.runCreationQuery;
|
||||
}
|
||||
if (parserConfig.askOsmCal) {
|
||||
// @ts-ignore
|
||||
console.log(" ====================================================".blue);
|
||||
console.log(' ===================================================='.blue);
|
||||
// @ts-ignore
|
||||
console.log(" ==== demander à OSM cal ====".blue);
|
||||
console.log(' ==== demander à OSM cal ===='.blue);
|
||||
// @ts-ignore
|
||||
console.log(" ====================================================".blue);
|
||||
console.log(' ===================================================='.blue);
|
||||
|
||||
fetch(parserConfig.osmcal_url, { method: "GET" })
|
||||
fetch(parserConfig.osmcal_url, { method: 'GET' })
|
||||
.then((res) => res.json())
|
||||
.then((arrayOfEvents) => {
|
||||
console.log("json osmcal count", arrayOfEvents.length);
|
||||
console.log('json osmcal count', arrayOfEvents.length);
|
||||
|
||||
utilsTools.writeFile("osmcal.json", arrayOfEvents);
|
||||
utilsTools.writeFile('osmcal.json', arrayOfEvents, 'json');
|
||||
|
||||
arrayOfEvents.forEach((eventFound) => {
|
||||
console.log(eventFound.name);
|
||||
@ -103,23 +102,25 @@ const runCreationQuery = utilsTools.runCreationQuery;
|
||||
if (parserConfig.runAddQueriesToMobilizon) {
|
||||
// @ts-ignore
|
||||
console.log(
|
||||
"➕ rajouter les évènements manquants à la base mobilizon".blue
|
||||
'➕ rajouter les évènements manquants à la base mobilizon'.blue,
|
||||
);
|
||||
|
||||
utilsTools.makeQuery();
|
||||
utilsTools.runCreationQuery();
|
||||
utilsTools.makeQuery();
|
||||
utilsTools.runCreationQuery();
|
||||
// @ts-ignore
|
||||
console.log(" ---------- ".green);
|
||||
console.log(' ---------- '.green);
|
||||
// @ts-ignore
|
||||
console.log(counterOfEventsToAdd, " évènements ajoutés ".green);
|
||||
console.log(counterOfEventsToAdd, ' évènements ajoutés '.green);
|
||||
// @ts-ignore
|
||||
console.log(" ---------- ".green);
|
||||
console.log(' ---------- '.green);
|
||||
// @ts-ignore
|
||||
console.log("✅ ça c'est fait ".green);
|
||||
console.log('✅ fermer la connec postgresql');
|
||||
await utilsTools.client.end();
|
||||
console.log('✅ ça c\'est fait '.green);
|
||||
}
|
||||
await utilsTools.client.end();
|
||||
|
||||
})();
|
||||
|
||||
exports = () => {
|
||||
console.log("hey ho", this);
|
||||
console.log('✅ hey ho', this);
|
||||
};
|
||||
|
||||
File diff suppressed because one or more lines are too long
58
utils.ts
58
utils.ts
@ -12,7 +12,7 @@ class utils {
|
||||
client;
|
||||
makeQuery = () => {
|
||||
this.createEventQueries = `INSERT INTO events(title, description, organizer_actor_id,inserted_at,updated_at, uuid, url, status, category, options,participants_stats, begins_on, ends_on) VALUES ${this.agendadulibre.queryToAdd} ${this.osmcal.queryToAdd};`;
|
||||
this.writeFile("event_creation_query.psql", this.createEventQueries);
|
||||
this.writeFile("event_creation_query.psql", this.createEventQueries, "psql");
|
||||
}
|
||||
runCreationQuery = async () => {
|
||||
if (this.createEventQueries) {
|
||||
@ -31,6 +31,7 @@ class utils {
|
||||
};
|
||||
createEventQueries = "";
|
||||
counterOfEventsToAdd = 0;
|
||||
localMobilizonEventsByTitle=[];
|
||||
|
||||
|
||||
convertRssDate(rssDate) {
|
||||
@ -49,12 +50,15 @@ class utils {
|
||||
console.log("converted", converted);
|
||||
}
|
||||
|
||||
postgresEventsExisting = [];
|
||||
writeFile = (fileName, data) => {
|
||||
writeFile = (fileName, data, formatData) => {
|
||||
let dataToSave = data;
|
||||
if (formatData == 'json') {
|
||||
dataToSave = JSON.stringify(data, null, 4)
|
||||
}
|
||||
// write file to disk
|
||||
fs.writeFile(
|
||||
`./sources_examples/${fileName}`,
|
||||
JSON.stringify(data),
|
||||
dataToSave,
|
||||
"utf8",
|
||||
(err) => {
|
||||
if (err) {
|
||||
@ -72,7 +76,31 @@ class utils {
|
||||
},
|
||||
addQuery: (event) => {
|
||||
|
||||
this.agendadulibre.counterOfEventsToAdd++;
|
||||
if (this.osmcal.queryToAdd) {
|
||||
this.osmcal.queryToAdd += ` , `;
|
||||
}
|
||||
let title = "'" + htmlEscape(event.name+ ' '+ event.location.short) + "'";
|
||||
let content = "'" + htmlEscape(event.date.human+ ' <br/>'+ event.location.detailed + ' <br/>' + event.location.venue + ' <br/>' +event.url + ' <br/>' + event.coords[0]+','+event.coords[1]) + "'";
|
||||
console.log(' ')
|
||||
console.log(' title', title)
|
||||
console.log(' content', content)
|
||||
|
||||
let uuid = uuidv4();
|
||||
let uuidString = "'" + uuid + "'";
|
||||
let eventUrl =
|
||||
"'" + parserConfig.mobilizon_public_url + "/events/" + uuid + "'";
|
||||
let begins_on = "'" + this.convertRssDate(event.date.start) + "'";
|
||||
let ends_on = "'" + this.convertRssDate(event.date.end) + "'";
|
||||
let baseOptions =
|
||||
'{"offers": [], "program": null, "attendees": [], "show_end_time": true, "show_start_time": true, "comment_moderation": "allow_all", "anonymous_participation": true, "participation_condition": [], "show_participation_price": false, "maximum_attendee_capacity": 0, "remaining_attendee_capacity": 0, "hide_organizer_when_group_event": false, "show_remaining_attendee_capacity": false}';
|
||||
let baseStats =
|
||||
'{"creator": 1, "rejected": 0, "moderator": 0, "participant": 0, "not_approved": 0, "administrator": 0, "not_confirmed": 0}';
|
||||
// begins_on , ends_on expecting date format like this: "2020-12-17 23:00:00"
|
||||
|
||||
this.osmcal.queryToAdd += `( ${title}, ${content}, ${parserConfig.feeder_mobilizon_user_id}, 'now()','now()', ${uuidString}, ${eventUrl}, 'confirmed' , 'meeting', ${baseOptions}, ${baseStats}, ${begins_on} , ${ends_on} )`;
|
||||
|
||||
|
||||
this.osmcal.counterOfEventsToAdd++;
|
||||
this.counterOfEventsToAdd++;
|
||||
|
||||
},
|
||||
@ -80,21 +108,29 @@ class utils {
|
||||
agendadulibre = {
|
||||
queryToAdd: "",
|
||||
counterOfEventsToAdd: 0,
|
||||
doesEventExists: (event) => {
|
||||
doesEventExists: (rssEvent) => {
|
||||
const eventAlreadyExists =
|
||||
-1 !== this.localMobilizonEventsByTitle.indexOf(rssEvent.title);
|
||||
if (!eventAlreadyExists) {
|
||||
if (parserConfig.debug) {
|
||||
console.log('ajouter l event ', htmlEscape(rssEvent.title));
|
||||
}
|
||||
this.agendadulibre.addQuery(rssEvent);
|
||||
}
|
||||
},
|
||||
addQuery: (rssEvent) => {
|
||||
addQuery: (event) => {
|
||||
if (this.agendadulibre.queryToAdd) {
|
||||
this.agendadulibre.queryToAdd += ` , `;
|
||||
}
|
||||
|
||||
let title = "'" + htmlEscape(rssEvent.title) + "'";
|
||||
let content = "'" + htmlEscape(rssEvent.content) + "'";
|
||||
let title = "'" + htmlEscape(event.title) + "'";
|
||||
let content = "'" + htmlEscape(event.content) + "'";
|
||||
let uuid = uuidv4();
|
||||
let uuidString = "'" + uuid + "'";
|
||||
let eventUrl =
|
||||
"'" + parserConfig.mobilizon_public_url + "/events/" + uuid + "'";
|
||||
let begins_on = "'" + this.convertRssDate(rssEvent.date) + "'";
|
||||
let ends_on = "'" + this.convertRssDate(rssEvent.date) + "'";
|
||||
let begins_on = "'" + this.convertRssDate(event.date) + "'";
|
||||
let ends_on = "'" + this.convertRssDate(event.date) + "'";
|
||||
let baseOptions =
|
||||
'{"offers": [], "program": null, "attendees": [], "show_end_time": true, "show_start_time": true, "comment_moderation": "allow_all", "anonymous_participation": true, "participation_condition": [], "show_participation_price": false, "maximum_attendee_capacity": 0, "remaining_attendee_capacity": 0, "hide_organizer_when_group_event": false, "show_remaining_attendee_capacity": false}';
|
||||
let baseStats =
|
||||
|
||||
Loading…
Reference in New Issue
Block a user