un script nodejs pour enrichir une instance mobilizon à partir d'un flux rss présentant des évènements. ce script détecte les évènements déjà existants dans l'instance mobilizon et ne crée que ceux qui n'y sont pas encore présent.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 

373 lines
18 KiB

// @ts-ignore
import parserConfig from "./config.ts";
import {v4 as uuidv4} from "uuid";
import {Client} from "pg";
import {htmlEscape} from "escape-goat";
const moment = require("moment");
const fs = require("fs");
let createEventQueryMobilizon = "mutation createEvent($organizerActorId: ID!, $attributedToId: ID, $title: String!, $description: String!, $beginsOn: DateTime!, $endsOn: DateTime, $status: EventStatus, $visibility: EventVisibility, $joinOptions: EventJoinOptions, $draft: Boolean, $tags: [String], $picture: MediaInput, $onlineAddress: String, $phoneAddress: String, $category: String, $physicalAddress: AddressInput, $options: EventOptionsInput, $contacts: [Contact]) {\n createEvent(\n organizerActorId: $organizerActorId\n attributedToId: $attributedToId\n title: $title\n description: $description\n beginsOn: $beginsOn\n endsOn: $endsOn\n status: $status\n visibility: $visibility\n joinOptions: $joinOptions\n draft: $draft\n tags: $tags\n picture: $picture\n onlineAddress: $onlineAddress\n phoneAddress: $phoneAddress\n category: $category\n physicalAddress: $physicalAddress\n options: $options\n contacts: $contacts\n ) {\n ...FullEvent\n __typename\n }\n}\n\nfragment FullEvent on Event {\n id\n uuid\n url\n local\n title\n description\n beginsOn\n endsOn\n status\n visibility\n joinOptions\n draft\n picture {\n id\n url\n name\n metadata {\n width\n height\n blurhash\n __typename\n }\n __typename\n }\n publishAt\n onlineAddress\n phoneAddress\n physicalAddress {\n ...AdressFragment\n __typename\n }\n organizerActor {\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n domain\n name\n url\n id\n summary\n __typename\n }\n contacts {\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n name\n summary\n domain\n url\n id\n __typename\n }\n attributedTo {\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n name\n summary\n domain\n url\n id\n __typename\n }\n participantStats {\n going\n notApproved\n participant\n __typename\n }\n tags {\n ...TagFragment\n __typename\n }\n relatedEvents {\n id\n uuid\n title\n beginsOn\n picture {\n id\n url\n name\n metadata {\n width\n height\n blurhash\n __typename\n }\n __typename\n }\n physicalAddress {\n id\n description\n __typename\n }\n organizerActor {\n id\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n domain\n name\n __typename\n }\n __typename\n }\n options {\n ...EventOptions\n __typename\n }\n metadata {\n key\n title\n value\n type\n __typename\n }\n __typename\n}\n\nfragment AdressFragment on Address {\n id\n description\n geom\n street\n locality\n postalCode\n region\n country\n type\n url\n originId\n __typename\n}\n\nfragment TagFragment on Tag {\n id\n slug\n title\n __typename\n}\n\nfragment EventOptions on EventOptions {\n maximumAttendeeCapacity\n remainingAttendeeCapacity\n showRemainingAttendeeCapacity\n anonymousParticipation\n showStartTime\n showEndTime\n offers {\n price\n priceCurrency\n url\n __typename\n }\n participationConditions {\n title\n content\n url\n __typename\n }\n attendees\n program\n commentModeration\n showParticipationPrice\n hideOrganizerWhenGroupEvent\n __typename\n}\n";
/**
* utilitaries to manipulate scraped object and prepare queries to import in mobilizon
*/
class utils {
/**
* postgres functions
*/
client: any;
createEventQueries: string;
makeQuery = () => {
let createEventQueries: string = `INSERT INTO events(title, description, organizer_actor_id, inserted_at, updated_at, uuid, url, status, category, options, participants_stats, begins_on, ends_on) VALUES ${this.agendadulibre.queryToAdd} ${this.osmcal.queryToAdd};`;
this.createEventQueries = createEventQueries;
this.writeFile("event_creation_query.psql", this.createEventQueries, "psql");
}
runCreationQuery = async () => {
if (this.createEventQueries) {
console.log(" ");
console.log(" ⚙ ");
console.log(" ");
console.log(" createEventQueries");
console.log(this.createEventQueries);
const res = await this.client.query(this.createEventQueries);
console.log("res", res);
return res;
} else {
console.log(" DISABLED createEventQueries");
}
};
/**
* memorizing properties
*/
counterOfEventsToAdd = 0;
localMobilizonEventsByTitle: Array<string> = [];
/**
* converters
*/
geocoderNominatim(coords: any) {
// https://nominatim.openstreetmap.org/reverse?lat=<value>&lon=<value>&<params>
console.log('https://nominatim.openstreetmap.org/reverse?lat=' + coords[0] + '&lon=' + coords[0]) + '&format=json'
}
convertRssDateBDD(rssDate: any) {
let converted = moment(rssDate)
.format("YYYY-MM-DD LTS")
.slice(0, -3)
.concat(".000000"); // in js format like 2021-03-12T19:00:00Z
console.log("converted", converted);
// like 2021-01-03 15:31:02.918940
return converted;
}
convertRssDate(rssDate: string): string {
console.log('rssDate', rssDate);
if (rssDate) {
let converted = new Date(rssDate).toISOString(); // in js format like 2021-03-12T19:00:00Z
console.log("converted", converted);
// like 2021-01-03 15:31:02.918940
return converted;
} else {
return "";
}
}
convertCoordinateLinkOsmCal(coords: any) {
this.geocoderNominatim(coords);
return ` <div class='coordinates'>
<a href='https://www.openstreetmap.org/directions?from=&to=${coords[0]}%2C${coords[1]}'>voir le lieu sur une carte
</a>
</div> `
}
testdateconvert() {
let converted = this.convertRssDate("2021-03-12T19:00:00Z");
}
/**
* file management
*/
writeFile(fileName: string, data: any, formatData: any = 'json') {
let dataToSave = data;
if (formatData == 'json') {
dataToSave = JSON.stringify(data, null, 4)
}
// write file to disk
fs.writeFile(
`./output/${fileName}`,
dataToSave,
"utf8",
(err: any) => {
if (err) {
console.log(`Error writing file: ${err}`);
} else {
console.log(`File ${fileName} is written successfully!`);
}
}
);
};
/** ==============================
* importation sources
*/
public osmcal = {
queryToAdd: "",
counterOfEventsToAdd: 0,
getTitle: (event: any) => {
return event.name;
//+ ' '+ event.location.short
},
doesEventExists: (event: any): boolean => {
const eventAlreadyExists =
-1 !== this.localMobilizonEventsByTitle.indexOf(this.osmcal.getTitle(event));
if (!eventAlreadyExists) {
if (parserConfig.debug) {
console.log('ajouter l event ', htmlEscape(this.osmcal.getTitle(event)));
}
this.osmcal.addQuery(event);
}
return eventAlreadyExists;
},
addQuery: (event: any) => {
if (this.osmcal.queryToAdd) {
this.osmcal.queryToAdd += ` , `;
}
let title = "'" + htmlEscape(this.osmcal.getTitle(event)) + "'";
let content = "'" + htmlEscape(`${event.date.human} <br/>${+event.url} <br/> `) + "'";
if (event.location) {
content += `${event.location.detailed} `
if (event.location.venue) {
content += ` <br/>
${event.location.venue}`
}
if (event.location.coords) {
content += ` <br/> ` + this.convertCoordinateLinkOsmCal(event.location.coords)
}
}
console.log(' ')
console.log(' title', title)
let uuid = uuidv4();
let uuidString = "'" + uuid + "'";
let eventUrl =
"'" + parserConfig.mobilizon_public_url + "/events/" + uuid + "'";
let begins_on = "'" + this.convertRssDate(event.date.start) + "'";
let ends_on = "'" + this.convertRssDate(event.date.end) + "'";
let baseOptions =
'{"offers": [], "program": null, "attendees": [], "show_end_time": true, "show_start_time": true, "comment_moderation": "allow_all", "anonymous_participation": true, "participation_condition": [], "show_participation_price": false, "maximum_attendee_capacity": 0, "remaining_attendee_capacity": 0, "hide_organizer_when_group_event": false, "show_remaining_attendee_capacity": false}';
let baseStats =
'{"creator": 1, "rejected": 0, "moderator": 0, "participant": 0, "not_approved": 0, "administrator": 0, "not_confirmed": 0}';
// begins_on , ends_on expecting date format like this: "2020-12-17 23:00:00"
this.osmcal.queryToAdd += `( ${title}, ${content}, ${parserConfig.feeder_mobilizon_user_id}, 'now()','now()', ${uuidString}, ${eventUrl}, 'confirmed' , 'meeting', ${baseOptions}, ${baseStats}, ${begins_on} , ${ends_on} )`;
this.osmcal.counterOfEventsToAdd++;
this.counterOfEventsToAdd++;
},
};
public agendadulibre: any = {
queryToAdd: [],
queryToAddBDD: "",
counterOfEventsToAdd: 0,
doesEventExists: (event: any) => {
if (this.localMobilizonEventsByTitle.length) {
const eventAlreadyExists =
-1 !== this.localMobilizonEventsByTitle.indexOf(this.agendadulibre.uniqTitle(event));
if (!eventAlreadyExists) {
if (parserConfig.debug) {
console.log('ajouter l event ', htmlEscape(this.agendadulibre.uniqTitle(event)));
}
this.agendadulibre.addQuery(event);
}
return eventAlreadyExists;
} else {
console.log('aucun évènement dans localMobilizonEventsByTitle');
}
return false;
},
/**
* convert events from data scraping of the agenda du libre, to a string used as a comparison with new events
* @param event
*/
uniqTitle(event: any): string {
return event.start_date + ' ' + event.title
},
/**
* convert events from bdd to a string used as a comparison with new events
* @param event
*/
uniqTitleBDD(event: any): string {
return event.begins_on + ' ' + event.title
},
doesEventExistsFromJsonScrap: (event: any): boolean => {
const eventAlreadyExists =
-1 !== this.localMobilizonEventsByTitle.indexOf(htmlEscape(event.start_date + ' ' + event.title));
if (!eventAlreadyExists) {
if (parserConfig.debug) {
console.log('ajouter l event ', htmlEscape(event.start_date + ' ' + event.title));
}
this.agendadulibre.addQuery(event);
}
return eventAlreadyExists;
},
addQueryFromJsonScrap: (event: any) => {
console.log('event', event.title);
this.agendadulibre.queryToAdd.push(
{
operationName: "createEvent",
query: createEventQueryMobilizon,
variables: {
attributedToId: null,
beginsOn: event.start_time,
contacts: [],
description: "<p>" + event.description + "</p>",
draft: false,
endsOn: event.end_time,
joinOptions: "FREE",
onlineAddress: event.url,
options: {
anonymousParticipation: true,
attendees: [],
commentModeration: "ALLOW_ALL",
hideOrganizerWhenGroupEvent: false,
maximumAttendeeCapacity: 200,
offers: [],
participationConditions: [],
program: "",
remainingAttendeeCapacity: 0,
showEndTime: true,
showParticipationPrice: false,
showRemainingAttendeeCapacity: false,
showStartTime: true
},
organizerActorId: "3",
phoneAddress: "",
status: "CONFIRMED",
tags: [
"osm",
"openstreetmap",
"imported"
],
title: event.title,
visibility: "PUBLIC"
}
}
);
return this.agendadulibre.queryToAdd;
},
addQuery: (event: any) => {
console.log('event', event.title);
this.agendadulibre.queryToAdd.push(
{
operationName: "createEvent",
query: createEventQueryMobilizon,
variables: {
attributedToId: null,
beginsOn: event.date,
contacts: [],
description: "<p>" + event.content + "</p>",
draft: false,
endsOn: event.date,
joinOptions: "FREE",
onlineAddress: event.link,
options: {
anonymousParticipation: true,
attendees: [],
commentModeration: "ALLOW_ALL",
hideOrganizerWhenGroupEvent: false,
maximumAttendeeCapacity: 200,
offers: [],
participationConditions: [],
program: "",
remainingAttendeeCapacity: 0,
showEndTime: true,
showParticipationPrice: false,
showRemainingAttendeeCapacity: false,
showStartTime: true
},
organizerActorId: "3",
phoneAddress: "",
status: "CONFIRMED",
tags: [
"osm",
"openstreetmap",
"imported"
],
title: event.title,
visibility: "PUBLIC"
}
}
);
return this.agendadulibre.queryToAdd;
},
addQueryBDD: (event: any) => {
if (this.agendadulibre.queryToAdd) {
this.agendadulibre.queryToAdd += ` , `;
}
let title = "'" + htmlEscape(event.title) + "'";
let content = "'" + htmlEscape(event.content) + "'";
let uuid = uuidv4();
let uuidString = "'" + uuid + "'";
let eventUrl =
"'" + parserConfig.mobilizon_public_url + "/events/" + uuid + "'";
let begins_on = "'" + this.convertRssDate(event.date) + "'";
let ends_on = "'" + this.convertRssDate(event.date) + "'";
let baseOptions =
'{"offers": [], "program": null, "attendees": [], "show_end_time": true, "show_start_time": true, "comment_moderation": "allow_all", "anonymous_participation": true, "participation_condition": [], "show_participation_price": false, "maximum_attendee_capacity": 0, "remaining_attendee_capacity": 0, "hide_organizer_when_group_event": false, "show_remaining_attendee_capacity": false}';
let baseStats =
'{"creator": 1, "rejected": 0, "moderator": 0, "participant": 0, "not_approved": 0, "administrator": 0, "not_confirmed": 0}';
// begins_on , ends_on expecting date format like this: "2020-12-17 23:00:00"
this.agendadulibre.queryToAdd += `( ${title}, ${content}, ${parserConfig.feeder_mobilizon_user_id}, 'now()','now()', ${uuidString}, ${eventUrl}, 'confirmed' , 'meeting', ${baseOptions}, ${baseStats}, ${begins_on} , ${ends_on} )`;
this.agendadulibre.counterOfEventsToAdd++;
this.counterOfEventsToAdd++;
}
};
setupClientPostgresql = () => {
this.client = new Client({
host: "localhost",
user: parserConfig.db_user,
password: parserConfig.db_pass,
database: parserConfig.db_name,
});
}
createEventQueriesForApi(EventsToCreate: any[]) {
if (EventsToCreate.length) {
console.log('we will create events', EventsToCreate.length);
} else {
console.log('no events to create');
}
}
}
export default utils;