un script nodejs pour enrichir une instance mobilizon à partir d'un flux rss présentant des évènements. ce script détecte les évènements déjà existants dans l'instance mobilizon et ne crée que ceux qui n'y sont pas encore présent.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 

411 lines
18 KiB

// @ts-ignore
import parserConfig from "./config.ts";
import {v4 as uuidv4} from "uuid";
import {Client} from "pg";
import {htmlEscape} from "escape-goat";
import fetch from 'node-fetch';
const moment = require("moment");
const fs = require("fs");
let organizerActorId = 3;
let createEventQueryMobilizon = "mutation createEvent($organizerActorId: ID!, $attributedToId: ID, $title: String!, $description: String!, $beginsOn: DateTime!, $endsOn: DateTime, $status: EventStatus, $visibility: EventVisibility, $joinOptions: EventJoinOptions, $draft: Boolean, $tags: [String], $picture: MediaInput, $onlineAddress: String, $phoneAddress: String, $category: String, $physicalAddress: AddressInput, $options: EventOptionsInput, $contacts: [Contact]) {\n createEvent(\n organizerActorId: $organizerActorId\n attributedToId: $attributedToId\n title: $title\n description: $description\n beginsOn: $beginsOn\n endsOn: $endsOn\n status: $status\n visibility: $visibility\n joinOptions: $joinOptions\n draft: $draft\n tags: $tags\n picture: $picture\n onlineAddress: $onlineAddress\n phoneAddress: $phoneAddress\n category: $category\n physicalAddress: $physicalAddress\n options: $options\n contacts: $contacts\n ) {\n ...FullEvent\n __typename\n }\n}\n\nfragment FullEvent on Event {\n id\n uuid\n url\n local\n title\n description\n beginsOn\n endsOn\n status\n visibility\n joinOptions\n draft\n picture {\n id\n url\n name\n metadata {\n width\n height\n blurhash\n __typename\n }\n __typename\n }\n publishAt\n onlineAddress\n phoneAddress\n physicalAddress {\n ...AdressFragment\n __typename\n }\n organizerActor {\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n domain\n name\n url\n id\n summary\n __typename\n }\n contacts {\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n name\n summary\n domain\n url\n id\n __typename\n }\n attributedTo {\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n name\n summary\n domain\n url\n id\n __typename\n }\n participantStats {\n going\n notApproved\n participant\n __typename\n }\n tags {\n ...TagFragment\n __typename\n }\n relatedEvents {\n id\n uuid\n title\n beginsOn\n picture {\n id\n url\n name\n metadata {\n width\n height\n blurhash\n __typename\n }\n __typename\n }\n physicalAddress {\n id\n description\n __typename\n }\n organizerActor {\n id\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n domain\n name\n __typename\n }\n __typename\n }\n options {\n ...EventOptions\n __typename\n }\n metadata {\n key\n title\n value\n type\n __typename\n }\n __typename\n}\n\nfragment AdressFragment on Address {\n id\n description\n geom\n street\n locality\n postalCode\n region\n country\n type\n url\n originId\n __typename\n}\n\nfragment TagFragment on Tag {\n id\n slug\n title\n __typename\n}\n\nfragment EventOptions on EventOptions {\n maximumAttendeeCapacity\n remainingAttendeeCapacity\n showRemainingAttendeeCapacity\n anonymousParticipation\n showStartTime\n showEndTime\n offers {\n price\n priceCurrency\n url\n __typename\n }\n participationConditions {\n title\n content\n url\n __typename\n }\n attendees\n program\n commentModeration\n showParticipationPrice\n hideOrganizerWhenGroupEvent\n __typename\n}\n";
function proxy_fetch(theUrl: string, theOptions: any) {
return fetch(theUrl, theOptions);
}
/**
* utilitaries to manipulate scraped object and prepare queries to import in mobilizon
*/
class utils {
/**
* postgres functions
*/
client: any;
createEventQueries: string;
makeQuery = () => {
let createEventQueries: string = `INSERT INTO events(title, description, organizer_actor_id, inserted_at, updated_at, uuid, url, status, category, options, participants_stats, begins_on, ends_on) VALUES ${this.agendadulibre.queryToAdd} ${this.osmcal.queryToAdd};`;
this.createEventQueries = createEventQueries;
this.writeFile("event_creation_query.psql", this.createEventQueries, "psql");
}
runCreationQuery = async () => {
if (this.createEventQueries) {
console.log(" ");
console.log(" ⚙ ");
console.log(" ");
console.log(" createEventQueries");
console.log(this.createEventQueries);
const res = await this.client.query(this.createEventQueries);
console.log("res", res);
return res;
} else {
console.log(" DISABLED createEventQueries");
}
};
/**
* memorizing properties
*/
counterOfEventsToAdd = 0;
localMobilizonEventsByTitle: Array<string> = [];
newEvents: Array<any> = [];
/**
* converters
*/
geocoderNominatim(coords: any) {
// https://nominatim.openstreetmap.org/reverse?lat=<value>&lon=<value>&<params>
console.log('https://nominatim.openstreetmap.org/reverse?lat=' + coords[0] + '&lon=' + coords[0]) + '&format=json'
}
convertRssDateBDD(rssDate: any) {
let converted = moment(rssDate)
.format("YYYY-MM-DD LTS")
.slice(0, -3)
.concat(".000000"); // in js format like 2021-03-12T19:00:00Z
console.log("converted", converted);
// like 2021-01-03 15:31:02.918940
return converted;
}
convertRssDate(rssDate: string): string {
console.log('rssDate', rssDate);
if (rssDate) {
let converted = new Date(rssDate).toISOString(); // in js format like 2021-03-12T19:00:00Z
console.log("converted", converted);
// like 2021-01-03 15:31:02.918940
return converted;
} else {
return "";
}
}
convertCoordinateLinkOsmCal(coords: any) {
this.geocoderNominatim(coords);
return ` <div class='coordinates'>
<a href='https://www.openstreetmap.org/directions?from=&to=${coords[0]}%2C${coords[1]}'>voir le lieu sur une carte
</a>
</div> `
}
/**
* file management
*/
writeFile(fileName: string, data: any, formatData: any = 'json') {
let dataToSave = data;
if (formatData == 'json') {
dataToSave = JSON.stringify(data, null, 4)
}
// write file to disk
fs.writeFile(
`./output/${fileName}`,
dataToSave,
"utf8",
(err: any) => {
if (err) {
console.log(`Error writing file: ${err}`);
} else {
console.log(`File ${fileName} is written successfully!`);
}
}
);
};
/** ==============================
* importation sources
*/
public osmcal = {
queryToAdd: "",
counterOfEventsToAdd: 0,
getTitle: (event: any) => {
return event.name;
//+ ' '+ event.location.short
},
doesEventExists: (event: any): boolean => {
const eventAlreadyExists =
-1 !== this.localMobilizonEventsByTitle.indexOf(this.osmcal.getTitle(event));
if (!eventAlreadyExists) {
if (parserConfig.debug) {
console.log('ajouter l event ', htmlEscape(this.osmcal.getTitle(event)));
}
this.osmcal.addQuery(event);
}
return eventAlreadyExists;
},
addQuery: (event: any) => {
if (this.osmcal.queryToAdd) {
this.osmcal.queryToAdd += ` , `;
}
let title = "'" + htmlEscape(this.osmcal.getTitle(event)) + "'";
let content = "'" + htmlEscape(`${event.date.human} <br/>${+event.url} <br/> `) + "'";
if (event.location) {
content += `${event.location.detailed} `
if (event.location.venue) {
content += ` <br/>
${event.location.venue}`
}
if (event.location.coords) {
content += ` <br/> ` + this.convertCoordinateLinkOsmCal(event.location.coords)
}
}
console.log(' ')
console.log(' title', title)
let uuid = uuidv4();
let uuidString = "'" + uuid + "'";
let eventUrl =
"'" + parserConfig.mobilizon_public_url + "/events/" + uuid + "'";
let begins_on = "'" + this.convertRssDate(event.date.start) + "'";
let ends_on = "'" + this.convertRssDate(event.date.end) + "'";
let baseOptions =
'{"offers": [], "program": null, "attendees": [], "show_end_time": true, "show_start_time": true, "comment_moderation": "allow_all", "anonymous_participation": true, "participation_condition": [], "show_participation_price": false, "maximum_attendee_capacity": 0, "remaining_attendee_capacity": 0, "hide_organizer_when_group_event": false, "show_remaining_attendee_capacity": false}';
let baseStats =
'{"creator": 1, "rejected": 0, "moderator": 0, "participant": 0, "not_approved": 0, "administrator": 0, "not_confirmed": 0}';
// begins_on , ends_on expecting date format like this: "2020-12-17 23:00:00"
this.osmcal.queryToAdd += `( ${title}, ${content}, ${parserConfig.feeder_mobilizon_user_id}, 'now()','now()', ${uuidString}, ${eventUrl}, 'confirmed' , 'meeting', ${baseOptions}, ${baseStats}, ${begins_on} , ${ends_on} )`;
this.osmcal.counterOfEventsToAdd++;
this.counterOfEventsToAdd++;
},
};
public agendadulibre: any = {
queryToAdd: [],
queryToAddBDD: "",
counterOfEventsToAdd: 0,
doesEventExists: (event: any) => {
if (this.localMobilizonEventsByTitle.length) {
const eventAlreadyExists =
-1 !== this.localMobilizonEventsByTitle.indexOf(this.agendadulibre.uniqTitle(event));
return eventAlreadyExists;
} else {
console.log('aucun évènement dans localMobilizonEventsByTitle');
}
return false;
},
/**
* convert events from data scraping of the agenda du libre, to a string used as a comparison with new events
* @param event
*/
uniqTitle(event: any): string {
return moment(new Date(event.start_time))
.format("YYYY-MM-DD") + ' ' + event.title
},
/**
* convert events from bdd to a string used as a comparison with new events
* @param event
*/
uniqTitleBDD(event: any): string {
return moment(event.begins_on)
.format("YYYY-MM-DD") + ' ' + event.title
},
doesEventExistsFromJsonScrap: (event: any): boolean => {
const eventAlreadyExists =
-1 !== this.localMobilizonEventsByTitle.indexOf(this.agendadulibre.uniqTitle(event));
return eventAlreadyExists;
},
addQueryFromJsonScrap: (event: any) => {
let tags = event.tags.map(element => {
return element.name
})
tags.push("imported")
let newQuery = {
operationName: "createEvent",
query: createEventQueryMobilizon,
variables: {
attributedToId: null,
beginsOn: event.start_time,
contacts: [],
description:
"<address>" +
"<span class='city'>"+ event.city+"</span><br/>"+
"<span class='address'>"+ event.address+"</span><br/>"+
"<span class='place_name'>"+ event.place_name+"</span><br/>"+
"</address>"+
"<span class='contact'>"+ event.contact+"</span><br/>"+
"<p class='event_description'>" + event.description + "</p>",
draft: false,
endsOn: event.end_time,
joinOptions: "FREE",
onlineAddress: event.url,
options: {
anonymousParticipation: true,
attendees: [],
commentModeration: "ALLOW_ALL",
hideOrganizerWhenGroupEvent: false,
// maximumAttendeeCapacity: 200,
offers: [],
participationConditions: [],
program: "",
remainingAttendeeCapacity: 0,
showEndTime: true,
showParticipationPrice: false,
showRemainingAttendeeCapacity: false,
showStartTime: true
},
organizerActorId: organizerActorId,
phoneAddress: "",
status: "CONFIRMED",
tags: [
"imported"
],
title: event.title,
visibility: "PUBLIC"
}
}
return newQuery;
},
addQuery: (event: any) => {
console.log('event', event.title);
this.agendadulibre.queryToAdd.push(
{
operationName: "createEvent",
query: createEventQueryMobilizon,
variables: {
attributedToId: null,
beginsOn: event.date,
contacts: [],
description: "<p>" + event.content + "</p>",
draft: false,
endsOn: event.date,
joinOptions: "FREE",
onlineAddress: event.link,
options: {
anonymousParticipation: true,
attendees: [],
commentModeration: "ALLOW_ALL",
hideOrganizerWhenGroupEvent: false,
maximumAttendeeCapacity: 200,
offers: [],
participationConditions: [],
program: "",
remainingAttendeeCapacity: 0,
showEndTime: true,
showParticipationPrice: false,
showRemainingAttendeeCapacity: false,
showStartTime: true
},
organizerActorId: organizerActorId,
phoneAddress: "",
status: "CONFIRMED",
tags: [
"osm",
"openstreetmap",
"imported"
],
title: event.title,
visibility: "PUBLIC"
}
}
);
return this.agendadulibre.queryToAdd;
},
addNewEventsFromQueries: () => {
if (parserConfig.runAddQueriesToMobilizonAPI) {
console.log(
'➕ rajouter les évènements manquants par l\'API GraphQL',
this.agendadulibre.queryToAdd.length
);
let limiter = parserConfig.limit_persistence_of_new_events;
let counter = 0;
let bearerTokenIsOK = true;
this.agendadulibre.queryToAdd.forEach((graphQLquery: any) => {
if (limiter && counter <= parserConfig.max_new_events && bearerTokenIsOK) {
counter++;
console.log(counter, ' * ', graphQLquery.variables.title);
const body = {
operationName: "createEvent",
query: graphQLquery.query,
variables: graphQLquery.variables
}
if (parserConfig.enableFetch) {
let theUrl: string = parserConfig.dev_mode ? parserConfig.dev_url : parserConfig.mobilizon_public_url;
let theOptions: any = {
method: "post",
body: JSON.stringify(body),
headers: {
"Content-Type": "application/json",
"authorization": "Bearer " + parserConfig.bearer_token,
}
}
proxy_fetch(theUrl, theOptions)
.then((res: any) => {
let status = res.status;
console.log('status', status);
if (status === 401) {
console.error(' /!\\ ------------------ ERROR: Bearer token invalid ------------------')
bearerTokenIsOK = false;
} else if (status === 200) {
console.log('succès');
}
res.json()
})
.then((json: any) => console.log(json))
.catch((err: any) => console.log(err))
} else {
console.log('---- le fetch est désactivé');
}
}
});
}
}
};
setupClientPostgresql = () => {
this.client = new Client({
host: "localhost",
user: parserConfig.db_user,
password: parserConfig.db_pass,
database: parserConfig.db_name,
});
}
createEventQueriesForApi(EventsToCreate: any[]) {
if (EventsToCreate.length) {
console.log('we will create events', EventsToCreate.length);
} else {
console.log('no events to create');
}
}
}
export default utils;