rss-feeder-mobilizon/utils.ts

412 lines
18 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

// @ts-ignore
import parserConfig from "./config.ts";
import {v4 as uuidv4} from "uuid";
import {Client} from "pg";
import {htmlEscape} from "escape-goat";
import fetch from 'node-fetch';
const moment = require("moment");
const fs = require("fs");
let organizerActorId = 3;
let createEventQueryMobilizon = "mutation createEvent($organizerActorId: ID!, $attributedToId: ID, $title: String!, $description: String!, $beginsOn: DateTime!, $endsOn: DateTime, $status: EventStatus, $visibility: EventVisibility, $joinOptions: EventJoinOptions, $draft: Boolean, $tags: [String], $picture: MediaInput, $onlineAddress: String, $phoneAddress: String, $category: String, $physicalAddress: AddressInput, $options: EventOptionsInput, $contacts: [Contact]) {\n createEvent(\n organizerActorId: $organizerActorId\n attributedToId: $attributedToId\n title: $title\n description: $description\n beginsOn: $beginsOn\n endsOn: $endsOn\n status: $status\n visibility: $visibility\n joinOptions: $joinOptions\n draft: $draft\n tags: $tags\n picture: $picture\n onlineAddress: $onlineAddress\n phoneAddress: $phoneAddress\n category: $category\n physicalAddress: $physicalAddress\n options: $options\n contacts: $contacts\n ) {\n ...FullEvent\n __typename\n }\n}\n\nfragment FullEvent on Event {\n id\n uuid\n url\n local\n title\n description\n beginsOn\n endsOn\n status\n visibility\n joinOptions\n draft\n picture {\n id\n url\n name\n metadata {\n width\n height\n blurhash\n __typename\n }\n __typename\n }\n publishAt\n onlineAddress\n phoneAddress\n physicalAddress {\n ...AdressFragment\n __typename\n }\n organizerActor {\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n domain\n name\n url\n id\n summary\n __typename\n }\n contacts {\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n name\n summary\n domain\n url\n id\n __typename\n }\n attributedTo {\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n name\n summary\n domain\n url\n id\n __typename\n }\n participantStats {\n going\n notApproved\n participant\n __typename\n }\n tags {\n ...TagFragment\n __typename\n }\n relatedEvents {\n id\n uuid\n title\n beginsOn\n picture {\n id\n url\n name\n metadata {\n width\n height\n blurhash\n __typename\n }\n __typename\n }\n physicalAddress {\n id\n description\n __typename\n }\n organizerActor {\n id\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n domain\n name\n __typename\n }\n __typename\n }\n options {\n ...EventOptions\n __typename\n }\n metadata {\n key\n title\n value\n type\n __typename\n }\n __typename\n}\n\nfragment AdressFragment on Address {\n id\n description\n geom\n street\n locality\n postalCode\n region\n country\n type\n url\n originId\n __typename\n}\n\nfragment TagFragment on Tag {\n id\n slug\n title\n __typename\n}\n\nfragment EventOptions on EventOptions {\n maximumAttendeeCapacity\n remainingAttendeeCapacity\n showRemainingAttendeeCapacity\n anonymousParticipation\n showStartTime\n showEndTime\n offers {\n price\n priceCurrency\n url\n __typename\n }\n participationConditions {\n title\n content\n url\n __typename\n }\n attendees\n program\n commentModeration\n showParticipationPrice\n hideOrganizerWhenGroupEvent\n __typename\n}\n";
function proxy_fetch(theUrl: string, theOptions: any) {
return fetch(theUrl, theOptions);
}
/**
* utilitaries to manipulate scraped object and prepare queries to import in mobilizon
*/
class utils {
/**
* postgres functions
*/
client: any;
createEventQueries: string;
makeQuery = () => {
let createEventQueries: string = `INSERT INTO events(title, description, organizer_actor_id, inserted_at, updated_at, uuid, url, status, category, options, participants_stats, begins_on, ends_on) VALUES ${this.agendadulibre.queryToAdd} ${this.osmcal.queryToAdd};`;
this.createEventQueries = createEventQueries;
this.writeFile("event_creation_query.psql", this.createEventQueries, "psql");
}
runCreationQuery = async () => {
if (this.createEventQueries) {
console.log(" ");
console.log(" ⚙️⚙️⚙️ ");
console.log(" ");
console.log(" createEventQueries");
console.log(this.createEventQueries);
const res = await this.client.query(this.createEventQueries);
console.log("res", res);
return res;
} else {
console.log(" DISABLED createEventQueries");
}
};
/**
* memorizing properties
*/
counterOfEventsToAdd = 0;
localMobilizonEventsByTitle: Array<string> = [];
newEvents: Array<any> = [];
/**
* converters
*/
geocoderNominatim(coords: any) {
// https://nominatim.openstreetmap.org/reverse?lat=<value>&lon=<value>&<params>
console.log('https://nominatim.openstreetmap.org/reverse?lat=' + coords[0] + '&lon=' + coords[0]) + '&format=json'
}
convertRssDateBDD(rssDate: any) {
let converted = moment(rssDate)
.format("YYYY-MM-DD LTS")
.slice(0, -3)
.concat(".000000"); // in js format like 2021-03-12T19:00:00Z
console.log("converted", converted);
// like 2021-01-03 15:31:02.918940
return converted;
}
convertRssDate(rssDate: string): string {
console.log('rssDate', rssDate);
if (rssDate) {
let converted = new Date(rssDate).toISOString(); // in js format like 2021-03-12T19:00:00Z
console.log("converted", converted);
// like 2021-01-03 15:31:02.918940
return converted;
} else {
return "";
}
}
convertCoordinateLinkOsmCal(coords: any) {
this.geocoderNominatim(coords);
return ` <div class='coordinates'>
<a href='https://www.openstreetmap.org/directions?from=&to=${coords[0]}%2C${coords[1]}'>voir le lieu sur une carte
</a>
</div> `
}
/**
* file management
*/
writeFile(fileName: string, data: any, formatData: any = 'json') {
let dataToSave = data;
if (formatData == 'json') {
dataToSave = JSON.stringify(data, null, 4)
}
// write file to disk
fs.writeFile(
`./output/${fileName}`,
dataToSave,
"utf8",
(err: any) => {
if (err) {
console.log(`Error writing file: ${err}`);
} else {
console.log(`File ${fileName} is written successfully!`);
}
}
);
};
/** ==============================
* importation sources
*/
public osmcal = {
queryToAdd: "",
counterOfEventsToAdd: 0,
getTitle: (event: any) => {
return event.name;
//+ ' '+ event.location.short
},
doesEventExists: (event: any): boolean => {
const eventAlreadyExists =
-1 !== this.localMobilizonEventsByTitle.indexOf(this.osmcal.getTitle(event));
if (!eventAlreadyExists) {
if (parserConfig.debug) {
console.log('ajouter l event ', htmlEscape(this.osmcal.getTitle(event)));
}
this.osmcal.addQuery(event);
}
return eventAlreadyExists;
},
addQuery: (event: any) => {
if (this.osmcal.queryToAdd) {
this.osmcal.queryToAdd += ` , `;
}
let title = "'" + htmlEscape(this.osmcal.getTitle(event)) + "'";
let content = "'" + htmlEscape(`${event.date.human} <br/>${+event.url} <br/> `) + "'";
if (event.location) {
content += `${event.location.detailed} `
if (event.location.venue) {
content += ` <br/>
${event.location.venue}`
}
if (event.location.coords) {
content += ` <br/> ` + this.convertCoordinateLinkOsmCal(event.location.coords)
}
}
console.log(' ')
console.log(' title', title)
let uuid = uuidv4();
let uuidString = "'" + uuid + "'";
let eventUrl =
"'" + parserConfig.mobilizon_public_url + "/events/" + uuid + "'";
let begins_on = "'" + this.convertRssDate(event.date.start) + "'";
let ends_on = "'" + this.convertRssDate(event.date.end) + "'";
let baseOptions =
'{"offers": [], "program": null, "attendees": [], "show_end_time": true, "show_start_time": true, "comment_moderation": "allow_all", "anonymous_participation": true, "participation_condition": [], "show_participation_price": false, "maximum_attendee_capacity": 0, "remaining_attendee_capacity": 0, "hide_organizer_when_group_event": false, "show_remaining_attendee_capacity": false}';
let baseStats =
'{"creator": 1, "rejected": 0, "moderator": 0, "participant": 0, "not_approved": 0, "administrator": 0, "not_confirmed": 0}';
// begins_on , ends_on expecting date format like this: "2020-12-17 23:00:00"
this.osmcal.queryToAdd += `( ${title}, ${content}, ${parserConfig.feeder_mobilizon_user_id}, 'now()','now()', ${uuidString}, ${eventUrl}, 'confirmed' , 'meeting', ${baseOptions}, ${baseStats}, ${begins_on} , ${ends_on} )`;
this.osmcal.counterOfEventsToAdd++;
this.counterOfEventsToAdd++;
},
};
public agendadulibre: any = {
queryToAdd: [],
queryToAddBDD: "",
counterOfEventsToAdd: 0,
doesEventExists: (event: any) => {
if (this.localMobilizonEventsByTitle.length) {
const eventAlreadyExists =
-1 !== this.localMobilizonEventsByTitle.indexOf(this.agendadulibre.uniqTitle(event));
return eventAlreadyExists;
} else {
console.log('aucun évènement dans localMobilizonEventsByTitle');
}
return false;
},
/**
* convert events from data scraping of the agenda du libre, to a string used as a comparison with new events
* @param event
*/
uniqTitle(event: any): string {
return moment(new Date(event.start_time))
.format("YYYY-MM-DD") + ' ' + event.title
},
/**
* convert events from bdd to a string used as a comparison with new events
* @param event
*/
uniqTitleBDD(event: any): string {
return moment(event.begins_on)
.format("YYYY-MM-DD") + ' ' + event.title
},
doesEventExistsFromJsonScrap: (event: any): boolean => {
const eventAlreadyExists =
-1 !== this.localMobilizonEventsByTitle.indexOf(this.agendadulibre.uniqTitle(event));
return eventAlreadyExists;
},
addQueryFromJsonScrap: (event: any) => {
let tags = event.tags.map(element => {
return element.name
})
tags.push("imported")
let newQuery = {
operationName: "createEvent",
query: createEventQueryMobilizon,
variables: {
attributedToId: null,
beginsOn: event.start_time,
contacts: [],
description:
"<address>" +
"<span class='city'>"+ event.city+"</span><br/>"+
"<span class='address'>"+ event.address+"</span><br/>"+
"<span class='place_name'>"+ event.place_name+"</span><br/>"+
"</address>"+
"<span class='contact'>"+ event.contact+"</span><br/>"+
"<p class='event_description'>" + event.description + "</p>",
draft: false,
endsOn: event.end_time,
joinOptions: "FREE",
onlineAddress: event.url,
options: {
anonymousParticipation: true,
attendees: [],
commentModeration: "ALLOW_ALL",
hideOrganizerWhenGroupEvent: false,
// maximumAttendeeCapacity: 200,
offers: [],
participationConditions: [],
program: "",
remainingAttendeeCapacity: 0,
showEndTime: true,
showParticipationPrice: false,
showRemainingAttendeeCapacity: false,
showStartTime: true
},
organizerActorId: organizerActorId,
phoneAddress: "",
status: "CONFIRMED",
tags: [
"imported"
],
title: event.title,
visibility: "PUBLIC"
}
}
return newQuery;
},
addQuery: (event: any) => {
console.log('event', event.title);
this.agendadulibre.queryToAdd.push(
{
operationName: "createEvent",
query: createEventQueryMobilizon,
variables: {
attributedToId: null,
beginsOn: event.date,
contacts: [],
description: "<p>" + event.content + "</p>",
draft: false,
endsOn: event.date,
joinOptions: "FREE",
onlineAddress: event.link,
options: {
anonymousParticipation: true,
attendees: [],
commentModeration: "ALLOW_ALL",
hideOrganizerWhenGroupEvent: false,
maximumAttendeeCapacity: 200,
offers: [],
participationConditions: [],
program: "",
remainingAttendeeCapacity: 0,
showEndTime: true,
showParticipationPrice: false,
showRemainingAttendeeCapacity: false,
showStartTime: true
},
organizerActorId: organizerActorId,
phoneAddress: "",
status: "CONFIRMED",
tags: [
"osm",
"openstreetmap",
"imported"
],
title: event.title,
visibility: "PUBLIC"
}
}
);
return this.agendadulibre.queryToAdd;
},
addNewEventsFromQueries: () => {
if (parserConfig.runAddQueriesToMobilizonAPI) {
console.log(
' rajouter les évènements manquants par l\'API GraphQL',
this.agendadulibre.queryToAdd.length
);
let limiter = parserConfig.limit_persistence_of_new_events;
let counter = 0;
let bearerTokenIsOK = true;
this.agendadulibre.queryToAdd.forEach((graphQLquery: any) => {
if (limiter && counter <= parserConfig.max_new_events && bearerTokenIsOK) {
counter++;
console.log(counter, ' * ', graphQLquery.variables.title);
const body = {
operationName: "createEvent",
query: graphQLquery.query,
variables: graphQLquery.variables
}
if (parserConfig.enableFetch) {
let theUrl: string = parserConfig.dev_mode ? parserConfig.dev_url : parserConfig.mobilizon_public_url;
let theOptions: any = {
method: "post",
body: JSON.stringify(body),
headers: {
"Content-Type": "application/json",
"authorization": "Bearer " + parserConfig.bearer_token,
}
}
proxy_fetch(theUrl, theOptions)
.then((res: any) => {
let status = res.status;
console.log('status', status);
if (status === 401) {
console.error(' /!\\ ------------------ ERROR: Bearer token invalid ------------------')
bearerTokenIsOK = false;
} else if (status === 200) {
console.log('succès');
}
res.json()
})
.then((json: any) => console.log(json))
.catch((err: any) => console.log(err))
} else {
console.log('---- le fetch est désactivé');
}
}
});
}
}
};
setupClientPostgresql = () => {
this.client = new Client({
host: "localhost",
user: parserConfig.db_user,
password: parserConfig.db_pass,
database: parserConfig.db_name,
});
}
createEventQueriesForApi(EventsToCreate: any[]) {
if (EventsToCreate.length) {
console.log('we will create events', EventsToCreate.length);
} else {
console.log('no events to create');
}
}
}
export default utils;