tykayn/rss-feeder-mobilizon
2
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

list events ok

Browse Source
This commit is contained in:
Tykayn 2022-01-10 11:48:28 +01:00 committed by tykayn
parent 52a0ec6284
commit 3f012c258f
4 changed files with 419 additions and 141 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
config.ts
View File

@ -7,15 +7,17 @@ const parserConfig = {
// mobilizon instance
feeder_mobilizon_user_id: "3", // organizer_actor_id pour rattacher les nouveaux évènements à un utilisateur. 3 est le premier admin créé.
mobilizon_public_url: "https://mobilizon.openstreetmap.fr", // url publique de l'instance mobilizon pour créer les url d'évènements
dev_url: "http://localhost:4000/api", // url publique de l'instance mobilizon pour créer les url d'évènements
// database of mobilizon
db_name: "mobilizon_dev",
db_user: "mobilizon",
db_pass: "0eHmz99Rb2WEhPKbwGJuAU-Da04jVV36QjuOOeJDOsK_Q1w7pikqIR04WqHoZLQi",
// other configs
askAgendaDuLibre: true, // should we fetch the Agenda du libre feed ?
askOsmCal: true,
askOsmCal: false,
runAddQueriesToMobilizonBDD: false,
debug: false,
organizerActorId: 3, // ID of the user who imports events in the mobilizon instance. 3 is the first admin user created, usually.
runAddQueriesToMobilizonAPI: true
};

195
index.ts
View File

@ -2,7 +2,6 @@
// imports
import utils from './utils';
import parserConfig from './config';
import { htmlEscape } from 'escape-goat';
// @ts-ignore
import Parser from 'rss-parser';
@ -17,8 +16,8 @@ console.log(' ==== Rss Feeder for mobilizon - by tykayn ===='.blue);
// @ts-ignore
console.log(' ==============================================='.blue);
if (parserConfig.debug) {
// @ts-ignore
console.log('configMobilizon'.blue, config);
// @ts-ignore
console.log('configMobilizon'.blue, config);
}
const utilsTools = new utils();
@ -30,95 +29,127 @@ let counterOfEventsToAdd = 0;
const addEventQuery = utilsTools.agendadulibre.addQuery;
const runCreationQuery = utilsTools.runCreationQuery;
(async () => {
console.log('⌛ interroger la BDD mobilizon postgresql');
console.log('⌛ interroger la BDD mobilizon postgresql');
utilsTools.setupClientPostgresql();
async function init() {
await utilsTools.client.connect();
console.log('✅ OK connecté à postgresql');
console.log(' ');
const res = await utilsTools.client.query('SELECT * from events');
console.log('💾 évènements enregistrés dans mobilizon : ', res.rows.length); // Hello world!
console.log('res', res);
utilsTools.localMobilizonEventsByTitle = res.rows;
utilsTools.setupClientPostgresql();
if (parserConfig.askAgendaDuLibre) {
// @ts-ignore
console.log(' ===================================================='.blue);
// @ts-ignore
console.log(' ==== demander à l\'agenda du libre son flux RSS ===='.blue);
// @ts-ignore
console.log(' ===================================================='.blue);
await utilsTools.client.connect();
console.log('✅ OK connecté à postgresql');
console.log(' ');
const res = await utilsTools.client.query('SELECT * from events');
console.log('💾 évènements enregistrés dans mobilizon : ', res.rows.length); // Hello world!
// console.log('res', res);
utilsTools.localMobilizonEventsByTitle = res.rows;
console.log('⌛ lecture du flux rss : ' + parserConfig.rss_feed_url);
fetch(parserConfig.rss_feed_url, { method: 'GET' })
if (parserConfig.askAgendaDuLibre) {
// @ts-ignore
console.log(' ===================================================='.blue);
// @ts-ignore
console.log(' ==== demander à l\'agenda du libre son flux RSS ===='.blue);
// @ts-ignore
console.log(' ===================================================='.blue);
.then((arrayOfEvents:any) => {
console.log('xml rss_feed_url count', Object.keys(arrayOfEvents).length);
console.log('⌛ lecture du flux rss : ' + parserConfig.rss_feed_url);
fetch(parserConfig.rss_feed_url, {method: 'GET'})
});
.then((arrayOfEvents: any) => {
console.log('xml rss_feed_url count', Object.keys(arrayOfEvents).length);
let feed = await parser.parseURL(parserConfig.rss_feed_url);
console.log('✅ flux rss obtenu');
console.log(feed.title);
});
console.log('⚙️ interprétation des évènements');
console.log(
'⌛ trouver les évènements qui ne font pas partie de la BDD mobilizon postgresql',
);
feed.items.forEach((eventFound) => {
console.log(eventFound.title);
// console.log('item', item)
utilsTools.agendadulibre.doesEventExists(eventFound);
});
}
if (parserConfig.askOsmCal) {
// @ts-ignore
console.log(' ===================================================='.blue);
// @ts-ignore
console.log(' ==== demander à OSM cal ===='.blue);
// @ts-ignore
console.log(' ===================================================='.blue);
let feed = await parser.parseURL(parserConfig.rss_feed_url);
console.log('✅ flux rss obtenu');
console.log(feed.title);
fetch(parserConfig.osmcal_url, { method: 'GET' })
.then((res:any) => res.json())
.then((arrayOfEvents:any) => {
console.log('json osmcal count', arrayOfEvents.length);
utilsTools.writeFile('osmcal.json', arrayOfEvents, 'json');
console.log('arrayOfEvents[0]', arrayOfEvents[1])
utilsTools.osmcal.doesEventExists(arrayOfEvents[1])
let EventsToCreate:any = [];
arrayOfEvents.forEach((eventFound:any) => {
if(!utilsTools.osmcal.doesEventExists(eventFound)){
EventsToCreate.push(eventFound);
};
console.log('⚙️ interprétation des évènements');
console.log(
'⌛ trouver les évènements qui ne font pas partie de la BDD mobilizon postgresql',
);
feed.items.forEach((eventFound) => {
console.log(eventFound.title);
// console.log('item', item)
utilsTools.agendadulibre.doesEventExists(eventFound);
});
utilsTools.createEventQueriesForApi(EventsToCreate)
}
if (parserConfig.askOsmCal) {
// @ts-ignore
console.log(' ===================================================='.blue);
// @ts-ignore
console.log(' ==== demander à OSM cal ===='.blue);
// @ts-ignore
console.log(' ===================================================='.blue);
});
}
fetch(parserConfig.osmcal_url, {method: 'GET'})
.then((res: any) => res.json())
.then((arrayOfEvents: any) => {
console.log('json osmcal count', arrayOfEvents.length);
if (parserConfig.runAddQueriesToMobilizonAPI) {
console.log(
'➕ rajouter les évènements manquants par l\'API GraphQL',
);
// utilsTools.writeFile('osmcal.json', arrayOfEvents, 'json');
let EventsToCreate: any = [];
arrayOfEvents.forEach((eventFound: any) => {
if (!utilsTools.osmcal.doesEventExists(eventFound)) {
EventsToCreate.push(eventFound);
}
;
});
utilsTools.createEventQueriesForApi(EventsToCreate)
});
}
if (parserConfig.runAddQueriesToMobilizonAPI) {
console.log(
'➕ rajouter les évènements manquants par l\'API GraphQL',
utilsTools.agendadulibre.queryToAdd.length
);
let limiter = true;
let counter = 0;
utilsTools.agendadulibre.queryToAdd.forEach((graphQLquery: any) => {
}
if (parserConfig.runAddQueriesToMobilizonBDD) {
// @ts-ignore
console.log(
'➕ rajouter les évènements manquants à la base mobilizon',
);
if (limiter && !counter) {
counter++;
console.log('graphQLquery.variables.title', graphQLquery.variables.title);
utilsTools.makeQuery();
utilsTools.runCreationQuery();
// @ts-ignore
const body = {
operationName: "createEvent",
query: graphQLquery.query,
variables: graphQLquery.variables
}
// fetch(parserConfig.dev_url, {
// method: "post",
// body: JSON.stringify(body),
// headers: {"Content-Type": "application/json"}
// })
// .then(res => {
// res.json()
// console.log('succès');
// })
// .then(json => console.log(json))
// .catch(err => console.log(err))
}
if (parserConfig.runAddQueriesToMobilizonBDD) {
// @ts-ignore
console.log(
'➕ rajouter les évènements manquants à la base mobilizon',
);
utilsTools.makeQuery();
utilsTools.runCreationQuery();
}
});
}
// @ts-ignore
console.log(' ---------- '.green);
// @ts-ignore
console.log(counterOfEventsToAdd, ' évènements ajoutés '.green);
@ -126,10 +157,10 @@ const runCreationQuery = utilsTools.runCreationQuery;
console.log(' ---------- '.green);
// @ts-ignore
console.log('✅ fermer la connec postgresql');
await utilsTools.client.end();
utilsTools.client.end();
console.log('✅ ça c\'est fait ');
}
console.log(' ');
}
})();
console.log('✅ hey ho', this);
(init)();
console.log('✅ hey ho', this);

274
sources_examples/rss/events.rss
View File

File diff suppressed because one or more lines are too long

87
utils.ts
View File

@ -13,11 +13,14 @@ class utils {
/**
* postgres functions
*/
client:any;
client: any;
makeQuery = () => {
this.createEventQueries = `INSERT INTO events(title, description, organizer_actor_id, inserted_at, updated_at,
uuid, url, status, category, options, participants_stats,
begins_on, ends_on) VALUES ${this.agendadulibre.queryToAdd} ${this.osmcal.queryToAdd};`;
begins_on, ends_on)
VALUES
${this.agendadulibre.queryToAdd}
${this.osmcal.queryToAdd};`;
this.writeFile("event_creation_query.psql", this.createEventQueries, "psql");
}
runCreationQuery = async () => {
@ -41,7 +44,7 @@ class utils {
*/
createEventQueries = "";
counterOfEventsToAdd = 0;
localMobilizonEventsByTitle : Array<string> = [];
localMobilizonEventsByTitle: Array<string> = [];
/**
@ -50,10 +53,11 @@ class utils {
geocoderNominatim(coords: any) {
// https://nominatim.openstreetmap.org/reverse?lat=<value>&lon=<value>&<params>
console.log('https://nominatim.openstreetmap.org/reverse?lat=' + coords[0] + '&lon=' + coords[0]) + '&format=json'
}
convertRssDate(rssDate: any) {
convertRssDateBDD(rssDate: any) {
let converted = moment(rssDate)
.format("YYYY-MM-DD LTS")
.slice(0, -3)
@ -64,6 +68,20 @@ class utils {
return converted;
}
convertRssDate(rssDate: string): string {
console.log('rssDate', rssDate);
if (rssDate) {
let converted = new Date(rssDate).toISOString(); // in js format like 2021-03-12T19:00:00Z
console.log("converted", converted);
// like 2021-01-03 15:31:02.918940
return converted;
} else {
return "";
}
}
convertCoordinateLinkOsmCal(coords: any) {
@ -77,7 +95,6 @@ class utils {
testdateconvert() {
let converted = this.convertRssDate("2021-03-12T19:00:00Z");
console.log("converted", converted);
}
/**
@ -93,7 +110,7 @@ class utils {
`./sources_examples/${fileName}`,
dataToSave,
"utf8",
(err:any) => {
(err: any) => {
if (err) {
console.log(`Error writing file: ${err}`);
} else {
@ -114,7 +131,7 @@ class utils {
return event.name;
//+ ' '+ event.location.short
},
doesEventExists: (event: any) : boolean => {
doesEventExists: (event: any): boolean => {
const eventAlreadyExists =
-1 !== this.localMobilizonEventsByTitle.indexOf(this.osmcal.getTitle(event));
if (!eventAlreadyExists) {
@ -168,8 +185,9 @@ class utils {
},
};
agendadulibre = {
queryToAdd: "",
agendadulibre:any = {
queryToAdd: [],
queryToAddBDD: "",
counterOfEventsToAdd: 0,
doesEventExists: (event: any) => {
if (this.localMobilizonEventsByTitle.length) {
@ -182,10 +200,57 @@ class utils {
this.agendadulibre.addQuery(event);
}
return eventAlreadyExists;
} else {
console.log('aucun évènement dans localMobilizonEventsByTitle');
}
return false;
},
addQuery: (event: any) => {
this.agendadulibre.queryToAdd.push(
{
operationName: "createEvent",
query: "mutation createEvent($organizerActorId: ID!, $attributedToId: ID, $title: String!, $description: String!, $beginsOn: DateTime!, $endsOn: DateTime, $status: EventStatus, $visibility: EventVisibility, $joinOptions: EventJoinOptions, $draft: Boolean, $tags: [String], $picture: MediaInput, $onlineAddress: String, $phoneAddress: String, $category: String, $physicalAddress: AddressInput, $options: EventOptionsInput, $contacts: [Contact]) {\n createEvent(\n organizerActorId: $organizerActorId\n attributedToId: $attributedToId\n title: $title\n description: $description\n beginsOn: $beginsOn\n endsOn: $endsOn\n status: $status\n visibility: $visibility\n joinOptions: $joinOptions\n draft: $draft\n tags: $tags\n picture: $picture\n onlineAddress: $onlineAddress\n phoneAddress: $phoneAddress\n category: $category\n physicalAddress: $physicalAddress\n options: $options\n contacts: $contacts\n ) {\n ...FullEvent\n __typename\n }\n}\n\nfragment FullEvent on Event {\n id\n uuid\n url\n local\n title\n description\n beginsOn\n endsOn\n status\n visibility\n joinOptions\n draft\n picture {\n id\n url\n name\n metadata {\n width\n height\n blurhash\n __typename\n }\n __typename\n }\n publishAt\n onlineAddress\n phoneAddress\n physicalAddress {\n ...AdressFragment\n __typename\n }\n organizerActor {\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n domain\n name\n url\n id\n summary\n __typename\n }\n contacts {\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n name\n summary\n domain\n url\n id\n __typename\n }\n attributedTo {\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n name\n summary\n domain\n url\n id\n __typename\n }\n participantStats {\n going\n notApproved\n participant\n __typename\n }\n tags {\n ...TagFragment\n __typename\n }\n relatedEvents {\n id\n uuid\n title\n beginsOn\n picture {\n id\n url\n name\n metadata {\n width\n height\n blurhash\n __typename\n }\n __typename\n }\n physicalAddress {\n id\n description\n __typename\n }\n organizerActor {\n id\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n domain\n name\n __typename\n }\n __typename\n }\n options {\n ...EventOptions\n __typename\n }\n metadata {\n key\n title\n value\n type\n __typename\n }\n __typename\n}\n\nfragment AdressFragment on Address {\n id\n description\n geom\n street\n locality\n postalCode\n region\n country\n type\n url\n originId\n __typename\n}\n\nfragment TagFragment on Tag {\n id\n slug\n title\n __typename\n}\n\nfragment EventOptions on EventOptions {\n maximumAttendeeCapacity\n remainingAttendeeCapacity\n showRemainingAttendeeCapacity\n anonymousParticipation\n showStartTime\n showEndTime\n offers {\n price\n priceCurrency\n url\n __typename\n }\n participationConditions {\n title\n content\n url\n __typename\n }\n attendees\n program\n commentModeration\n showParticipationPrice\n hideOrganizerWhenGroupEvent\n __typename\n}\n",
variables: {
attributedToId: null,
beginsOn: event.date_start,
contacts: [],
description: "<p>"+event.content+"</p>",
draft: false,
endsOn: event.date_end,
joinOptions: "FREE",
onlineAddress: "https://cipherbliss.com",
options: {
anonymousParticipation: true,
attendees: [],
commentModeration: "ALLOW_ALL",
hideOrganizerWhenGroupEvent: false,
maximumAttendeeCapacity: 200,
offers: [],
participationConditions: [],
program: "",
remainingAttendeeCapacity: 0,
showEndTime: true,
showParticipationPrice: false,
showRemainingAttendeeCapacity: false,
showStartTime: true
},
organizerActorId: "3",
phoneAddress: "",
status: "CONFIRMED",
tags: [
"osm",
"openstreetmap",
"imported"
],
title: event.title,
visibility: "PUBLIC"
}
}
);
return this.agendadulibre.queryToAdd;
},
addQueryBDD: (event: any) => {
if (this.agendadulibre.queryToAdd) {
this.agendadulibre.queryToAdd += ` , `;
}
@ -221,9 +286,9 @@ class utils {
}
createEventQueriesForApi(EventsToCreate: any[]) {
if(EventsToCreate.length){
if (EventsToCreate.length) {
console.log('we will create events', EventsToCreate.length);
}else{
} else {
console.log('no events to create');
}