tykayn
/
rss-feeder-mobilizon
2
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

compare events date

This commit is contained in:
Tykayn 2022-01-13 14:30:45 +01:00 committed by tykayn
parent 7228429013
commit bea58afa1e
4 changed files with 89 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
config.ts
View File

@ -20,10 +20,12 @@ const parserConfig = {
organizerActorId: 3, // ID of the user who imports events in the mobilizon instance. 3 is the first admin user created, usually. organizerActorId: 3, // ID of the user who imports events in the mobilizon instance. 3 is the first admin user created, usually.
runAddQueriesToMobilizonAPI: true, runAddQueriesToMobilizonAPI: true,
enableFetch: true, // enable persisting of new events via API enableFetch: true, // enable persisting of new events via API
// enableFetch: false,
dev_mode: true, // dev mode uses localhost instance of mobilizon running on port 4000 dev_mode: true, // dev mode uses localhost instance of mobilizon running on port 4000
limit_persistence_of_new_events:true, limit_persistence_of_new_events:true,
max_new_events: 1, max_new_events_in_scrapping: 10,
bearer_token: "eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJtb2JpbGl6b24iLCJleHAiOjE2NDIwNzIyMTQsImlhdCI6MTY0MjA3MTMxNCwiaXNzIjoibW9iaWxpem9uIiwianRpIjoiMjBiNGMyNGEtODViYi00MWFlLTk2MjYtMjZhZjFlYThlMTliIiwibmJmIjoxNjQyMDcxMzEzLCJzdWIiOiJVc2VyOjEiLCJ0eXAiOiJhY2Nlc3MifQ.N-QBR0lfj8KX2icUjOVImahIKvZu1rVJrocCqWvC3Z9LnEaNJ5JEqSdtqnMZBD_v5zK0zfddhBMoa8Iv-KhkaA", max_new_events: 2,
bearer_token: "eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJtb2JpbGl6b24iLCJleHAiOjE2NDIwODA2NTAsImlhdCI6MTY0MjA3OTc1MCwiaXNzIjoibW9iaWxpem9uIiwianRpIjoiYmZjNWFkNjMtODA2NS00Y2FhLWJiY2EtYzA2MDZkNDEwNDdlIiwibmJmIjoxNjQyMDc5NzQ5LCJzdWIiOiJVc2VyOjEiLCJ0eXAiOiJhY2Nlc3MifQ.Ar5oUTs0cFOEkznYmhXgrFkGRQv4yJm34lN_gkNSrCTrvXfRBTID4YzIrbzvKOkYvIZmw0dHb7gu1WO_tATBkQ",
ccpl: "https://www.cc-paysdelimours.fr/agenda" ccpl: "https://www.cc-paysdelimours.fr/agenda"

46
importers/adl.ts
View File

@ -25,37 +25,53 @@ async function runImportEvents() {
const res = await utilsTools.client.query('SELECT * from events'); const res = await utilsTools.client.query('SELECT * from events');
console.log('💾 évènements enregistrés dans mobilizon : ', res.rows.length); console.log('💾 évènements enregistrés dans mobilizon : ', res.rows.length);
res.rows.forEach((bdd_event: any) => { res.rows.forEach((bdd_event: any) => {
utilsTools.localMobilizonEventsByTitle.push(utilsTools.agendadulibre.uniqTitleBDD(bdd_event)); if (utilsTools.localMobilizonEventsByTitle.indexOf(utilsTools.agendadulibre.uniqTitleBDD(bdd_event)) === -1) {
utilsTools.localMobilizonEventsByTitle.push(utilsTools.agendadulibre.uniqTitleBDD(bdd_event));
}
}) })
console.log('utilsTools.localMobilizonEventsByTitle', utilsTools.localMobilizonEventsByTitle); console.log('utilsTools.localMobilizonEventsByTitle', utilsTools.localMobilizonEventsByTitle.length);
// get json file for ADL // get json file for ADL
await fs.readFile(filepath, 'utf8', function (err, data) { await fs.readFile(filepath, 'utf8', function (err, data) {
if (err) { if (err) {
return console.log(err); return console.log(err);
} }
filecontent = JSON.parse(data) filecontent = JSON.parse(data)
// console.log('events in the scrapped json', filecontent.length); filecontent = filecontent.slice(0,parserConfig.max_new_events_in_scrapping)
console.log('events in the scrapped json', filecontent.length);
let ii = 0; let ii = 0;
filecontent.forEach((event: any) => { filecontent.forEach((event: any) => {
ii++ ii++
// compare events with existing events // compare events with existing events
// let date = new Date(event.start_time); let eventAlreadyExists = utilsTools.agendadulibre.doesEventExistsFromJsonScrap(event);
// console.log(ii, date.toLocaleDateString('fr-fr', { weekday:"long", year:"numeric", month:"short", day:"numeric"}) , event.title ); if (!eventAlreadyExists) {
utilsTools.agendadulibre.doesEventExistsFromJsonScrap(event); if (parserConfig.debug) {
console.log('ajouter l event ', utilsTools.agendadulibre.uniqTitle(event));
}
let newQuery = utilsTools.agendadulibre.addQueryFromJsonScrap(event);
utilsTools.newEvents.push({
event,
newQuery
});
}
}) })
// console.log('utilsTools.localMobilizonEventsByTitle', utilsTools.localMobilizonEventsByTitle);
console.log('end looking for events'); console.log('end looking for events');
// import only new events // import only new events
console.log('nouveaux évènements à ajouter: ', utilsTools.agendadulibre.queryToAdd.length); console.log('nouveaux évènements à ajouter: ', utilsTools.newEvents.length);
let limiter = parserConfig.limit_persistence_of_new_events; let limiter = parserConfig.limit_persistence_of_new_events;
let counter = 0; let counter = 0;
utilsTools.agendadulibre.queryToAdd.forEach((graphQLquery: any) => { console.log('utilsTools.localMobilizonEventsByTitle', utilsTools.localMobilizonEventsByTitle);
if (limiter && counter <= parserConfig.max_new_events) { utilsTools.newEvents.forEach((pair: any) => {
let graphQLquery = pair.newQuery;
if (limiter && counter < parserConfig.max_new_events) {
counter++; counter++;
console.log(counter, ' * ', graphQLquery.variables.beginsOn, graphQLquery.variables.title); console.log(counter, ' * ', utilsTools.agendadulibre.uniqTitle(pair.event));
const body = { const body = {
operationName: "createEvent", operationName: "createEvent",
@ -73,7 +89,12 @@ async function runImportEvents() {
"authorization": "Bearer " + parserConfig.bearer_token, "authorization": "Bearer " + parserConfig.bearer_token,
} }
} }
fetchEvent(url, options) if(!utilsTools.agendadulibre.doesEventExistsFromJsonScrap(pair.event)){
console.log('ajouter');
fetchEvent(url, options)
}else{
console.log('nope');
}
} else { } else {
console.log('---- le fetch est désactivé'); console.log('---- le fetch est désactivé');
} }
@ -107,7 +128,6 @@ const fetchEvent = (theUrl, theOptions) => {
// fs.stat(filepath, function (err, stat) { // fs.stat(filepath, function (err, stat) {
// if (err == null) { // if (err == null) {
runImportEvents(); runImportEvents();
console.log('fetch', fetch);
// } else if (err.code === 'ENOENT') { // } else if (err.code === 'ENOENT') {
// // file does not exist // // file does not exist
// console.log('Scrapped json file does not exist. Run a scraper like "ts-node scrapers/adl.ts" before using this importer: ', err.code); // console.log('Scrapped json file does not exist. Run a scraper like "ts-node scrapers/adl.ts" before using this importer: ', err.code);

14
models/Event.js
View File

@ -1,14 +0,0 @@
const mongoose = require('mongoose');
let EventMobilizonSchema = new mongoose.Schema({
title: String,
date_start: Date,
date_end: Date,
description: Date,
url: Date,
dateCrawled: Date
});
let EventMobilizon = mongoose.model('EventMobilizon', userSchema);
module.exports = EventMobilizon;

114
utils.ts
View File

@ -4,6 +4,7 @@ import {v4 as uuidv4} from "uuid";
import {Client} from "pg"; import {Client} from "pg";
import {htmlEscape} from "escape-goat"; import {htmlEscape} from "escape-goat";
import fetch from 'node-fetch'; import fetch from 'node-fetch';
const moment = require("moment"); const moment = require("moment");
const fs = require("fs"); const fs = require("fs");
@ -12,7 +13,7 @@ let organizerActorId = 3;
let createEventQueryMobilizon = "mutation createEvent($organizerActorId: ID!, $attributedToId: ID, $title: String!, $description: String!, $beginsOn: DateTime!, $endsOn: DateTime, $status: EventStatus, $visibility: EventVisibility, $joinOptions: EventJoinOptions, $draft: Boolean, $tags: [String], $picture: MediaInput, $onlineAddress: String, $phoneAddress: String, $category: String, $physicalAddress: AddressInput, $options: EventOptionsInput, $contacts: [Contact]) {\n createEvent(\n organizerActorId: $organizerActorId\n attributedToId: $attributedToId\n title: $title\n description: $description\n beginsOn: $beginsOn\n endsOn: $endsOn\n status: $status\n visibility: $visibility\n joinOptions: $joinOptions\n draft: $draft\n tags: $tags\n picture: $picture\n onlineAddress: $onlineAddress\n phoneAddress: $phoneAddress\n category: $category\n physicalAddress: $physicalAddress\n options: $options\n contacts: $contacts\n ) {\n ...FullEvent\n __typename\n }\n}\n\nfragment FullEvent on Event {\n id\n uuid\n url\n local\n title\n description\n beginsOn\n endsOn\n status\n visibility\n joinOptions\n draft\n picture {\n id\n url\n name\n metadata {\n width\n height\n blurhash\n __typename\n }\n __typename\n }\n publishAt\n onlineAddress\n phoneAddress\n physicalAddress {\n ...AdressFragment\n __typename\n }\n organizerActor {\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n domain\n name\n url\n id\n summary\n __typename\n }\n contacts {\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n name\n summary\n domain\n url\n id\n __typename\n }\n attributedTo {\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n name\n summary\n domain\n url\n id\n __typename\n }\n participantStats {\n going\n notApproved\n participant\n __typename\n }\n tags {\n ...TagFragment\n __typename\n }\n relatedEvents {\n id\n uuid\n title\n beginsOn\n picture {\n id\n url\n name\n metadata {\n width\n height\n blurhash\n __typename\n }\n __typename\n }\n physicalAddress {\n id\n description\n __typename\n }\n organizerActor {\n id\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n domain\n name\n __typename\n }\n __typename\n }\n options {\n ...EventOptions\n __typename\n }\n metadata {\n key\n title\n value\n type\n __typename\n }\n __typename\n}\n\nfragment AdressFragment on Address {\n id\n description\n geom\n street\n locality\n postalCode\n region\n country\n type\n url\n originId\n __typename\n}\n\nfragment TagFragment on Tag {\n id\n slug\n title\n __typename\n}\n\nfragment EventOptions on EventOptions {\n maximumAttendeeCapacity\n remainingAttendeeCapacity\n showRemainingAttendeeCapacity\n anonymousParticipation\n showStartTime\n showEndTime\n offers {\n price\n priceCurrency\n url\n __typename\n }\n participationConditions {\n title\n content\n url\n __typename\n }\n attendees\n program\n commentModeration\n showParticipationPrice\n hideOrganizerWhenGroupEvent\n __typename\n}\n"; let createEventQueryMobilizon = "mutation createEvent($organizerActorId: ID!, $attributedToId: ID, $title: String!, $description: String!, $beginsOn: DateTime!, $endsOn: DateTime, $status: EventStatus, $visibility: EventVisibility, $joinOptions: EventJoinOptions, $draft: Boolean, $tags: [String], $picture: MediaInput, $onlineAddress: String, $phoneAddress: String, $category: String, $physicalAddress: AddressInput, $options: EventOptionsInput, $contacts: [Contact]) {\n createEvent(\n organizerActorId: $organizerActorId\n attributedToId: $attributedToId\n title: $title\n description: $description\n beginsOn: $beginsOn\n endsOn: $endsOn\n status: $status\n visibility: $visibility\n joinOptions: $joinOptions\n draft: $draft\n tags: $tags\n picture: $picture\n onlineAddress: $onlineAddress\n phoneAddress: $phoneAddress\n category: $category\n physicalAddress: $physicalAddress\n options: $options\n contacts: $contacts\n ) {\n ...FullEvent\n __typename\n }\n}\n\nfragment FullEvent on Event {\n id\n uuid\n url\n local\n title\n description\n beginsOn\n endsOn\n status\n visibility\n joinOptions\n draft\n picture {\n id\n url\n name\n metadata {\n width\n height\n blurhash\n __typename\n }\n __typename\n }\n publishAt\n onlineAddress\n phoneAddress\n physicalAddress {\n ...AdressFragment\n __typename\n }\n organizerActor {\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n domain\n name\n url\n id\n summary\n __typename\n }\n contacts {\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n name\n summary\n domain\n url\n id\n __typename\n }\n attributedTo {\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n name\n summary\n domain\n url\n id\n __typename\n }\n participantStats {\n going\n notApproved\n participant\n __typename\n }\n tags {\n ...TagFragment\n __typename\n }\n relatedEvents {\n id\n uuid\n title\n beginsOn\n picture {\n id\n url\n name\n metadata {\n width\n height\n blurhash\n __typename\n }\n __typename\n }\n physicalAddress {\n id\n description\n __typename\n }\n organizerActor {\n id\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n domain\n name\n __typename\n }\n __typename\n }\n options {\n ...EventOptions\n __typename\n }\n metadata {\n key\n title\n value\n type\n __typename\n }\n __typename\n}\n\nfragment AdressFragment on Address {\n id\n description\n geom\n street\n locality\n postalCode\n region\n country\n type\n url\n originId\n __typename\n}\n\nfragment TagFragment on Tag {\n id\n slug\n title\n __typename\n}\n\nfragment EventOptions on EventOptions {\n maximumAttendeeCapacity\n remainingAttendeeCapacity\n showRemainingAttendeeCapacity\n anonymousParticipation\n showStartTime\n showEndTime\n offers {\n price\n priceCurrency\n url\n __typename\n }\n participationConditions {\n title\n content\n url\n __typename\n }\n attendees\n program\n commentModeration\n showParticipationPrice\n hideOrganizerWhenGroupEvent\n __typename\n}\n";
function proxy_fetch(theUrl: string, theOptions: any) { function proxy_fetch(theUrl: string, theOptions: any) {
return fetch(theUrl,theOptions); return fetch(theUrl, theOptions);
} }
/** /**
@ -51,7 +52,7 @@ class utils {
*/ */
counterOfEventsToAdd = 0; counterOfEventsToAdd = 0;
localMobilizonEventsByTitle: Array<string> = []; localMobilizonEventsByTitle: Array<string> = [];
newEvents: Array<any> = [];
/** /**
* converters * converters
@ -99,9 +100,6 @@ class utils {
</div> ` </div> `
} }
testdateconvert() {
let converted = this.convertRssDate("2021-03-12T19:00:00Z");
}
/** /**
* file management * file management
@ -199,12 +197,7 @@ class utils {
if (this.localMobilizonEventsByTitle.length) { if (this.localMobilizonEventsByTitle.length) {
const eventAlreadyExists = const eventAlreadyExists =
-1 !== this.localMobilizonEventsByTitle.indexOf(this.agendadulibre.uniqTitle(event)); -1 !== this.localMobilizonEventsByTitle.indexOf(this.agendadulibre.uniqTitle(event));
if (!eventAlreadyExists) {
if (parserConfig.debug) {
console.log('ajouter l event ', htmlEscape(this.agendadulibre.uniqTitle(event)));
}
this.agendadulibre.addQuery(event);
}
return eventAlreadyExists; return eventAlreadyExists;
} else { } else {
console.log('aucun évènement dans localMobilizonEventsByTitle'); console.log('aucun évènement dans localMobilizonEventsByTitle');
@ -216,68 +209,65 @@ class utils {
* @param event * @param event
*/ */
uniqTitle(event: any): string { uniqTitle(event: any): string {
return event.start_date + ' ' + event.title
return moment(new Date(event.start_time))
.format("YYYY-MM-DD") + ' ' + event.title
}, },
/** /**
* convert events from bdd to a string used as a comparison with new events * convert events from bdd to a string used as a comparison with new events
* @param event * @param event
*/ */
uniqTitleBDD(event: any): string { uniqTitleBDD(event: any): string {
return event.begins_on + ' ' + event.title return moment(event.begins_on)
.format("YYYY-MM-DD") + ' ' + event.title
}, },
doesEventExistsFromJsonScrap: (event: any): boolean => { doesEventExistsFromJsonScrap: (event: any): boolean => {
console.log('this.agendadulibre.uniqTitle(event)', this.agendadulibre.uniqTitle(event));
const eventAlreadyExists = const eventAlreadyExists =
-1 !== this.localMobilizonEventsByTitle.indexOf(htmlEscape(event.start_date + ' ' + event.title)); -1 !== this.localMobilizonEventsByTitle.indexOf(this.agendadulibre.uniqTitle(event));
if (!eventAlreadyExists) { console.log('eventAlreadyExists', eventAlreadyExists);
if (parserConfig.debug) {
console.log('ajouter l event ', htmlEscape(event.start_date + ' ' + event.title));
}
this.agendadulibre.addQueryFromJsonScrap(event);
}
return eventAlreadyExists; return eventAlreadyExists;
}, },
addQueryFromJsonScrap: (event: any) => { addQueryFromJsonScrap: (event: any) => {
this.agendadulibre.queryToAdd.push( let newQuery = {
{ operationName: "createEvent",
operationName: "createEvent", query: createEventQueryMobilizon,
query: createEventQueryMobilizon, variables: {
variables: { attributedToId: null,
attributedToId: null, beginsOn: event.start_time,
beginsOn: event.start_time, contacts: [],
contacts: [], description: "<p>" + event.description + "</p>",
description: "<p>" + event.description + "</p>", draft: false,
draft: false, endsOn: event.end_time,
endsOn: event.end_time, joinOptions: "FREE",
joinOptions: "FREE", onlineAddress: event.url,
onlineAddress: event.url, options: {
options: { anonymousParticipation: true,
anonymousParticipation: true, attendees: [],
attendees: [], commentModeration: "ALLOW_ALL",
commentModeration: "ALLOW_ALL", hideOrganizerWhenGroupEvent: false,
hideOrganizerWhenGroupEvent: false, maximumAttendeeCapacity: 200,
maximumAttendeeCapacity: 200, offers: [],
offers: [], participationConditions: [],
participationConditions: [], program: "",
program: "", remainingAttendeeCapacity: 0,
remainingAttendeeCapacity: 0, showEndTime: true,
showEndTime: true, showParticipationPrice: false,
showParticipationPrice: false, showRemainingAttendeeCapacity: false,
showRemainingAttendeeCapacity: false, showStartTime: true
showStartTime: true },
}, organizerActorId: organizerActorId,
organizerActorId: organizerActorId, phoneAddress: "",
phoneAddress: "", status: "CONFIRMED",
status: "CONFIRMED", tags: [
tags: [ "imported"
"imported" ],
], title: event.title,
title: event.title, visibility: "PUBLIC"
visibility: "PUBLIC"
}
} }
); }
return this.agendadulibre.queryToAdd; return newQuery;
}, },
addQuery: (event: any) => { addQuery: (event: any) => {
console.log('event', event.title); console.log('event', event.title);
@ -325,7 +315,7 @@ class utils {
return this.agendadulibre.queryToAdd; return this.agendadulibre.queryToAdd;
}, },
addNewEventsFromQueries:()=>{ addNewEventsFromQueries: () => {
if (parserConfig.runAddQueriesToMobilizonAPI) { if (parserConfig.runAddQueriesToMobilizonAPI) {
console.log( console.log(
@ -349,8 +339,8 @@ class utils {
} }
if (parserConfig.enableFetch) { if (parserConfig.enableFetch) {
let theUrl :string = parserConfig.dev_mode ? parserConfig.dev_url : parserConfig.mobilizon_public_url; let theUrl: string = parserConfig.dev_mode ? parserConfig.dev_url : parserConfig.mobilizon_public_url;
let theOptions:any = { let theOptions: any = {
method: "post", method: "post",
body: JSON.stringify(body), body: JSON.stringify(body),
headers: { headers: {