tykayn
/
rss-feeder-mobilizon
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

compare events date

master
Tykayn 7 months ago committed by tykayn
parent
7228429013
commit
bea58afa1e
4 changed files with 89 additions and 91 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 6
      config.ts
  2. 46
      importers/adl.ts
  3. 14
      models/Event.js
  4. 114
      utils.ts

6
config.ts
Unescape View File

@ -20,10 +20,12 @@ const parserConfig = {
organizerActorId: 3, // ID of the user who imports events in the mobilizon instance. 3 is the first admin user created, usually.
runAddQueriesToMobilizonAPI: true,
enableFetch: true, // enable persisting of new events via API
// enableFetch: false,
dev_mode: true, // dev mode uses localhost instance of mobilizon running on port 4000
limit_persistence_of_new_events:true,
max_new_events: 1,
bearer_token: "eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJtb2JpbGl6b24iLCJleHAiOjE2NDIwNzIyMTQsImlhdCI6MTY0MjA3MTMxNCwiaXNzIjoibW9iaWxpem9uIiwianRpIjoiMjBiNGMyNGEtODViYi00MWFlLTk2MjYtMjZhZjFlYThlMTliIiwibmJmIjoxNjQyMDcxMzEzLCJzdWIiOiJVc2VyOjEiLCJ0eXAiOiJhY2Nlc3MifQ.N-QBR0lfj8KX2icUjOVImahIKvZu1rVJrocCqWvC3Z9LnEaNJ5JEqSdtqnMZBD_v5zK0zfddhBMoa8Iv-KhkaA",
max_new_events_in_scrapping: 10,
max_new_events: 2,
bearer_token: "eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJtb2JpbGl6b24iLCJleHAiOjE2NDIwODA2NTAsImlhdCI6MTY0MjA3OTc1MCwiaXNzIjoibW9iaWxpem9uIiwianRpIjoiYmZjNWFkNjMtODA2NS00Y2FhLWJiY2EtYzA2MDZkNDEwNDdlIiwibmJmIjoxNjQyMDc5NzQ5LCJzdWIiOiJVc2VyOjEiLCJ0eXAiOiJhY2Nlc3MifQ.Ar5oUTs0cFOEkznYmhXgrFkGRQv4yJm34lN_gkNSrCTrvXfRBTID4YzIrbzvKOkYvIZmw0dHb7gu1WO_tATBkQ",
ccpl: "https://www.cc-paysdelimours.fr/agenda"

46
importers/adl.ts
Unescape View File

@ -25,37 +25,53 @@ async function runImportEvents() {
const res = await utilsTools.client.query('SELECT * from events');
console.log('💾 évènements enregistrés dans mobilizon : ', res.rows.length);
res.rows.forEach((bdd_event: any) => {
utilsTools.localMobilizonEventsByTitle.push(utilsTools.agendadulibre.uniqTitleBDD(bdd_event));
if (utilsTools.localMobilizonEventsByTitle.indexOf(utilsTools.agendadulibre.uniqTitleBDD(bdd_event)) === -1) {
utilsTools.localMobilizonEventsByTitle.push(utilsTools.agendadulibre.uniqTitleBDD(bdd_event));
}
})
console.log('utilsTools.localMobilizonEventsByTitle', utilsTools.localMobilizonEventsByTitle);
console.log('utilsTools.localMobilizonEventsByTitle', utilsTools.localMobilizonEventsByTitle.length);
// get json file for ADL
await fs.readFile(filepath, 'utf8', function (err, data) {
if (err) {
return console.log(err);
}
filecontent = JSON.parse(data)
// console.log('events in the scrapped json', filecontent.length);
filecontent = filecontent.slice(0,parserConfig.max_new_events_in_scrapping)
console.log('events in the scrapped json', filecontent.length);
let ii = 0;
filecontent.forEach((event: any) => {
ii++
// compare events with existing events
// let date = new Date(event.start_time);
// console.log(ii, date.toLocaleDateString('fr-fr', { weekday:"long", year:"numeric", month:"short", day:"numeric"}) , event.title );
utilsTools.agendadulibre.doesEventExistsFromJsonScrap(event);
let eventAlreadyExists = utilsTools.agendadulibre.doesEventExistsFromJsonScrap(event);
if (!eventAlreadyExists) {
if (parserConfig.debug) {
console.log('ajouter l event ', utilsTools.agendadulibre.uniqTitle(event));
}
let newQuery = utilsTools.agendadulibre.addQueryFromJsonScrap(event);
utilsTools.newEvents.push({
event,
newQuery
});
}
})
// console.log('utilsTools.localMobilizonEventsByTitle', utilsTools.localMobilizonEventsByTitle);
console.log('end looking for events');
// import only new events
console.log('nouveaux évènements à ajouter: ', utilsTools.agendadulibre.queryToAdd.length);
console.log('nouveaux évènements à ajouter: ', utilsTools.newEvents.length);
let limiter = parserConfig.limit_persistence_of_new_events;
let counter = 0;
utilsTools.agendadulibre.queryToAdd.forEach((graphQLquery: any) => {
if (limiter && counter <= parserConfig.max_new_events) {
console.log('utilsTools.localMobilizonEventsByTitle', utilsTools.localMobilizonEventsByTitle);
utilsTools.newEvents.forEach((pair: any) => {
let graphQLquery = pair.newQuery;
if (limiter && counter < parserConfig.max_new_events) {
counter++;
console.log(counter, ' * ', graphQLquery.variables.beginsOn, graphQLquery.variables.title);
console.log(counter, ' * ', utilsTools.agendadulibre.uniqTitle(pair.event));
const body = {
operationName: "createEvent",
@ -73,7 +89,12 @@ async function runImportEvents() {
"authorization": "Bearer " + parserConfig.bearer_token,
}
}
fetchEvent(url, options)
if(!utilsTools.agendadulibre.doesEventExistsFromJsonScrap(pair.event)){
console.log('ajouter');
fetchEvent(url, options)
}else{
console.log('nope');
}
} else {
console.log('---- le fetch est désactivé');
}
@ -107,7 +128,6 @@ const fetchEvent = (theUrl, theOptions) => {
// fs.stat(filepath, function (err, stat) {
// if (err == null) {
runImportEvents();
console.log('fetch', fetch);
// } else if (err.code === 'ENOENT') {
// // file does not exist
// console.log('Scrapped json file does not exist. Run a scraper like "ts-node scrapers/adl.ts" before using this importer: ', err.code);

14
models/Event.js
Unescape View File

@ -1,14 +0,0 @@
const mongoose = require('mongoose');
let EventMobilizonSchema = new mongoose.Schema({
title: String,
date_start: Date,
date_end: Date,
description: Date,
url: Date,
dateCrawled: Date
});
let EventMobilizon = mongoose.model('EventMobilizon', userSchema);
module.exports = EventMobilizon;

114
utils.ts
Unescape View File

@ -4,6 +4,7 @@ import {v4 as uuidv4} from "uuid";
import {Client} from "pg";
import {htmlEscape} from "escape-goat";
import fetch from 'node-fetch';
const moment = require("moment");
const fs = require("fs");
@ -12,7 +13,7 @@ let organizerActorId = 3;
let createEventQueryMobilizon = "mutation createEvent($organizerActorId: ID!, $attributedToId: ID, $title: String!, $description: String!, $beginsOn: DateTime!, $endsOn: DateTime, $status: EventStatus, $visibility: EventVisibility, $joinOptions: EventJoinOptions, $draft: Boolean, $tags: [String], $picture: MediaInput, $onlineAddress: String, $phoneAddress: String, $category: String, $physicalAddress: AddressInput, $options: EventOptionsInput, $contacts: [Contact]) {\n createEvent(\n organizerActorId: $organizerActorId\n attributedToId: $attributedToId\n title: $title\n description: $description\n beginsOn: $beginsOn\n endsOn: $endsOn\n status: $status\n visibility: $visibility\n joinOptions: $joinOptions\n draft: $draft\n tags: $tags\n picture: $picture\n onlineAddress: $onlineAddress\n phoneAddress: $phoneAddress\n category: $category\n physicalAddress: $physicalAddress\n options: $options\n contacts: $contacts\n ) {\n ...FullEvent\n __typename\n }\n}\n\nfragment FullEvent on Event {\n id\n uuid\n url\n local\n title\n description\n beginsOn\n endsOn\n status\n visibility\n joinOptions\n draft\n picture {\n id\n url\n name\n metadata {\n width\n height\n blurhash\n __typename\n }\n __typename\n }\n publishAt\n onlineAddress\n phoneAddress\n physicalAddress {\n ...AdressFragment\n __typename\n }\n organizerActor {\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n domain\n name\n url\n id\n summary\n __typename\n }\n contacts {\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n name\n summary\n domain\n url\n id\n __typename\n }\n attributedTo {\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n name\n summary\n domain\n url\n id\n __typename\n }\n participantStats {\n going\n notApproved\n participant\n __typename\n }\n tags {\n ...TagFragment\n __typename\n }\n relatedEvents {\n id\n uuid\n title\n beginsOn\n picture {\n id\n url\n name\n metadata {\n width\n height\n blurhash\n __typename\n }\n __typename\n }\n physicalAddress {\n id\n description\n __typename\n }\n organizerActor {\n id\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n domain\n name\n __typename\n }\n __typename\n }\n options {\n ...EventOptions\n __typename\n }\n metadata {\n key\n title\n value\n type\n __typename\n }\n __typename\n}\n\nfragment AdressFragment on Address {\n id\n description\n geom\n street\n locality\n postalCode\n region\n country\n type\n url\n originId\n __typename\n}\n\nfragment TagFragment on Tag {\n id\n slug\n title\n __typename\n}\n\nfragment EventOptions on EventOptions {\n maximumAttendeeCapacity\n remainingAttendeeCapacity\n showRemainingAttendeeCapacity\n anonymousParticipation\n showStartTime\n showEndTime\n offers {\n price\n priceCurrency\n url\n __typename\n }\n participationConditions {\n title\n content\n url\n __typename\n }\n attendees\n program\n commentModeration\n showParticipationPrice\n hideOrganizerWhenGroupEvent\n __typename\n}\n";
function proxy_fetch(theUrl: string, theOptions: any) {
return fetch(theUrl,theOptions);
return fetch(theUrl, theOptions);
}
/**
@ -51,7 +52,7 @@ class utils {
*/
counterOfEventsToAdd = 0;
localMobilizonEventsByTitle: Array<string> = [];
newEvents: Array<any> = [];
/**
* converters
@ -99,9 +100,6 @@ class utils {
</div> `
}
testdateconvert() {
let converted = this.convertRssDate("2021-03-12T19:00:00Z");
}
/**
* file management
@ -199,12 +197,7 @@ class utils {
if (this.localMobilizonEventsByTitle.length) {
const eventAlreadyExists =
-1 !== this.localMobilizonEventsByTitle.indexOf(this.agendadulibre.uniqTitle(event));
if (!eventAlreadyExists) {
if (parserConfig.debug) {
console.log('ajouter l event ', htmlEscape(this.agendadulibre.uniqTitle(event)));
}
this.agendadulibre.addQuery(event);
}
return eventAlreadyExists;
} else {
console.log('aucun évènement dans localMobilizonEventsByTitle');
@ -216,68 +209,65 @@ class utils {
* @param event
*/
uniqTitle(event: any): string {
return event.start_date + ' ' + event.title
return moment(new Date(event.start_time))
.format("YYYY-MM-DD") + ' ' + event.title
},
/**
* convert events from bdd to a string used as a comparison with new events
* @param event
*/
uniqTitleBDD(event: any): string {
return event.begins_on + ' ' + event.title
return moment(event.begins_on)
.format("YYYY-MM-DD") + ' ' + event.title
},
doesEventExistsFromJsonScrap: (event: any): boolean => {
console.log('this.agendadulibre.uniqTitle(event)', this.agendadulibre.uniqTitle(event));
const eventAlreadyExists =
-1 !== this.localMobilizonEventsByTitle.indexOf(htmlEscape(event.start_date + ' ' + event.title));
if (!eventAlreadyExists) {
if (parserConfig.debug) {
console.log('ajouter l event ', htmlEscape(event.start_date + ' ' + event.title));
}
this.agendadulibre.addQueryFromJsonScrap(event);
}
-1 !== this.localMobilizonEventsByTitle.indexOf(this.agendadulibre.uniqTitle(event));
console.log('eventAlreadyExists', eventAlreadyExists);
return eventAlreadyExists;
},
addQueryFromJsonScrap: (event: any) => {
this.agendadulibre.queryToAdd.push(
{
operationName: "createEvent",
query: createEventQueryMobilizon,
variables: {
attributedToId: null,
beginsOn: event.start_time,
contacts: [],
description: "<p>" + event.description + "</p>",
draft: false,
endsOn: event.end_time,
joinOptions: "FREE",
onlineAddress: event.url,
options: {
anonymousParticipation: true,
attendees: [],
commentModeration: "ALLOW_ALL",
hideOrganizerWhenGroupEvent: false,
maximumAttendeeCapacity: 200,
offers: [],
participationConditions: [],
program: "",
remainingAttendeeCapacity: 0,
showEndTime: true,
showParticipationPrice: false,
showRemainingAttendeeCapacity: false,
showStartTime: true
},
organizerActorId: organizerActorId,
phoneAddress: "",
status: "CONFIRMED",
tags: [
"imported"
],
title: event.title,
visibility: "PUBLIC"
}
let newQuery = {
operationName: "createEvent",
query: createEventQueryMobilizon,
variables: {
attributedToId: null,
beginsOn: event.start_time,
contacts: [],
description: "<p>" + event.description + "</p>",
draft: false,
endsOn: event.end_time,
joinOptions: "FREE",
onlineAddress: event.url,
options: {
anonymousParticipation: true,
attendees: [],
commentModeration: "ALLOW_ALL",
hideOrganizerWhenGroupEvent: false,
maximumAttendeeCapacity: 200,
offers: [],
participationConditions: [],
program: "",
remainingAttendeeCapacity: 0,
showEndTime: true,
showParticipationPrice: false,
showRemainingAttendeeCapacity: false,
showStartTime: true
},
organizerActorId: organizerActorId,
phoneAddress: "",
status: "CONFIRMED",
tags: [
"imported"
],
title: event.title,
visibility: "PUBLIC"
}
);
}
return this.agendadulibre.queryToAdd;
return newQuery;
},
addQuery: (event: any) => {
console.log('event', event.title);
@ -325,7 +315,7 @@ class utils {
return this.agendadulibre.queryToAdd;
},
addNewEventsFromQueries:()=>{
addNewEventsFromQueries: () => {
if (parserConfig.runAddQueriesToMobilizonAPI) {
console.log(
@ -349,8 +339,8 @@ class utils {
}
if (parserConfig.enableFetch) {
let theUrl :string = parserConfig.dev_mode ? parserConfig.dev_url : parserConfig.mobilizon_public_url;
let theOptions:any = {
let theUrl: string = parserConfig.dev_mode ? parserConfig.dev_url : parserConfig.mobilizon_public_url;
let theOptions: any = {
method: "post",
body: JSON.stringify(body),
headers: {

Write Preview
Loading…
Cancel
Save