From bea58afa1e22d47e4c2ed8d64b3b0c63fe20cb36 Mon Sep 17 00:00:00 2001 From: Tykayn Date: Thu, 13 Jan 2022 14:30:45 +0100 Subject: [PATCH] compare events date --- config.ts | 6 ++- importers/adl.ts | 46 +++++++++++++------ models/Event.js | 14 ------ utils.ts | 114 +++++++++++++++++++++-------------------------- 4 files changed, 89 insertions(+), 91 deletions(-) delete mode 100644 models/Event.js diff --git a/config.ts b/config.ts index ad74bd8..9684457 100644 --- a/config.ts +++ b/config.ts @@ -20,10 +20,12 @@ const parserConfig = { organizerActorId: 3, // ID of the user who imports events in the mobilizon instance. 3 is the first admin user created, usually. runAddQueriesToMobilizonAPI: true, enableFetch: true, // enable persisting of new events via API + // enableFetch: false, dev_mode: true, // dev mode uses localhost instance of mobilizon running on port 4000 limit_persistence_of_new_events:true, - max_new_events: 1, - bearer_token: "eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJtb2JpbGl6b24iLCJleHAiOjE2NDIwNzIyMTQsImlhdCI6MTY0MjA3MTMxNCwiaXNzIjoibW9iaWxpem9uIiwianRpIjoiMjBiNGMyNGEtODViYi00MWFlLTk2MjYtMjZhZjFlYThlMTliIiwibmJmIjoxNjQyMDcxMzEzLCJzdWIiOiJVc2VyOjEiLCJ0eXAiOiJhY2Nlc3MifQ.N-QBR0lfj8KX2icUjOVImahIKvZu1rVJrocCqWvC3Z9LnEaNJ5JEqSdtqnMZBD_v5zK0zfddhBMoa8Iv-KhkaA", + max_new_events_in_scrapping: 10, + max_new_events: 2, + bearer_token: "eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJtb2JpbGl6b24iLCJleHAiOjE2NDIwODA2NTAsImlhdCI6MTY0MjA3OTc1MCwiaXNzIjoibW9iaWxpem9uIiwianRpIjoiYmZjNWFkNjMtODA2NS00Y2FhLWJiY2EtYzA2MDZkNDEwNDdlIiwibmJmIjoxNjQyMDc5NzQ5LCJzdWIiOiJVc2VyOjEiLCJ0eXAiOiJhY2Nlc3MifQ.Ar5oUTs0cFOEkznYmhXgrFkGRQv4yJm34lN_gkNSrCTrvXfRBTID4YzIrbzvKOkYvIZmw0dHb7gu1WO_tATBkQ", ccpl: "https://www.cc-paysdelimours.fr/agenda" diff --git a/importers/adl.ts b/importers/adl.ts index 1133f2f..9d54bd0 100644 --- a/importers/adl.ts +++ b/importers/adl.ts @@ -25,37 +25,53 @@ async function runImportEvents() { const res = await utilsTools.client.query('SELECT * from events'); console.log('💾 évènements enregistrés dans mobilizon : ', res.rows.length); res.rows.forEach((bdd_event: any) => { - utilsTools.localMobilizonEventsByTitle.push(utilsTools.agendadulibre.uniqTitleBDD(bdd_event)); + if (utilsTools.localMobilizonEventsByTitle.indexOf(utilsTools.agendadulibre.uniqTitleBDD(bdd_event)) === -1) { + + utilsTools.localMobilizonEventsByTitle.push(utilsTools.agendadulibre.uniqTitleBDD(bdd_event)); + } }) - console.log('utilsTools.localMobilizonEventsByTitle', utilsTools.localMobilizonEventsByTitle); + console.log('utilsTools.localMobilizonEventsByTitle', utilsTools.localMobilizonEventsByTitle.length); // get json file for ADL await fs.readFile(filepath, 'utf8', function (err, data) { if (err) { return console.log(err); } filecontent = JSON.parse(data) - // console.log('events in the scrapped json', filecontent.length); + filecontent = filecontent.slice(0,parserConfig.max_new_events_in_scrapping) + console.log('events in the scrapped json', filecontent.length); + let ii = 0; filecontent.forEach((event: any) => { ii++ // compare events with existing events - // let date = new Date(event.start_time); - // console.log(ii, date.toLocaleDateString('fr-fr', { weekday:"long", year:"numeric", month:"short", day:"numeric"}) , event.title ); - utilsTools.agendadulibre.doesEventExistsFromJsonScrap(event); - + let eventAlreadyExists = utilsTools.agendadulibre.doesEventExistsFromJsonScrap(event); + if (!eventAlreadyExists) { + if (parserConfig.debug) { + console.log('ajouter l event ', utilsTools.agendadulibre.uniqTitle(event)); + } + let newQuery = utilsTools.agendadulibre.addQueryFromJsonScrap(event); + utilsTools.newEvents.push({ + event, + newQuery + }); + } }) + // console.log('utilsTools.localMobilizonEventsByTitle', utilsTools.localMobilizonEventsByTitle); console.log('end looking for events'); // import only new events - console.log('nouveaux évènements à ajouter: ', utilsTools.agendadulibre.queryToAdd.length); + console.log('nouveaux évènements à ajouter: ', utilsTools.newEvents.length); let limiter = parserConfig.limit_persistence_of_new_events; let counter = 0; - utilsTools.agendadulibre.queryToAdd.forEach((graphQLquery: any) => { - if (limiter && counter <= parserConfig.max_new_events) { + console.log('utilsTools.localMobilizonEventsByTitle', utilsTools.localMobilizonEventsByTitle); + utilsTools.newEvents.forEach((pair: any) => { + let graphQLquery = pair.newQuery; + + if (limiter && counter < parserConfig.max_new_events) { counter++; - console.log(counter, ' * ', graphQLquery.variables.beginsOn, graphQLquery.variables.title); + console.log(counter, ' * ', utilsTools.agendadulibre.uniqTitle(pair.event)); const body = { operationName: "createEvent", @@ -73,7 +89,12 @@ async function runImportEvents() { "authorization": "Bearer " + parserConfig.bearer_token, } } - fetchEvent(url, options) + if(!utilsTools.agendadulibre.doesEventExistsFromJsonScrap(pair.event)){ + console.log('ajouter'); + fetchEvent(url, options) + }else{ + console.log('nope'); + } } else { console.log('---- le fetch est désactivé'); } @@ -107,7 +128,6 @@ const fetchEvent = (theUrl, theOptions) => { // fs.stat(filepath, function (err, stat) { // if (err == null) { runImportEvents(); -console.log('fetch', fetch); // } else if (err.code === 'ENOENT') { // // file does not exist // console.log('Scrapped json file does not exist. Run a scraper like "ts-node scrapers/adl.ts" before using this importer: ', err.code); diff --git a/models/Event.js b/models/Event.js deleted file mode 100644 index aa747ef..0000000 --- a/models/Event.js +++ /dev/null @@ -1,14 +0,0 @@ -const mongoose = require('mongoose'); - -let EventMobilizonSchema = new mongoose.Schema({ - title: String, - date_start: Date, - date_end: Date, - description: Date, - url: Date, - dateCrawled: Date -}); - -let EventMobilizon = mongoose.model('EventMobilizon', userSchema); - -module.exports = EventMobilizon; \ No newline at end of file diff --git a/utils.ts b/utils.ts index a12a8ab..6902bb4 100644 --- a/utils.ts +++ b/utils.ts @@ -4,6 +4,7 @@ import {v4 as uuidv4} from "uuid"; import {Client} from "pg"; import {htmlEscape} from "escape-goat"; import fetch from 'node-fetch'; + const moment = require("moment"); const fs = require("fs"); @@ -12,7 +13,7 @@ let organizerActorId = 3; let createEventQueryMobilizon = "mutation createEvent($organizerActorId: ID!, $attributedToId: ID, $title: String!, $description: String!, $beginsOn: DateTime!, $endsOn: DateTime, $status: EventStatus, $visibility: EventVisibility, $joinOptions: EventJoinOptions, $draft: Boolean, $tags: [String], $picture: MediaInput, $onlineAddress: String, $phoneAddress: String, $category: String, $physicalAddress: AddressInput, $options: EventOptionsInput, $contacts: [Contact]) {\n createEvent(\n organizerActorId: $organizerActorId\n attributedToId: $attributedToId\n title: $title\n description: $description\n beginsOn: $beginsOn\n endsOn: $endsOn\n status: $status\n visibility: $visibility\n joinOptions: $joinOptions\n draft: $draft\n tags: $tags\n picture: $picture\n onlineAddress: $onlineAddress\n phoneAddress: $phoneAddress\n category: $category\n physicalAddress: $physicalAddress\n options: $options\n contacts: $contacts\n ) {\n ...FullEvent\n __typename\n }\n}\n\nfragment FullEvent on Event {\n id\n uuid\n url\n local\n title\n description\n beginsOn\n endsOn\n status\n visibility\n joinOptions\n draft\n picture {\n id\n url\n name\n metadata {\n width\n height\n blurhash\n __typename\n }\n __typename\n }\n publishAt\n onlineAddress\n phoneAddress\n physicalAddress {\n ...AdressFragment\n __typename\n }\n organizerActor {\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n domain\n name\n url\n id\n summary\n __typename\n }\n contacts {\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n name\n summary\n domain\n url\n id\n __typename\n }\n attributedTo {\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n name\n summary\n domain\n url\n id\n __typename\n }\n participantStats {\n going\n notApproved\n participant\n __typename\n }\n tags {\n ...TagFragment\n __typename\n }\n relatedEvents {\n id\n uuid\n title\n beginsOn\n picture {\n id\n url\n name\n metadata {\n width\n height\n blurhash\n __typename\n }\n __typename\n }\n physicalAddress {\n id\n description\n __typename\n }\n organizerActor {\n id\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n domain\n name\n __typename\n }\n __typename\n }\n options {\n ...EventOptions\n __typename\n }\n metadata {\n key\n title\n value\n type\n __typename\n }\n __typename\n}\n\nfragment AdressFragment on Address {\n id\n description\n geom\n street\n locality\n postalCode\n region\n country\n type\n url\n originId\n __typename\n}\n\nfragment TagFragment on Tag {\n id\n slug\n title\n __typename\n}\n\nfragment EventOptions on EventOptions {\n maximumAttendeeCapacity\n remainingAttendeeCapacity\n showRemainingAttendeeCapacity\n anonymousParticipation\n showStartTime\n showEndTime\n offers {\n price\n priceCurrency\n url\n __typename\n }\n participationConditions {\n title\n content\n url\n __typename\n }\n attendees\n program\n commentModeration\n showParticipationPrice\n hideOrganizerWhenGroupEvent\n __typename\n}\n"; function proxy_fetch(theUrl: string, theOptions: any) { - return fetch(theUrl,theOptions); + return fetch(theUrl, theOptions); } /** @@ -51,7 +52,7 @@ class utils { */ counterOfEventsToAdd = 0; localMobilizonEventsByTitle: Array = []; - + newEvents: Array = []; /** * converters @@ -99,9 +100,6 @@ class utils { ` } - testdateconvert() { - let converted = this.convertRssDate("2021-03-12T19:00:00Z"); - } /** * file management @@ -199,12 +197,7 @@ class utils { if (this.localMobilizonEventsByTitle.length) { const eventAlreadyExists = -1 !== this.localMobilizonEventsByTitle.indexOf(this.agendadulibre.uniqTitle(event)); - if (!eventAlreadyExists) { - if (parserConfig.debug) { - console.log('ajouter l event ', htmlEscape(this.agendadulibre.uniqTitle(event))); - } - this.agendadulibre.addQuery(event); - } + return eventAlreadyExists; } else { console.log('aucun évènement dans localMobilizonEventsByTitle'); @@ -216,68 +209,65 @@ class utils { * @param event */ uniqTitle(event: any): string { - return event.start_date + ' ' + event.title + + return moment(new Date(event.start_time)) + .format("YYYY-MM-DD") + ' ' + event.title }, /** * convert events from bdd to a string used as a comparison with new events * @param event */ uniqTitleBDD(event: any): string { - return event.begins_on + ' ' + event.title + return moment(event.begins_on) + .format("YYYY-MM-DD") + ' ' + event.title }, doesEventExistsFromJsonScrap: (event: any): boolean => { + console.log('this.agendadulibre.uniqTitle(event)', this.agendadulibre.uniqTitle(event)); const eventAlreadyExists = - -1 !== this.localMobilizonEventsByTitle.indexOf(htmlEscape(event.start_date + ' ' + event.title)); - if (!eventAlreadyExists) { - if (parserConfig.debug) { - console.log('ajouter l event ', htmlEscape(event.start_date + ' ' + event.title)); - } - this.agendadulibre.addQueryFromJsonScrap(event); - } + -1 !== this.localMobilizonEventsByTitle.indexOf(this.agendadulibre.uniqTitle(event)); + console.log('eventAlreadyExists', eventAlreadyExists); return eventAlreadyExists; }, addQueryFromJsonScrap: (event: any) => { - this.agendadulibre.queryToAdd.push( - { - operationName: "createEvent", - query: createEventQueryMobilizon, - variables: { - attributedToId: null, - beginsOn: event.start_time, - contacts: [], - description: "

" + event.description + "

", - draft: false, - endsOn: event.end_time, - joinOptions: "FREE", - onlineAddress: event.url, - options: { - anonymousParticipation: true, - attendees: [], - commentModeration: "ALLOW_ALL", - hideOrganizerWhenGroupEvent: false, - maximumAttendeeCapacity: 200, - offers: [], - participationConditions: [], - program: "", - remainingAttendeeCapacity: 0, - showEndTime: true, - showParticipationPrice: false, - showRemainingAttendeeCapacity: false, - showStartTime: true - }, - organizerActorId: organizerActorId, - phoneAddress: "", - status: "CONFIRMED", - tags: [ - "imported" - ], - title: event.title, - visibility: "PUBLIC" - } + let newQuery = { + operationName: "createEvent", + query: createEventQueryMobilizon, + variables: { + attributedToId: null, + beginsOn: event.start_time, + contacts: [], + description: "

" + event.description + "

", + draft: false, + endsOn: event.end_time, + joinOptions: "FREE", + onlineAddress: event.url, + options: { + anonymousParticipation: true, + attendees: [], + commentModeration: "ALLOW_ALL", + hideOrganizerWhenGroupEvent: false, + maximumAttendeeCapacity: 200, + offers: [], + participationConditions: [], + program: "", + remainingAttendeeCapacity: 0, + showEndTime: true, + showParticipationPrice: false, + showRemainingAttendeeCapacity: false, + showStartTime: true + }, + organizerActorId: organizerActorId, + phoneAddress: "", + status: "CONFIRMED", + tags: [ + "imported" + ], + title: event.title, + visibility: "PUBLIC" } - ); + } - return this.agendadulibre.queryToAdd; + return newQuery; }, addQuery: (event: any) => { console.log('event', event.title); @@ -325,7 +315,7 @@ class utils { return this.agendadulibre.queryToAdd; }, - addNewEventsFromQueries:()=>{ + addNewEventsFromQueries: () => { if (parserConfig.runAddQueriesToMobilizonAPI) { console.log( @@ -349,8 +339,8 @@ class utils { } if (parserConfig.enableFetch) { - let theUrl :string = parserConfig.dev_mode ? parserConfig.dev_url : parserConfig.mobilizon_public_url; - let theOptions:any = { + let theUrl: string = parserConfig.dev_mode ? parserConfig.dev_url : parserConfig.mobilizon_public_url; + let theOptions: any = { method: "post", body: JSON.stringify(body), headers: {