compare events date

config.ts

@@ -20,10 +20,12 @@ const parserConfig = {
   organizerActorId: 3, // ID of the user who imports events in the mobilizon instance. 3 is the first admin user created, usually.
   runAddQueriesToMobilizonAPI: true,
   enableFetch: true, // enable persisting of new events via API
+  // enableFetch: false,
   dev_mode: true, // dev mode uses localhost instance of mobilizon running on port 4000
   limit_persistence_of_new_events:true,
-  max_new_events: 1,
+  max_new_events_in_scrapping: 10,
-  bearer_token: "eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJtb2JpbGl6b24iLCJleHAiOjE2NDIwNzIyMTQsImlhdCI6MTY0MjA3MTMxNCwiaXNzIjoibW9iaWxpem9uIiwianRpIjoiMjBiNGMyNGEtODViYi00MWFlLTk2MjYtMjZhZjFlYThlMTliIiwibmJmIjoxNjQyMDcxMzEzLCJzdWIiOiJVc2VyOjEiLCJ0eXAiOiJhY2Nlc3MifQ.N-QBR0lfj8KX2icUjOVImahIKvZu1rVJrocCqWvC3Z9LnEaNJ5JEqSdtqnMZBD_v5zK0zfddhBMoa8Iv-KhkaA",
+  max_new_events: 2,
+  bearer_token: "eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJtb2JpbGl6b24iLCJleHAiOjE2NDIwODA2NTAsImlhdCI6MTY0MjA3OTc1MCwiaXNzIjoibW9iaWxpem9uIiwianRpIjoiYmZjNWFkNjMtODA2NS00Y2FhLWJiY2EtYzA2MDZkNDEwNDdlIiwibmJmIjoxNjQyMDc5NzQ5LCJzdWIiOiJVc2VyOjEiLCJ0eXAiOiJhY2Nlc3MifQ.Ar5oUTs0cFOEkznYmhXgrFkGRQv4yJm34lN_gkNSrCTrvXfRBTID4YzIrbzvKOkYvIZmw0dHb7gu1WO_tATBkQ",
 
   ccpl: "https://www.cc-paysdelimours.fr/agenda"
 

importers/adl.ts

@@ -25,37 +25,53 @@ async function runImportEvents() {
     const res = await utilsTools.client.query('SELECT * from events');
     console.log('💾 évènements enregistrés dans mobilizon : ', res.rows.length);
     res.rows.forEach((bdd_event: any) => {
+        if (utilsTools.localMobilizonEventsByTitle.indexOf(utilsTools.agendadulibre.uniqTitleBDD(bdd_event)) === -1) {
+
             utilsTools.localMobilizonEventsByTitle.push(utilsTools.agendadulibre.uniqTitleBDD(bdd_event));
+        }
     })
 
-    console.log('utilsTools.localMobilizonEventsByTitle', utilsTools.localMobilizonEventsByTitle);
+    console.log('utilsTools.localMobilizonEventsByTitle', utilsTools.localMobilizonEventsByTitle.length);
     // get json file for ADL
     await fs.readFile(filepath, 'utf8', function (err, data) {
         if (err) {
             return console.log(err);
         }
         filecontent = JSON.parse(data)
-        // console.log('events in the scrapped json', filecontent.length);
+        filecontent = filecontent.slice(0,parserConfig.max_new_events_in_scrapping)
+        console.log('events in the scrapped json', filecontent.length);
+
         let ii = 0;
         filecontent.forEach((event: any) => {
             ii++
             // compare events with existing events
-            // let date = new Date(event.start_time);
+            let eventAlreadyExists = utilsTools.agendadulibre.doesEventExistsFromJsonScrap(event);
-            // console.log(ii,  date.toLocaleDateString('fr-fr', { weekday:"long", year:"numeric", month:"short", day:"numeric"}) , event.title );
+            if (!eventAlreadyExists) {
-            utilsTools.agendadulibre.doesEventExistsFromJsonScrap(event);
+                if (parserConfig.debug) {
-
+                    console.log('ajouter l event ', utilsTools.agendadulibre.uniqTitle(event));
+                }
+                let newQuery = utilsTools.agendadulibre.addQueryFromJsonScrap(event);
+                utilsTools.newEvents.push({
+                    event,
+                    newQuery
+                });
+            }
         })
+        // console.log('utilsTools.localMobilizonEventsByTitle', utilsTools.localMobilizonEventsByTitle);
         console.log('end  looking for events');
 
 // import only new events
-        console.log('nouveaux évènements à ajouter: ', utilsTools.agendadulibre.queryToAdd.length);
+        console.log('nouveaux évènements à ajouter: ', utilsTools.newEvents.length);
         let limiter = parserConfig.limit_persistence_of_new_events;
         let counter = 0;
 
-        utilsTools.agendadulibre.queryToAdd.forEach((graphQLquery: any) => {
+        console.log('utilsTools.localMobilizonEventsByTitle', utilsTools.localMobilizonEventsByTitle);
-            if (limiter && counter <= parserConfig.max_new_events) {
+        utilsTools.newEvents.forEach((pair: any) => {
+            let graphQLquery = pair.newQuery;
+
+            if (limiter && counter < parserConfig.max_new_events) {
                 counter++;
-                console.log(counter, ' * ', graphQLquery.variables.beginsOn, graphQLquery.variables.title);
+                console.log(counter, ' * ', utilsTools.agendadulibre.uniqTitle(pair.event));
 
                 const body = {
                     operationName: "createEvent",
@@ -73,7 +89,12 @@ async function runImportEvents() {
                             "authorization": "Bearer " + parserConfig.bearer_token,
                         }
                     }
+                    if(!utilsTools.agendadulibre.doesEventExistsFromJsonScrap(pair.event)){
+                        console.log('ajouter');
                          fetchEvent(url, options)
+                    }else{
+                        console.log('nope');
+                    }
                 } else {
                     console.log('---- le fetch est désactivé');
                 }
@@ -107,7 +128,6 @@ const fetchEvent = (theUrl, theOptions) => {
 // fs.stat(filepath, function (err, stat) {
 //     if (err == null) {
 runImportEvents();
-console.log('fetch', fetch);
 //     } else if (err.code === 'ENOENT') {
 //         // file does not exist
 //         console.log('Scrapped json file does not exist. Run a scraper like "ts-node scrapers/adl.ts" before using this importer: ', err.code);

models/Event.js

@@ -1,14 +0,0 @@
-const mongoose = require('mongoose');
-
-let EventMobilizonSchema = new mongoose.Schema({
-    title: String,
-    date_start: Date,
-    date_end: Date,
-    description: Date,
-    url: Date,
-    dateCrawled: Date
-});
-
-let EventMobilizon = mongoose.model('EventMobilizon', userSchema);
-
-module.exports = EventMobilizon;

utils.ts

@@ -4,6 +4,7 @@ import {v4 as uuidv4} from "uuid";
 import {Client} from "pg";
 import {htmlEscape} from "escape-goat";
 import fetch from 'node-fetch';
+
 const moment = require("moment");
 
 const fs = require("fs");
@@ -12,7 +13,7 @@ let organizerActorId = 3;
 let createEventQueryMobilizon = "mutation createEvent($organizerActorId: ID!, $attributedToId: ID, $title: String!, $description: String!, $beginsOn: DateTime!, $endsOn: DateTime, $status: EventStatus, $visibility: EventVisibility, $joinOptions: EventJoinOptions, $draft: Boolean, $tags: [String], $picture: MediaInput, $onlineAddress: String, $phoneAddress: String, $category: String, $physicalAddress: AddressInput, $options: EventOptionsInput, $contacts: [Contact]) {\n  createEvent(\n    organizerActorId: $organizerActorId\n    attributedToId: $attributedToId\n    title: $title\n    description: $description\n    beginsOn: $beginsOn\n    endsOn: $endsOn\n    status: $status\n    visibility: $visibility\n    joinOptions: $joinOptions\n    draft: $draft\n    tags: $tags\n    picture: $picture\n    onlineAddress: $onlineAddress\n    phoneAddress: $phoneAddress\n    category: $category\n    physicalAddress: $physicalAddress\n    options: $options\n    contacts: $contacts\n  ) {\n    ...FullEvent\n    __typename\n  }\n}\n\nfragment FullEvent on Event {\n  id\n  uuid\n  url\n  local\n  title\n  description\n  beginsOn\n  endsOn\n  status\n  visibility\n  joinOptions\n  draft\n  picture {\n    id\n    url\n    name\n    metadata {\n      width\n      height\n      blurhash\n      __typename\n    }\n    __typename\n  }\n  publishAt\n  onlineAddress\n  phoneAddress\n  physicalAddress {\n    ...AdressFragment\n    __typename\n  }\n  organizerActor {\n    avatar {\n      id\n      url\n      __typename\n    }\n    preferredUsername\n    domain\n    name\n    url\n    id\n    summary\n    __typename\n  }\n  contacts {\n    avatar {\n      id\n      url\n      __typename\n    }\n    preferredUsername\n    name\n    summary\n    domain\n    url\n    id\n    __typename\n  }\n  attributedTo {\n    avatar {\n      id\n      url\n      __typename\n    }\n    preferredUsername\n    name\n    summary\n    domain\n    url\n    id\n    __typename\n  }\n  participantStats {\n    going\n    notApproved\n    participant\n    __typename\n  }\n  tags {\n    ...TagFragment\n    __typename\n  }\n  relatedEvents {\n    id\n    uuid\n    title\n    beginsOn\n    picture {\n      id\n      url\n      name\n      metadata {\n        width\n        height\n        blurhash\n        __typename\n      }\n      __typename\n    }\n    physicalAddress {\n      id\n      description\n      __typename\n    }\n    organizerActor {\n      id\n      avatar {\n        id\n        url\n        __typename\n      }\n      preferredUsername\n      domain\n      name\n      __typename\n    }\n    __typename\n  }\n  options {\n    ...EventOptions\n    __typename\n  }\n  metadata {\n    key\n    title\n    value\n    type\n    __typename\n  }\n  __typename\n}\n\nfragment AdressFragment on Address {\n  id\n  description\n  geom\n  street\n  locality\n  postalCode\n  region\n  country\n  type\n  url\n  originId\n  __typename\n}\n\nfragment TagFragment on Tag {\n  id\n  slug\n  title\n  __typename\n}\n\nfragment EventOptions on EventOptions {\n  maximumAttendeeCapacity\n  remainingAttendeeCapacity\n  showRemainingAttendeeCapacity\n  anonymousParticipation\n  showStartTime\n  showEndTime\n  offers {\n    price\n    priceCurrency\n    url\n    __typename\n  }\n  participationConditions {\n    title\n    content\n    url\n    __typename\n  }\n  attendees\n  program\n  commentModeration\n  showParticipationPrice\n  hideOrganizerWhenGroupEvent\n  __typename\n}\n";
 
 function proxy_fetch(theUrl: string, theOptions: any) {
-    return fetch(theUrl,theOptions);
+    return fetch(theUrl, theOptions);
 }
 
 /**
@@ -51,7 +52,7 @@ class utils {
      */
     counterOfEventsToAdd = 0;
     localMobilizonEventsByTitle: Array<string> = [];
-
+    newEvents: Array<any> = [];
 
     /**
      * converters
@@ -99,9 +100,6 @@ class utils {
             </div>  `
     }
 
-    testdateconvert() {
-        let converted = this.convertRssDate("2021-03-12T19:00:00Z");
-    }
 
     /**
      * file management
@@ -199,12 +197,7 @@ class utils {
             if (this.localMobilizonEventsByTitle.length) {
                 const eventAlreadyExists =
                     -1 !== this.localMobilizonEventsByTitle.indexOf(this.agendadulibre.uniqTitle(event));
-                if (!eventAlreadyExists) {
+
-                    if (parserConfig.debug) {
-                        console.log('ajouter l event ', htmlEscape(this.agendadulibre.uniqTitle(event)));
-                    }
-                    this.agendadulibre.addQuery(event);
-                }
                 return eventAlreadyExists;
             } else {
                 console.log('aucun évènement dans localMobilizonEventsByTitle');
@@ -216,29 +209,27 @@ class utils {
          * @param event
          */
         uniqTitle(event: any): string {
-            return event.start_date + ' ' + event.title
+
+            return moment(new Date(event.start_time))
+                .format("YYYY-MM-DD") + ' ' + event.title
         },
         /**
          * convert events from bdd to a string used as a comparison with new events
          * @param event
          */
         uniqTitleBDD(event: any): string {
-            return event.begins_on + ' ' + event.title
+            return moment(event.begins_on)
+                .format("YYYY-MM-DD") + ' ' + event.title
         },
         doesEventExistsFromJsonScrap: (event: any): boolean => {
+            console.log('this.agendadulibre.uniqTitle(event)', this.agendadulibre.uniqTitle(event));
             const eventAlreadyExists =
-                -1 !== this.localMobilizonEventsByTitle.indexOf(htmlEscape(event.start_date + ' ' + event.title));
+                -1 !== this.localMobilizonEventsByTitle.indexOf(this.agendadulibre.uniqTitle(event));
-            if (!eventAlreadyExists) {
+            console.log('eventAlreadyExists', eventAlreadyExists);
-                if (parserConfig.debug) {
-                    console.log('ajouter l event ', htmlEscape(event.start_date + ' ' + event.title));
-                }
-                this.agendadulibre.addQueryFromJsonScrap(event);
-            }
             return eventAlreadyExists;
         },
         addQueryFromJsonScrap: (event: any) => {
-            this.agendadulibre.queryToAdd.push(
+            let newQuery = {
-                {
                 operationName: "createEvent",
                 query: createEventQueryMobilizon,
                 variables: {
@@ -275,9 +266,8 @@ class utils {
                     visibility: "PUBLIC"
                 }
             }
-            );
 
-            return this.agendadulibre.queryToAdd;
+            return newQuery;
         },
         addQuery: (event: any) => {
             console.log('event', event.title);
@@ -325,7 +315,7 @@ class utils {
 
             return this.agendadulibre.queryToAdd;
         },
-        addNewEventsFromQueries:()=>{
+        addNewEventsFromQueries: () => {
 
             if (parserConfig.runAddQueriesToMobilizonAPI) {
                 console.log(
@@ -349,8 +339,8 @@ class utils {
                         }
 
                         if (parserConfig.enableFetch) {
-                            let theUrl :string = parserConfig.dev_mode ? parserConfig.dev_url : parserConfig.mobilizon_public_url;
+                            let theUrl: string = parserConfig.dev_mode ? parserConfig.dev_url : parserConfig.mobilizon_public_url;
-                            let theOptions:any = {
+                            let theOptions: any = {
                                 method: "post",
                                 body: JSON.stringify(body),
                                 headers: {