From 7228429013125a83b95f7dd947dfcb6a9c036244 Mon Sep 17 00:00:00 2001 From: Tykayn Date: Thu, 13 Jan 2022 12:02:51 +0100 Subject: [PATCH] ok import but wrong title --- config.ts | 2 +- importers/adl.ts | 135 +++++++++++++++++++++++++++++++++-------------- tsconfig.json | 4 +- utils.ts | 96 ++++++++++++++++++++++----------- 4 files changed, 164 insertions(+), 73 deletions(-) diff --git a/config.ts b/config.ts index 6ae2c32..ad74bd8 100644 --- a/config.ts +++ b/config.ts @@ -23,7 +23,7 @@ const parserConfig = { dev_mode: true, // dev mode uses localhost instance of mobilizon running on port 4000 limit_persistence_of_new_events:true, max_new_events: 1, - bearer_token: "eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJtb2JpbGl6b24iLCJleHAiOjE2NDE4MTU0MDMsImlhdCI6MTY0MTgxNDUwMywiaXNzIjoibW9iaWxpem9uIiwianRpIjoiNTU0ZGRlY2YtZDNjMS00NzcwLWJkODAtMTAwMjM3ZjkwYmU2IiwibmJmIjoxNjQxODE0NTAyLCJzdWIiOiJVc2VyOjEiLCJ0eXAiOiJhY2Nlc3MifQ._tFmxJTIBY01vhv9Sj5uADIC_5pP6kx3xdgUgefS36r-lrOZlLeKvD0DpDPh4DM3-lkTonPtZlGRcIyr0I5DpQ", + bearer_token: "eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJtb2JpbGl6b24iLCJleHAiOjE2NDIwNzIyMTQsImlhdCI6MTY0MjA3MTMxNCwiaXNzIjoibW9iaWxpem9uIiwianRpIjoiMjBiNGMyNGEtODViYi00MWFlLTk2MjYtMjZhZjFlYThlMTliIiwibmJmIjoxNjQyMDcxMzEzLCJzdWIiOiJVc2VyOjEiLCJ0eXAiOiJhY2Nlc3MifQ.N-QBR0lfj8KX2icUjOVImahIKvZu1rVJrocCqWvC3Z9LnEaNJ5JEqSdtqnMZBD_v5zK0zfddhBMoa8Iv-KhkaA", ccpl: "https://www.cc-paysdelimours.fr/agenda" diff --git a/importers/adl.ts b/importers/adl.ts index 748b050..1133f2f 100644 --- a/importers/adl.ts +++ b/importers/adl.ts @@ -1,5 +1,7 @@ import * as fs from "fs"; import utils from "../utils"; +import parserConfig from "../config"; +import fetch from 'node-fetch'; let utilsTools = new utils(); @@ -13,52 +15,105 @@ const runCreationQuery = utilsTools.runCreationQuery; async function runImportEvents() { - console.log('File exists'); - // build list of existing events in mobilizon database + console.log('File exists'); + // build list of existing events in mobilizon database - await utilsTools.setupClientPostgresql(); - await utilsTools.client.connect(); - console.log('✅ OK connecté à postgresql'); - console.log(' '); - const res = await utilsTools.client.query('SELECT * from events'); - console.log('💾 évènements enregistrés dans mobilizon : ', res.rows.length); - res.rows.forEach((bdd_event: any) => { - console.log('eventObject', bdd_event); - utilsTools.localMobilizonEventsByTitle.push(utilsTools.agendadulibre.uniqTitleBDD(bdd_event)); - }) + await utilsTools.setupClientPostgresql(); + await utilsTools.client.connect(); + console.log('✅ OK connecté à postgresql'); + console.log(' '); + const res = await utilsTools.client.query('SELECT * from events'); + console.log('💾 évènements enregistrés dans mobilizon : ', res.rows.length); + res.rows.forEach((bdd_event: any) => { + utilsTools.localMobilizonEventsByTitle.push(utilsTools.agendadulibre.uniqTitleBDD(bdd_event)); + }) console.log('utilsTools.localMobilizonEventsByTitle', utilsTools.localMobilizonEventsByTitle); - // get json file for ADL - fs.readFile(filepath, 'utf8', function (err, data) { - if (err) { - return console.log(err); + // get json file for ADL + await fs.readFile(filepath, 'utf8', function (err, data) { + if (err) { + return console.log(err); + } + filecontent = JSON.parse(data) + // console.log('events in the scrapped json', filecontent.length); + let ii = 0; + filecontent.forEach((event: any) => { + ii++ + // compare events with existing events + // let date = new Date(event.start_time); + // console.log(ii, date.toLocaleDateString('fr-fr', { weekday:"long", year:"numeric", month:"short", day:"numeric"}) , event.title ); + utilsTools.agendadulibre.doesEventExistsFromJsonScrap(event); + + }) + console.log('end looking for events'); + +// import only new events + console.log('nouveaux évènements à ajouter: ', utilsTools.agendadulibre.queryToAdd.length); + let limiter = parserConfig.limit_persistence_of_new_events; + let counter = 0; + + utilsTools.agendadulibre.queryToAdd.forEach((graphQLquery: any) => { + if (limiter && counter <= parserConfig.max_new_events) { + counter++; + console.log(counter, ' * ', graphQLquery.variables.beginsOn, graphQLquery.variables.title); + + const body = { + operationName: "createEvent", + query: graphQLquery.query, + variables: graphQLquery.variables } - filecontent = JSON.parse(data) - // console.log('events in the scrapped json', filecontent.length); - let ii=0; - filecontent.forEach((event:any)=>{ - ii++ - let date = new Date(event.start_time); - // console.log(ii, date.toLocaleDateString('fr-fr', { weekday:"long", year:"numeric", month:"short", day:"numeric"}) , event.title ); - utilsTools.agendadulibre.doesEventExistsFromJsonScrap(event); - utilsTools.agendadulibre.addQueryFromJsonScrap(event); - }) - // compare events - // import only new events - - }); + if (parserConfig.enableFetch) { + let url: string = parserConfig.dev_mode ? parserConfig.dev_url : parserConfig.mobilizon_public_url; + let options: any = { + method: "post", + body: JSON.stringify(body), + headers: { + "Content-Type": "application/json", + "authorization": "Bearer " + parserConfig.bearer_token, + } + } + fetchEvent(url, options) + } else { + console.log('---- le fetch est désactivé'); + } + } + }) +// end running queries + }); } -fs.stat(filepath, function (err, stat) { - if (err == null) { - runImportEvents(); - } else if (err.code === 'ENOENT') { - // file does not exist - console.log('Scrapped json file does not exist. Run a scraper like "ts-node scrapers/adl.ts" before using this importer: ', err.code); - } else { - console.log('Some other error: ', err.code); - } -}); + +const fetchEvent = (theUrl, theOptions) => { + fetch(theUrl, theOptions) + .then((res: any) => { + let status = res.status; + console.log('status', status); + if (status === 401) { + console.error(' /!\\ ------------------ ERROR: Bearer token invalid ------------------') + + } else if (status === 200) { + console.log('succès'); + + } + res.json() + }) + .then((json: any) => console.log(json)) + .catch((err: any) => console.log(err)) +} + +// fs.stat(filepath, function (err, stat) { +// if (err == null) { +runImportEvents(); +console.log('fetch', fetch); +// } else if (err.code === 'ENOENT') { +// // file does not exist +// console.log('Scrapped json file does not exist. Run a scraper like "ts-node scrapers/adl.ts" before using this importer: ', err.code); +// } else { +// console.log('Some other error: ', err.code); +// } +// }); + + diff --git a/tsconfig.json b/tsconfig.json index 4fc4d0d..91c178d 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,9 +1,9 @@ { "compilerOptions": { - "lib": ["es5", "es6", "dom"], + "lib": ["es2017", "dom"], "outDir": "./dist/", "sourceMap": true, - "noImplicitAny": true, + "noImplicitAny": false, "module": "commonjs", "target": "es6", "moduleResolution": "node", diff --git a/utils.ts b/utils.ts index 9a040cf..a12a8ab 100644 --- a/utils.ts +++ b/utils.ts @@ -3,13 +3,18 @@ import parserConfig from "./config.ts"; import {v4 as uuidv4} from "uuid"; import {Client} from "pg"; import {htmlEscape} from "escape-goat"; - +import fetch from 'node-fetch'; const moment = require("moment"); const fs = require("fs"); +let organizerActorId = 3; let createEventQueryMobilizon = "mutation createEvent($organizerActorId: ID!, $attributedToId: ID, $title: String!, $description: String!, $beginsOn: DateTime!, $endsOn: DateTime, $status: EventStatus, $visibility: EventVisibility, $joinOptions: EventJoinOptions, $draft: Boolean, $tags: [String], $picture: MediaInput, $onlineAddress: String, $phoneAddress: String, $category: String, $physicalAddress: AddressInput, $options: EventOptionsInput, $contacts: [Contact]) {\n createEvent(\n organizerActorId: $organizerActorId\n attributedToId: $attributedToId\n title: $title\n description: $description\n beginsOn: $beginsOn\n endsOn: $endsOn\n status: $status\n visibility: $visibility\n joinOptions: $joinOptions\n draft: $draft\n tags: $tags\n picture: $picture\n onlineAddress: $onlineAddress\n phoneAddress: $phoneAddress\n category: $category\n physicalAddress: $physicalAddress\n options: $options\n contacts: $contacts\n ) {\n ...FullEvent\n __typename\n }\n}\n\nfragment FullEvent on Event {\n id\n uuid\n url\n local\n title\n description\n beginsOn\n endsOn\n status\n visibility\n joinOptions\n draft\n picture {\n id\n url\n name\n metadata {\n width\n height\n blurhash\n __typename\n }\n __typename\n }\n publishAt\n onlineAddress\n phoneAddress\n physicalAddress {\n ...AdressFragment\n __typename\n }\n organizerActor {\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n domain\n name\n url\n id\n summary\n __typename\n }\n contacts {\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n name\n summary\n domain\n url\n id\n __typename\n }\n attributedTo {\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n name\n summary\n domain\n url\n id\n __typename\n }\n participantStats {\n going\n notApproved\n participant\n __typename\n }\n tags {\n ...TagFragment\n __typename\n }\n relatedEvents {\n id\n uuid\n title\n beginsOn\n picture {\n id\n url\n name\n metadata {\n width\n height\n blurhash\n __typename\n }\n __typename\n }\n physicalAddress {\n id\n description\n __typename\n }\n organizerActor {\n id\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n domain\n name\n __typename\n }\n __typename\n }\n options {\n ...EventOptions\n __typename\n }\n metadata {\n key\n title\n value\n type\n __typename\n }\n __typename\n}\n\nfragment AdressFragment on Address {\n id\n description\n geom\n street\n locality\n postalCode\n region\n country\n type\n url\n originId\n __typename\n}\n\nfragment TagFragment on Tag {\n id\n slug\n title\n __typename\n}\n\nfragment EventOptions on EventOptions {\n maximumAttendeeCapacity\n remainingAttendeeCapacity\n showRemainingAttendeeCapacity\n anonymousParticipation\n showStartTime\n showEndTime\n offers {\n price\n priceCurrency\n url\n __typename\n }\n participationConditions {\n title\n content\n url\n __typename\n }\n attendees\n program\n commentModeration\n showParticipationPrice\n hideOrganizerWhenGroupEvent\n __typename\n}\n"; +function proxy_fetch(theUrl: string, theOptions: any) { + return fetch(theUrl,theOptions); +} + /** * utilitaries to manipulate scraped object and prepare queries to import in mobilizon */ @@ -227,13 +232,11 @@ class utils { if (parserConfig.debug) { console.log('ajouter l event ', htmlEscape(event.start_date + ' ' + event.title)); } - this.agendadulibre.addQuery(event); + this.agendadulibre.addQueryFromJsonScrap(event); } return eventAlreadyExists; }, addQueryFromJsonScrap: (event: any) => { - console.log('event', event.title); - this.agendadulibre.queryToAdd.push( { operationName: "createEvent", @@ -262,12 +265,10 @@ class utils { showRemainingAttendeeCapacity: false, showStartTime: true }, - organizerActorId: "3", + organizerActorId: organizerActorId, phoneAddress: "", status: "CONFIRMED", tags: [ - "osm", - "openstreetmap", "imported" ], title: event.title, @@ -308,7 +309,7 @@ class utils { showRemainingAttendeeCapacity: false, showStartTime: true }, - organizerActorId: "3", + organizerActorId: organizerActorId, phoneAddress: "", status: "CONFIRMED", tags: [ @@ -324,29 +325,64 @@ class utils { return this.agendadulibre.queryToAdd; }, - addQueryBDD: (event: any) => { - if (this.agendadulibre.queryToAdd) { - this.agendadulibre.queryToAdd += ` , `; + addNewEventsFromQueries:()=>{ + + if (parserConfig.runAddQueriesToMobilizonAPI) { + console.log( + '➕ rajouter les évènements manquants par l\'API GraphQL', + this.agendadulibre.queryToAdd.length + ); + let limiter = parserConfig.limit_persistence_of_new_events; + let counter = 0; + let bearerTokenIsOK = true; + + this.agendadulibre.queryToAdd.forEach((graphQLquery: any) => { + + if (limiter && counter <= parserConfig.max_new_events && bearerTokenIsOK) { + counter++; + console.log(counter, ' * ', graphQLquery.variables.title); + + const body = { + operationName: "createEvent", + query: graphQLquery.query, + variables: graphQLquery.variables + } + + if (parserConfig.enableFetch) { + let theUrl :string = parserConfig.dev_mode ? parserConfig.dev_url : parserConfig.mobilizon_public_url; + let theOptions:any = { + method: "post", + body: JSON.stringify(body), + headers: { + "Content-Type": "application/json", + "authorization": "Bearer " + parserConfig.bearer_token, + } + } + proxy_fetch(theUrl, theOptions) + .then((res: any) => { + let status = res.status; + console.log('status', status); + if (status === 401) { + console.error(' /!\\ ------------------ ERROR: Bearer token invalid ------------------') + bearerTokenIsOK = false; + } else if (status === 200) { + console.log('succès'); + + } + res.json() + }) + .then((json: any) => console.log(json)) + .catch((err: any) => console.log(err)) + } else { + console.log('---- le fetch est désactivé'); + } + + + } + + + }); } - - let title = "'" + htmlEscape(event.title) + "'"; - let content = "'" + htmlEscape(event.content) + "'"; - let uuid = uuidv4(); - let uuidString = "'" + uuid + "'"; - let eventUrl = - "'" + parserConfig.mobilizon_public_url + "/events/" + uuid + "'"; - let begins_on = "'" + this.convertRssDate(event.date) + "'"; - let ends_on = "'" + this.convertRssDate(event.date) + "'"; - let baseOptions = - '{"offers": [], "program": null, "attendees": [], "show_end_time": true, "show_start_time": true, "comment_moderation": "allow_all", "anonymous_participation": true, "participation_condition": [], "show_participation_price": false, "maximum_attendee_capacity": 0, "remaining_attendee_capacity": 0, "hide_organizer_when_group_event": false, "show_remaining_attendee_capacity": false}'; - let baseStats = - '{"creator": 1, "rejected": 0, "moderator": 0, "participant": 0, "not_approved": 0, "administrator": 0, "not_confirmed": 0}'; - // begins_on , ends_on expecting date format like this: "2020-12-17 23:00:00" - - this.agendadulibre.queryToAdd += `( ${title}, ${content}, ${parserConfig.feeder_mobilizon_user_id}, 'now()','now()', ${uuidString}, ${eventUrl}, 'confirmed' , 'meeting', ${baseOptions}, ${baseStats}, ${begins_on} , ${ends_on} )`; - - this.agendadulibre.counterOfEventsToAdd++; - this.counterOfEventsToAdd++; } };