tykayn
/
rss-feeder-mobilizon
2
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

ok import but wrong title

This commit is contained in:
Tykayn 2022-01-13 12:02:51 +01:00 committed by tykayn
parent f3cb3673a9
commit 7228429013
4 changed files with 164 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
config.ts
View File

@ -23,7 +23,7 @@ const parserConfig = {
dev_mode: true, // dev mode uses localhost instance of mobilizon running on port 4000 dev_mode: true, // dev mode uses localhost instance of mobilizon running on port 4000
limit_persistence_of_new_events:true, limit_persistence_of_new_events:true,
max_new_events: 1, max_new_events: 1,
bearer_token: "eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJtb2JpbGl6b24iLCJleHAiOjE2NDE4MTU0MDMsImlhdCI6MTY0MTgxNDUwMywiaXNzIjoibW9iaWxpem9uIiwianRpIjoiNTU0ZGRlY2YtZDNjMS00NzcwLWJkODAtMTAwMjM3ZjkwYmU2IiwibmJmIjoxNjQxODE0NTAyLCJzdWIiOiJVc2VyOjEiLCJ0eXAiOiJhY2Nlc3MifQ._tFmxJTIBY01vhv9Sj5uADIC_5pP6kx3xdgUgefS36r-lrOZlLeKvD0DpDPh4DM3-lkTonPtZlGRcIyr0I5DpQ", bearer_token: "eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJtb2JpbGl6b24iLCJleHAiOjE2NDIwNzIyMTQsImlhdCI6MTY0MjA3MTMxNCwiaXNzIjoibW9iaWxpem9uIiwianRpIjoiMjBiNGMyNGEtODViYi00MWFlLTk2MjYtMjZhZjFlYThlMTliIiwibmJmIjoxNjQyMDcxMzEzLCJzdWIiOiJVc2VyOjEiLCJ0eXAiOiJhY2Nlc3MifQ.N-QBR0lfj8KX2icUjOVImahIKvZu1rVJrocCqWvC3Z9LnEaNJ5JEqSdtqnMZBD_v5zK0zfddhBMoa8Iv-KhkaA",
ccpl: "https://www.cc-paysdelimours.fr/agenda" ccpl: "https://www.cc-paysdelimours.fr/agenda"

135
importers/adl.ts
View File

@ -1,5 +1,7 @@
import * as fs from "fs"; import * as fs from "fs";
import utils from "../utils"; import utils from "../utils";
import parserConfig from "../config";
import fetch from 'node-fetch';
let utilsTools = new utils(); let utilsTools = new utils();
@ -13,52 +15,105 @@ const runCreationQuery = utilsTools.runCreationQuery;
async function runImportEvents() { async function runImportEvents() {
console.log('File exists'); console.log('File exists');
// build list of existing events in mobilizon database // build list of existing events in mobilizon database
await utilsTools.setupClientPostgresql(); await utilsTools.setupClientPostgresql();
await utilsTools.client.connect(); await utilsTools.client.connect();
console.log('✅ OK connecté à postgresql'); console.log('✅ OK connecté à postgresql');
console.log(' '); console.log(' ');
const res = await utilsTools.client.query('SELECT * from events'); const res = await utilsTools.client.query('SELECT * from events');
console.log('💾 évènements enregistrés dans mobilizon : ', res.rows.length); console.log('💾 évènements enregistrés dans mobilizon : ', res.rows.length);
res.rows.forEach((bdd_event: any) => { res.rows.forEach((bdd_event: any) => {
console.log('eventObject', bdd_event); utilsTools.localMobilizonEventsByTitle.push(utilsTools.agendadulibre.uniqTitleBDD(bdd_event));
utilsTools.localMobilizonEventsByTitle.push(utilsTools.agendadulibre.uniqTitleBDD(bdd_event)); })
})
console.log('utilsTools.localMobilizonEventsByTitle', utilsTools.localMobilizonEventsByTitle); console.log('utilsTools.localMobilizonEventsByTitle', utilsTools.localMobilizonEventsByTitle);
// get json file for ADL // get json file for ADL
fs.readFile(filepath, 'utf8', function (err, data) { await fs.readFile(filepath, 'utf8', function (err, data) {
if (err) { if (err) {
return console.log(err); return console.log(err);
}
filecontent = JSON.parse(data)
// console.log('events in the scrapped json', filecontent.length);
let ii = 0;
filecontent.forEach((event: any) => {
ii++
// compare events with existing events
// let date = new Date(event.start_time);
// console.log(ii, date.toLocaleDateString('fr-fr', { weekday:"long", year:"numeric", month:"short", day:"numeric"}) , event.title );
utilsTools.agendadulibre.doesEventExistsFromJsonScrap(event);
})
console.log('end looking for events');
// import only new events
console.log('nouveaux évènements à ajouter: ', utilsTools.agendadulibre.queryToAdd.length);
let limiter = parserConfig.limit_persistence_of_new_events;
let counter = 0;
utilsTools.agendadulibre.queryToAdd.forEach((graphQLquery: any) => {
if (limiter && counter <= parserConfig.max_new_events) {
counter++;
console.log(counter, ' * ', graphQLquery.variables.beginsOn, graphQLquery.variables.title);
const body = {
operationName: "createEvent",
query: graphQLquery.query,
variables: graphQLquery.variables
} }
filecontent = JSON.parse(data)
// console.log('events in the scrapped json', filecontent.length);
let ii=0;
filecontent.forEach((event:any)=>{
ii++
let date = new Date(event.start_time);
// console.log(ii, date.toLocaleDateString('fr-fr', { weekday:"long", year:"numeric", month:"short", day:"numeric"}) , event.title ); if (parserConfig.enableFetch) {
utilsTools.agendadulibre.doesEventExistsFromJsonScrap(event); let url: string = parserConfig.dev_mode ? parserConfig.dev_url : parserConfig.mobilizon_public_url;
utilsTools.agendadulibre.addQueryFromJsonScrap(event); let options: any = {
}) method: "post",
// compare events body: JSON.stringify(body),
// import only new events headers: {
"Content-Type": "application/json",
}); "authorization": "Bearer " + parserConfig.bearer_token,
}
}
fetchEvent(url, options)
} else {
console.log('---- le fetch est désactivé');
}
}
})
// end running queries
});
} }
fs.stat(filepath, function (err, stat) {
if (err == null) { const fetchEvent = (theUrl, theOptions) => {
runImportEvents(); fetch(theUrl, theOptions)
} else if (err.code === 'ENOENT') { .then((res: any) => {
// file does not exist let status = res.status;
console.log('Scrapped json file does not exist. Run a scraper like "ts-node scrapers/adl.ts" before using this importer: ', err.code); console.log('status', status);
} else { if (status === 401) {
console.log('Some other error: ', err.code); console.error(' /!\\ ------------------ ERROR: Bearer token invalid ------------------')
}
}); } else if (status === 200) {
console.log('succès');
}
res.json()
})
.then((json: any) => console.log(json))
.catch((err: any) => console.log(err))
}
// fs.stat(filepath, function (err, stat) {
// if (err == null) {
runImportEvents();
console.log('fetch', fetch);
// } else if (err.code === 'ENOENT') {
// // file does not exist
// console.log('Scrapped json file does not exist. Run a scraper like "ts-node scrapers/adl.ts" before using this importer: ', err.code);
// } else {
// console.log('Some other error: ', err.code);
// }
// });

4
tsconfig.json
View File

@ -1,9 +1,9 @@
{ {
"compilerOptions": { "compilerOptions": {
"lib": ["es5", "es6", "dom"], "lib": ["es2017", "dom"],
"outDir": "./dist/", "outDir": "./dist/",
"sourceMap": true, "sourceMap": true,
"noImplicitAny": true, "noImplicitAny": false,
"module": "commonjs", "module": "commonjs",
"target": "es6", "target": "es6",
"moduleResolution": "node", "moduleResolution": "node",

96
utils.ts
View File

@ -3,13 +3,18 @@ import parserConfig from "./config.ts";
import {v4 as uuidv4} from "uuid"; import {v4 as uuidv4} from "uuid";
import {Client} from "pg"; import {Client} from "pg";
import {htmlEscape} from "escape-goat"; import {htmlEscape} from "escape-goat";
import fetch from 'node-fetch';
const moment = require("moment"); const moment = require("moment");
const fs = require("fs"); const fs = require("fs");
let organizerActorId = 3;
let createEventQueryMobilizon = "mutation createEvent($organizerActorId: ID!, $attributedToId: ID, $title: String!, $description: String!, $beginsOn: DateTime!, $endsOn: DateTime, $status: EventStatus, $visibility: EventVisibility, $joinOptions: EventJoinOptions, $draft: Boolean, $tags: [String], $picture: MediaInput, $onlineAddress: String, $phoneAddress: String, $category: String, $physicalAddress: AddressInput, $options: EventOptionsInput, $contacts: [Contact]) {\n createEvent(\n organizerActorId: $organizerActorId\n attributedToId: $attributedToId\n title: $title\n description: $description\n beginsOn: $beginsOn\n endsOn: $endsOn\n status: $status\n visibility: $visibility\n joinOptions: $joinOptions\n draft: $draft\n tags: $tags\n picture: $picture\n onlineAddress: $onlineAddress\n phoneAddress: $phoneAddress\n category: $category\n physicalAddress: $physicalAddress\n options: $options\n contacts: $contacts\n ) {\n ...FullEvent\n __typename\n }\n}\n\nfragment FullEvent on Event {\n id\n uuid\n url\n local\n title\n description\n beginsOn\n endsOn\n status\n visibility\n joinOptions\n draft\n picture {\n id\n url\n name\n metadata {\n width\n height\n blurhash\n __typename\n }\n __typename\n }\n publishAt\n onlineAddress\n phoneAddress\n physicalAddress {\n ...AdressFragment\n __typename\n }\n organizerActor {\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n domain\n name\n url\n id\n summary\n __typename\n }\n contacts {\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n name\n summary\n domain\n url\n id\n __typename\n }\n attributedTo {\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n name\n summary\n domain\n url\n id\n __typename\n }\n participantStats {\n going\n notApproved\n participant\n __typename\n }\n tags {\n ...TagFragment\n __typename\n }\n relatedEvents {\n id\n uuid\n title\n beginsOn\n picture {\n id\n url\n name\n metadata {\n width\n height\n blurhash\n __typename\n }\n __typename\n }\n physicalAddress {\n id\n description\n __typename\n }\n organizerActor {\n id\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n domain\n name\n __typename\n }\n __typename\n }\n options {\n ...EventOptions\n __typename\n }\n metadata {\n key\n title\n value\n type\n __typename\n }\n __typename\n}\n\nfragment AdressFragment on Address {\n id\n description\n geom\n street\n locality\n postalCode\n region\n country\n type\n url\n originId\n __typename\n}\n\nfragment TagFragment on Tag {\n id\n slug\n title\n __typename\n}\n\nfragment EventOptions on EventOptions {\n maximumAttendeeCapacity\n remainingAttendeeCapacity\n showRemainingAttendeeCapacity\n anonymousParticipation\n showStartTime\n showEndTime\n offers {\n price\n priceCurrency\n url\n __typename\n }\n participationConditions {\n title\n content\n url\n __typename\n }\n attendees\n program\n commentModeration\n showParticipationPrice\n hideOrganizerWhenGroupEvent\n __typename\n}\n"; let createEventQueryMobilizon = "mutation createEvent($organizerActorId: ID!, $attributedToId: ID, $title: String!, $description: String!, $beginsOn: DateTime!, $endsOn: DateTime, $status: EventStatus, $visibility: EventVisibility, $joinOptions: EventJoinOptions, $draft: Boolean, $tags: [String], $picture: MediaInput, $onlineAddress: String, $phoneAddress: String, $category: String, $physicalAddress: AddressInput, $options: EventOptionsInput, $contacts: [Contact]) {\n createEvent(\n organizerActorId: $organizerActorId\n attributedToId: $attributedToId\n title: $title\n description: $description\n beginsOn: $beginsOn\n endsOn: $endsOn\n status: $status\n visibility: $visibility\n joinOptions: $joinOptions\n draft: $draft\n tags: $tags\n picture: $picture\n onlineAddress: $onlineAddress\n phoneAddress: $phoneAddress\n category: $category\n physicalAddress: $physicalAddress\n options: $options\n contacts: $contacts\n ) {\n ...FullEvent\n __typename\n }\n}\n\nfragment FullEvent on Event {\n id\n uuid\n url\n local\n title\n description\n beginsOn\n endsOn\n status\n visibility\n joinOptions\n draft\n picture {\n id\n url\n name\n metadata {\n width\n height\n blurhash\n __typename\n }\n __typename\n }\n publishAt\n onlineAddress\n phoneAddress\n physicalAddress {\n ...AdressFragment\n __typename\n }\n organizerActor {\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n domain\n name\n url\n id\n summary\n __typename\n }\n contacts {\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n name\n summary\n domain\n url\n id\n __typename\n }\n attributedTo {\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n name\n summary\n domain\n url\n id\n __typename\n }\n participantStats {\n going\n notApproved\n participant\n __typename\n }\n tags {\n ...TagFragment\n __typename\n }\n relatedEvents {\n id\n uuid\n title\n beginsOn\n picture {\n id\n url\n name\n metadata {\n width\n height\n blurhash\n __typename\n }\n __typename\n }\n physicalAddress {\n id\n description\n __typename\n }\n organizerActor {\n id\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n domain\n name\n __typename\n }\n __typename\n }\n options {\n ...EventOptions\n __typename\n }\n metadata {\n key\n title\n value\n type\n __typename\n }\n __typename\n}\n\nfragment AdressFragment on Address {\n id\n description\n geom\n street\n locality\n postalCode\n region\n country\n type\n url\n originId\n __typename\n}\n\nfragment TagFragment on Tag {\n id\n slug\n title\n __typename\n}\n\nfragment EventOptions on EventOptions {\n maximumAttendeeCapacity\n remainingAttendeeCapacity\n showRemainingAttendeeCapacity\n anonymousParticipation\n showStartTime\n showEndTime\n offers {\n price\n priceCurrency\n url\n __typename\n }\n participationConditions {\n title\n content\n url\n __typename\n }\n attendees\n program\n commentModeration\n showParticipationPrice\n hideOrganizerWhenGroupEvent\n __typename\n}\n";
function proxy_fetch(theUrl: string, theOptions: any) {
return fetch(theUrl,theOptions);
}
/** /**
* utilitaries to manipulate scraped object and prepare queries to import in mobilizon * utilitaries to manipulate scraped object and prepare queries to import in mobilizon
*/ */
@ -227,13 +232,11 @@ class utils {
if (parserConfig.debug) { if (parserConfig.debug) {
console.log('ajouter l event ', htmlEscape(event.start_date + ' ' + event.title)); console.log('ajouter l event ', htmlEscape(event.start_date + ' ' + event.title));
} }
this.agendadulibre.addQuery(event); this.agendadulibre.addQueryFromJsonScrap(event);
} }
return eventAlreadyExists; return eventAlreadyExists;
}, },
addQueryFromJsonScrap: (event: any) => { addQueryFromJsonScrap: (event: any) => {
console.log('event', event.title);
this.agendadulibre.queryToAdd.push( this.agendadulibre.queryToAdd.push(
{ {
operationName: "createEvent", operationName: "createEvent",
@ -262,12 +265,10 @@ class utils {
showRemainingAttendeeCapacity: false, showRemainingAttendeeCapacity: false,
showStartTime: true showStartTime: true
}, },
organizerActorId: "3", organizerActorId: organizerActorId,
phoneAddress: "", phoneAddress: "",
status: "CONFIRMED", status: "CONFIRMED",
tags: [ tags: [
"osm",
"openstreetmap",
"imported" "imported"
], ],
title: event.title, title: event.title,
@ -308,7 +309,7 @@ class utils {
showRemainingAttendeeCapacity: false, showRemainingAttendeeCapacity: false,
showStartTime: true showStartTime: true
}, },
organizerActorId: "3", organizerActorId: organizerActorId,
phoneAddress: "", phoneAddress: "",
status: "CONFIRMED", status: "CONFIRMED",
tags: [ tags: [
@ -324,29 +325,64 @@ class utils {
return this.agendadulibre.queryToAdd; return this.agendadulibre.queryToAdd;
}, },
addQueryBDD: (event: any) => { addNewEventsFromQueries:()=>{
if (this.agendadulibre.queryToAdd) {
this.agendadulibre.queryToAdd += ` , `; if (parserConfig.runAddQueriesToMobilizonAPI) {
console.log(
'➕ rajouter les évènements manquants par l\'API GraphQL',
this.agendadulibre.queryToAdd.length
);
let limiter = parserConfig.limit_persistence_of_new_events;
let counter = 0;
let bearerTokenIsOK = true;
this.agendadulibre.queryToAdd.forEach((graphQLquery: any) => {
if (limiter && counter <= parserConfig.max_new_events && bearerTokenIsOK) {
counter++;
console.log(counter, ' * ', graphQLquery.variables.title);
const body = {
operationName: "createEvent",
query: graphQLquery.query,
variables: graphQLquery.variables
}
if (parserConfig.enableFetch) {
let theUrl :string = parserConfig.dev_mode ? parserConfig.dev_url : parserConfig.mobilizon_public_url;
let theOptions:any = {
method: "post",
body: JSON.stringify(body),
headers: {
"Content-Type": "application/json",
"authorization": "Bearer " + parserConfig.bearer_token,
}
}
proxy_fetch(theUrl, theOptions)
.then((res: any) => {
let status = res.status;
console.log('status', status);
if (status === 401) {
console.error(' /!\\ ------------------ ERROR: Bearer token invalid ------------------')
bearerTokenIsOK = false;
} else if (status === 200) {
console.log('succès');
}
res.json()
})
.then((json: any) => console.log(json))
.catch((err: any) => console.log(err))
} else {
console.log('---- le fetch est désactivé');
}
}
});
} }
let title = "'" + htmlEscape(event.title) + "'";
let content = "'" + htmlEscape(event.content) + "'";
let uuid = uuidv4();
let uuidString = "'" + uuid + "'";
let eventUrl =
"'" + parserConfig.mobilizon_public_url + "/events/" + uuid + "'";
let begins_on = "'" + this.convertRssDate(event.date) + "'";
let ends_on = "'" + this.convertRssDate(event.date) + "'";
let baseOptions =
'{"offers": [], "program": null, "attendees": [], "show_end_time": true, "show_start_time": true, "comment_moderation": "allow_all", "anonymous_participation": true, "participation_condition": [], "show_participation_price": false, "maximum_attendee_capacity": 0, "remaining_attendee_capacity": 0, "hide_organizer_when_group_event": false, "show_remaining_attendee_capacity": false}';
let baseStats =
'{"creator": 1, "rejected": 0, "moderator": 0, "participant": 0, "not_approved": 0, "administrator": 0, "not_confirmed": 0}';
// begins_on , ends_on expecting date format like this: "2020-12-17 23:00:00"
this.agendadulibre.queryToAdd += `( ${title}, ${content}, ${parserConfig.feeder_mobilizon_user_id}, 'now()','now()', ${uuidString}, ${eventUrl}, 'confirmed' , 'meeting', ${baseOptions}, ${baseStats}, ${begins_on} , ${ends_on} )`;
this.agendadulibre.counterOfEventsToAdd++;
this.counterOfEventsToAdd++;
} }
}; };