ok import but wrong title
This commit is contained in:
tykayn
parent
f3cb3673a9
commit
7228429013
@ -23,7 +23,7 @@ const parserConfig = {
|
||||
dev_mode: true, // dev mode uses localhost instance of mobilizon running on port 4000
|
||||
limit_persistence_of_new_events:true,
|
||||
max_new_events: 1,
|
||||
bearer_token: "eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJtb2JpbGl6b24iLCJleHAiOjE2NDE4MTU0MDMsImlhdCI6MTY0MTgxNDUwMywiaXNzIjoibW9iaWxpem9uIiwianRpIjoiNTU0ZGRlY2YtZDNjMS00NzcwLWJkODAtMTAwMjM3ZjkwYmU2IiwibmJmIjoxNjQxODE0NTAyLCJzdWIiOiJVc2VyOjEiLCJ0eXAiOiJhY2Nlc3MifQ._tFmxJTIBY01vhv9Sj5uADIC_5pP6kx3xdgUgefS36r-lrOZlLeKvD0DpDPh4DM3-lkTonPtZlGRcIyr0I5DpQ",
|
||||
bearer_token: "eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJtb2JpbGl6b24iLCJleHAiOjE2NDIwNzIyMTQsImlhdCI6MTY0MjA3MTMxNCwiaXNzIjoibW9iaWxpem9uIiwianRpIjoiMjBiNGMyNGEtODViYi00MWFlLTk2MjYtMjZhZjFlYThlMTliIiwibmJmIjoxNjQyMDcxMzEzLCJzdWIiOiJVc2VyOjEiLCJ0eXAiOiJhY2Nlc3MifQ.N-QBR0lfj8KX2icUjOVImahIKvZu1rVJrocCqWvC3Z9LnEaNJ5JEqSdtqnMZBD_v5zK0zfddhBMoa8Iv-KhkaA",
|
||||
|
||||
ccpl: "https://www.cc-paysdelimours.fr/agenda"
|
||||
|
||||
|
||||
135
importers/adl.ts
135
importers/adl.ts
@ -1,5 +1,7 @@
|
||||
import * as fs from "fs";
|
||||
import utils from "../utils";
|
||||
import parserConfig from "../config";
|
||||
import fetch from 'node-fetch';
|
||||
|
||||
let utilsTools = new utils();
|
||||
|
||||
@ -13,52 +15,105 @@ const runCreationQuery = utilsTools.runCreationQuery;
|
||||
async function runImportEvents() {
|
||||
|
||||
|
||||
console.log('File exists');
|
||||
// build list of existing events in mobilizon database
|
||||
console.log('File exists');
|
||||
// build list of existing events in mobilizon database
|
||||
|
||||
await utilsTools.setupClientPostgresql();
|
||||
await utilsTools.client.connect();
|
||||
console.log('✅ OK connecté à postgresql');
|
||||
console.log(' ');
|
||||
const res = await utilsTools.client.query('SELECT * from events');
|
||||
console.log('💾 évènements enregistrés dans mobilizon : ', res.rows.length);
|
||||
res.rows.forEach((bdd_event: any) => {
|
||||
console.log('eventObject', bdd_event);
|
||||
utilsTools.localMobilizonEventsByTitle.push(utilsTools.agendadulibre.uniqTitleBDD(bdd_event));
|
||||
})
|
||||
await utilsTools.setupClientPostgresql();
|
||||
await utilsTools.client.connect();
|
||||
console.log('✅ OK connecté à postgresql');
|
||||
console.log(' ');
|
||||
const res = await utilsTools.client.query('SELECT * from events');
|
||||
console.log('💾 évènements enregistrés dans mobilizon : ', res.rows.length);
|
||||
res.rows.forEach((bdd_event: any) => {
|
||||
utilsTools.localMobilizonEventsByTitle.push(utilsTools.agendadulibre.uniqTitleBDD(bdd_event));
|
||||
})
|
||||
|
||||
console.log('utilsTools.localMobilizonEventsByTitle', utilsTools.localMobilizonEventsByTitle);
|
||||
// get json file for ADL
|
||||
fs.readFile(filepath, 'utf8', function (err, data) {
|
||||
if (err) {
|
||||
return console.log(err);
|
||||
// get json file for ADL
|
||||
await fs.readFile(filepath, 'utf8', function (err, data) {
|
||||
if (err) {
|
||||
return console.log(err);
|
||||
}
|
||||
filecontent = JSON.parse(data)
|
||||
// console.log('events in the scrapped json', filecontent.length);
|
||||
let ii = 0;
|
||||
filecontent.forEach((event: any) => {
|
||||
ii++
|
||||
// compare events with existing events
|
||||
// let date = new Date(event.start_time);
|
||||
// console.log(ii, date.toLocaleDateString('fr-fr', { weekday:"long", year:"numeric", month:"short", day:"numeric"}) , event.title );
|
||||
utilsTools.agendadulibre.doesEventExistsFromJsonScrap(event);
|
||||
|
||||
})
|
||||
console.log('end looking for events');
|
||||
|
||||
// import only new events
|
||||
console.log('nouveaux évènements à ajouter: ', utilsTools.agendadulibre.queryToAdd.length);
|
||||
let limiter = parserConfig.limit_persistence_of_new_events;
|
||||
let counter = 0;
|
||||
|
||||
utilsTools.agendadulibre.queryToAdd.forEach((graphQLquery: any) => {
|
||||
if (limiter && counter <= parserConfig.max_new_events) {
|
||||
counter++;
|
||||
console.log(counter, ' * ', graphQLquery.variables.beginsOn, graphQLquery.variables.title);
|
||||
|
||||
const body = {
|
||||
operationName: "createEvent",
|
||||
query: graphQLquery.query,
|
||||
variables: graphQLquery.variables
|
||||
}
|
||||
filecontent = JSON.parse(data)
|
||||
// console.log('events in the scrapped json', filecontent.length);
|
||||
let ii=0;
|
||||
filecontent.forEach((event:any)=>{
|
||||
ii++
|
||||
let date = new Date(event.start_time);
|
||||
|
||||
// console.log(ii, date.toLocaleDateString('fr-fr', { weekday:"long", year:"numeric", month:"short", day:"numeric"}) , event.title );
|
||||
utilsTools.agendadulibre.doesEventExistsFromJsonScrap(event);
|
||||
utilsTools.agendadulibre.addQueryFromJsonScrap(event);
|
||||
})
|
||||
// compare events
|
||||
// import only new events
|
||||
|
||||
});
|
||||
if (parserConfig.enableFetch) {
|
||||
let url: string = parserConfig.dev_mode ? parserConfig.dev_url : parserConfig.mobilizon_public_url;
|
||||
let options: any = {
|
||||
method: "post",
|
||||
body: JSON.stringify(body),
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
"authorization": "Bearer " + parserConfig.bearer_token,
|
||||
}
|
||||
}
|
||||
fetchEvent(url, options)
|
||||
} else {
|
||||
console.log('---- le fetch est désactivé');
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
})
|
||||
// end running queries
|
||||
});
|
||||
|
||||
}
|
||||
fs.stat(filepath, function (err, stat) {
|
||||
if (err == null) {
|
||||
runImportEvents();
|
||||
} else if (err.code === 'ENOENT') {
|
||||
// file does not exist
|
||||
console.log('Scrapped json file does not exist. Run a scraper like "ts-node scrapers/adl.ts" before using this importer: ', err.code);
|
||||
} else {
|
||||
console.log('Some other error: ', err.code);
|
||||
}
|
||||
});
|
||||
|
||||
const fetchEvent = (theUrl, theOptions) => {
|
||||
fetch(theUrl, theOptions)
|
||||
.then((res: any) => {
|
||||
let status = res.status;
|
||||
console.log('status', status);
|
||||
if (status === 401) {
|
||||
console.error(' /!\\ ------------------ ERROR: Bearer token invalid ------------------')
|
||||
|
||||
} else if (status === 200) {
|
||||
console.log('succès');
|
||||
|
||||
}
|
||||
res.json()
|
||||
})
|
||||
.then((json: any) => console.log(json))
|
||||
.catch((err: any) => console.log(err))
|
||||
}
|
||||
|
||||
// fs.stat(filepath, function (err, stat) {
|
||||
// if (err == null) {
|
||||
runImportEvents();
|
||||
console.log('fetch', fetch);
|
||||
// } else if (err.code === 'ENOENT') {
|
||||
// // file does not exist
|
||||
// console.log('Scrapped json file does not exist. Run a scraper like "ts-node scrapers/adl.ts" before using this importer: ', err.code);
|
||||
// } else {
|
||||
// console.log('Some other error: ', err.code);
|
||||
// }
|
||||
// });
|
||||
|
||||
|
||||
|
||||
@ -1,9 +1,9 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"lib": ["es5", "es6", "dom"],
|
||||
"lib": ["es2017", "dom"],
|
||||
"outDir": "./dist/",
|
||||
"sourceMap": true,
|
||||
"noImplicitAny": true,
|
||||
"noImplicitAny": false,
|
||||
"module": "commonjs",
|
||||
"target": "es6",
|
||||
"moduleResolution": "node",
|
||||
|
||||
96
utils.ts
96
utils.ts
@ -3,13 +3,18 @@ import parserConfig from "./config.ts";
|
||||
import {v4 as uuidv4} from "uuid";
|
||||
import {Client} from "pg";
|
||||
import {htmlEscape} from "escape-goat";
|
||||
|
||||
import fetch from 'node-fetch';
|
||||
const moment = require("moment");
|
||||
|
||||
const fs = require("fs");
|
||||
let organizerActorId = 3;
|
||||
|
||||
let createEventQueryMobilizon = "mutation createEvent($organizerActorId: ID!, $attributedToId: ID, $title: String!, $description: String!, $beginsOn: DateTime!, $endsOn: DateTime, $status: EventStatus, $visibility: EventVisibility, $joinOptions: EventJoinOptions, $draft: Boolean, $tags: [String], $picture: MediaInput, $onlineAddress: String, $phoneAddress: String, $category: String, $physicalAddress: AddressInput, $options: EventOptionsInput, $contacts: [Contact]) {\n createEvent(\n organizerActorId: $organizerActorId\n attributedToId: $attributedToId\n title: $title\n description: $description\n beginsOn: $beginsOn\n endsOn: $endsOn\n status: $status\n visibility: $visibility\n joinOptions: $joinOptions\n draft: $draft\n tags: $tags\n picture: $picture\n onlineAddress: $onlineAddress\n phoneAddress: $phoneAddress\n category: $category\n physicalAddress: $physicalAddress\n options: $options\n contacts: $contacts\n ) {\n ...FullEvent\n __typename\n }\n}\n\nfragment FullEvent on Event {\n id\n uuid\n url\n local\n title\n description\n beginsOn\n endsOn\n status\n visibility\n joinOptions\n draft\n picture {\n id\n url\n name\n metadata {\n width\n height\n blurhash\n __typename\n }\n __typename\n }\n publishAt\n onlineAddress\n phoneAddress\n physicalAddress {\n ...AdressFragment\n __typename\n }\n organizerActor {\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n domain\n name\n url\n id\n summary\n __typename\n }\n contacts {\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n name\n summary\n domain\n url\n id\n __typename\n }\n attributedTo {\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n name\n summary\n domain\n url\n id\n __typename\n }\n participantStats {\n going\n notApproved\n participant\n __typename\n }\n tags {\n ...TagFragment\n __typename\n }\n relatedEvents {\n id\n uuid\n title\n beginsOn\n picture {\n id\n url\n name\n metadata {\n width\n height\n blurhash\n __typename\n }\n __typename\n }\n physicalAddress {\n id\n description\n __typename\n }\n organizerActor {\n id\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n domain\n name\n __typename\n }\n __typename\n }\n options {\n ...EventOptions\n __typename\n }\n metadata {\n key\n title\n value\n type\n __typename\n }\n __typename\n}\n\nfragment AdressFragment on Address {\n id\n description\n geom\n street\n locality\n postalCode\n region\n country\n type\n url\n originId\n __typename\n}\n\nfragment TagFragment on Tag {\n id\n slug\n title\n __typename\n}\n\nfragment EventOptions on EventOptions {\n maximumAttendeeCapacity\n remainingAttendeeCapacity\n showRemainingAttendeeCapacity\n anonymousParticipation\n showStartTime\n showEndTime\n offers {\n price\n priceCurrency\n url\n __typename\n }\n participationConditions {\n title\n content\n url\n __typename\n }\n attendees\n program\n commentModeration\n showParticipationPrice\n hideOrganizerWhenGroupEvent\n __typename\n}\n";
|
||||
|
||||
function proxy_fetch(theUrl: string, theOptions: any) {
|
||||
return fetch(theUrl,theOptions);
|
||||
}
|
||||
|
||||
/**
|
||||
* utilitaries to manipulate scraped object and prepare queries to import in mobilizon
|
||||
*/
|
||||
@ -227,13 +232,11 @@ class utils {
|
||||
if (parserConfig.debug) {
|
||||
console.log('ajouter l event ', htmlEscape(event.start_date + ' ' + event.title));
|
||||
}
|
||||
this.agendadulibre.addQuery(event);
|
||||
this.agendadulibre.addQueryFromJsonScrap(event);
|
||||
}
|
||||
return eventAlreadyExists;
|
||||
},
|
||||
addQueryFromJsonScrap: (event: any) => {
|
||||
console.log('event', event.title);
|
||||
|
||||
this.agendadulibre.queryToAdd.push(
|
||||
{
|
||||
operationName: "createEvent",
|
||||
@ -262,12 +265,10 @@ class utils {
|
||||
showRemainingAttendeeCapacity: false,
|
||||
showStartTime: true
|
||||
},
|
||||
organizerActorId: "3",
|
||||
organizerActorId: organizerActorId,
|
||||
phoneAddress: "",
|
||||
status: "CONFIRMED",
|
||||
tags: [
|
||||
"osm",
|
||||
"openstreetmap",
|
||||
"imported"
|
||||
],
|
||||
title: event.title,
|
||||
@ -308,7 +309,7 @@ class utils {
|
||||
showRemainingAttendeeCapacity: false,
|
||||
showStartTime: true
|
||||
},
|
||||
organizerActorId: "3",
|
||||
organizerActorId: organizerActorId,
|
||||
phoneAddress: "",
|
||||
status: "CONFIRMED",
|
||||
tags: [
|
||||
@ -324,29 +325,64 @@ class utils {
|
||||
|
||||
return this.agendadulibre.queryToAdd;
|
||||
},
|
||||
addQueryBDD: (event: any) => {
|
||||
if (this.agendadulibre.queryToAdd) {
|
||||
this.agendadulibre.queryToAdd += ` , `;
|
||||
addNewEventsFromQueries:()=>{
|
||||
|
||||
if (parserConfig.runAddQueriesToMobilizonAPI) {
|
||||
console.log(
|
||||
'➕ rajouter les évènements manquants par l\'API GraphQL',
|
||||
this.agendadulibre.queryToAdd.length
|
||||
);
|
||||
let limiter = parserConfig.limit_persistence_of_new_events;
|
||||
let counter = 0;
|
||||
let bearerTokenIsOK = true;
|
||||
|
||||
this.agendadulibre.queryToAdd.forEach((graphQLquery: any) => {
|
||||
|
||||
if (limiter && counter <= parserConfig.max_new_events && bearerTokenIsOK) {
|
||||
counter++;
|
||||
console.log(counter, ' * ', graphQLquery.variables.title);
|
||||
|
||||
const body = {
|
||||
operationName: "createEvent",
|
||||
query: graphQLquery.query,
|
||||
variables: graphQLquery.variables
|
||||
}
|
||||
|
||||
if (parserConfig.enableFetch) {
|
||||
let theUrl :string = parserConfig.dev_mode ? parserConfig.dev_url : parserConfig.mobilizon_public_url;
|
||||
let theOptions:any = {
|
||||
method: "post",
|
||||
body: JSON.stringify(body),
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
"authorization": "Bearer " + parserConfig.bearer_token,
|
||||
}
|
||||
}
|
||||
proxy_fetch(theUrl, theOptions)
|
||||
.then((res: any) => {
|
||||
let status = res.status;
|
||||
console.log('status', status);
|
||||
if (status === 401) {
|
||||
console.error(' /!\\ ------------------ ERROR: Bearer token invalid ------------------')
|
||||
bearerTokenIsOK = false;
|
||||
} else if (status === 200) {
|
||||
console.log('succès');
|
||||
|
||||
}
|
||||
res.json()
|
||||
})
|
||||
.then((json: any) => console.log(json))
|
||||
.catch((err: any) => console.log(err))
|
||||
} else {
|
||||
console.log('---- le fetch est désactivé');
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
});
|
||||
}
|
||||
|
||||
let title = "'" + htmlEscape(event.title) + "'";
|
||||
let content = "'" + htmlEscape(event.content) + "'";
|
||||
let uuid = uuidv4();
|
||||
let uuidString = "'" + uuid + "'";
|
||||
let eventUrl =
|
||||
"'" + parserConfig.mobilizon_public_url + "/events/" + uuid + "'";
|
||||
let begins_on = "'" + this.convertRssDate(event.date) + "'";
|
||||
let ends_on = "'" + this.convertRssDate(event.date) + "'";
|
||||
let baseOptions =
|
||||
'{"offers": [], "program": null, "attendees": [], "show_end_time": true, "show_start_time": true, "comment_moderation": "allow_all", "anonymous_participation": true, "participation_condition": [], "show_participation_price": false, "maximum_attendee_capacity": 0, "remaining_attendee_capacity": 0, "hide_organizer_when_group_event": false, "show_remaining_attendee_capacity": false}';
|
||||
let baseStats =
|
||||
'{"creator": 1, "rejected": 0, "moderator": 0, "participant": 0, "not_approved": 0, "administrator": 0, "not_confirmed": 0}';
|
||||
// begins_on , ends_on expecting date format like this: "2020-12-17 23:00:00"
|
||||
|
||||
this.agendadulibre.queryToAdd += `( ${title}, ${content}, ${parserConfig.feeder_mobilizon_user_id}, 'now()','now()', ${uuidString}, ${eventUrl}, 'confirmed' , 'meeting', ${baseOptions}, ${baseStats}, ${begins_on} , ${ends_on} )`;
|
||||
|
||||
this.agendadulibre.counterOfEventsToAdd++;
|
||||
this.counterOfEventsToAdd++;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user