2022-01-13 11:18:44 +01:00
|
|
|
import * as fs from "fs";
|
|
|
|
import utils from "../utils";
|
2022-01-13 12:02:51 +01:00
|
|
|
import parserConfig from "../config";
|
|
|
|
import fetch from 'node-fetch';
|
2022-01-13 11:18:44 +01:00
|
|
|
|
|
|
|
let utilsTools = new utils();
|
|
|
|
|
|
|
|
console.log('importation depuis le fichier local de l\' agenda du libre');
|
|
|
|
let filepath = './output/adl_json.json'
|
|
|
|
let filecontent;
|
|
|
|
let counterOfEventsToAdd = 0;
|
|
|
|
const addEventQuery = utilsTools.agendadulibre.addQuery;
|
|
|
|
const runCreationQuery = utilsTools.runCreationQuery;
|
|
|
|
|
|
|
|
async function runImportEvents() {
|
|
|
|
|
|
|
|
|
2022-01-13 12:02:51 +01:00
|
|
|
console.log('File exists');
|
|
|
|
// build list of existing events in mobilizon database
|
2022-01-13 11:18:44 +01:00
|
|
|
|
2022-01-13 12:02:51 +01:00
|
|
|
await utilsTools.setupClientPostgresql();
|
|
|
|
await utilsTools.client.connect();
|
|
|
|
console.log('✅ OK connecté à postgresql');
|
|
|
|
console.log(' ');
|
|
|
|
const res = await utilsTools.client.query('SELECT * from events');
|
|
|
|
console.log('💾 évènements enregistrés dans mobilizon : ', res.rows.length);
|
|
|
|
res.rows.forEach((bdd_event: any) => {
|
2022-01-13 14:30:45 +01:00
|
|
|
if (utilsTools.localMobilizonEventsByTitle.indexOf(utilsTools.agendadulibre.uniqTitleBDD(bdd_event)) === -1) {
|
|
|
|
|
|
|
|
utilsTools.localMobilizonEventsByTitle.push(utilsTools.agendadulibre.uniqTitleBDD(bdd_event));
|
|
|
|
}
|
2022-01-13 12:02:51 +01:00
|
|
|
})
|
2022-01-13 11:18:44 +01:00
|
|
|
|
2022-01-13 14:30:45 +01:00
|
|
|
console.log('utilsTools.localMobilizonEventsByTitle', utilsTools.localMobilizonEventsByTitle.length);
|
2022-01-13 12:02:51 +01:00
|
|
|
// get json file for ADL
|
|
|
|
await fs.readFile(filepath, 'utf8', function (err, data) {
|
|
|
|
if (err) {
|
|
|
|
return console.log(err);
|
|
|
|
}
|
|
|
|
filecontent = JSON.parse(data)
|
2022-01-13 14:30:45 +01:00
|
|
|
filecontent = filecontent.slice(0,parserConfig.max_new_events_in_scrapping)
|
|
|
|
console.log('events in the scrapped json', filecontent.length);
|
|
|
|
|
2022-01-13 12:02:51 +01:00
|
|
|
let ii = 0;
|
|
|
|
filecontent.forEach((event: any) => {
|
|
|
|
ii++
|
|
|
|
// compare events with existing events
|
2022-01-13 14:30:45 +01:00
|
|
|
let eventAlreadyExists = utilsTools.agendadulibre.doesEventExistsFromJsonScrap(event);
|
|
|
|
if (!eventAlreadyExists) {
|
|
|
|
if (parserConfig.debug) {
|
|
|
|
console.log('ajouter l event ', utilsTools.agendadulibre.uniqTitle(event));
|
|
|
|
}
|
|
|
|
let newQuery = utilsTools.agendadulibre.addQueryFromJsonScrap(event);
|
|
|
|
utilsTools.newEvents.push({
|
|
|
|
event,
|
|
|
|
newQuery
|
|
|
|
});
|
|
|
|
}
|
2022-01-13 12:02:51 +01:00
|
|
|
})
|
2022-01-13 14:30:45 +01:00
|
|
|
// console.log('utilsTools.localMobilizonEventsByTitle', utilsTools.localMobilizonEventsByTitle);
|
2022-01-13 12:02:51 +01:00
|
|
|
console.log('end looking for events');
|
|
|
|
|
|
|
|
// import only new events
|
2022-01-13 14:30:45 +01:00
|
|
|
console.log('nouveaux évènements à ajouter: ', utilsTools.newEvents.length);
|
2022-01-13 12:02:51 +01:00
|
|
|
let limiter = parserConfig.limit_persistence_of_new_events;
|
|
|
|
let counter = 0;
|
|
|
|
|
2022-01-13 14:30:45 +01:00
|
|
|
console.log('utilsTools.localMobilizonEventsByTitle', utilsTools.localMobilizonEventsByTitle);
|
|
|
|
utilsTools.newEvents.forEach((pair: any) => {
|
|
|
|
let graphQLquery = pair.newQuery;
|
|
|
|
|
|
|
|
if (limiter && counter < parserConfig.max_new_events) {
|
2022-01-13 12:02:51 +01:00
|
|
|
counter++;
|
2022-01-13 14:30:45 +01:00
|
|
|
console.log(counter, ' * ', utilsTools.agendadulibre.uniqTitle(pair.event));
|
2022-01-13 12:02:51 +01:00
|
|
|
|
|
|
|
const body = {
|
|
|
|
operationName: "createEvent",
|
|
|
|
query: graphQLquery.query,
|
|
|
|
variables: graphQLquery.variables
|
2022-01-13 11:18:44 +01:00
|
|
|
}
|
|
|
|
|
2022-01-13 12:02:51 +01:00
|
|
|
if (parserConfig.enableFetch) {
|
|
|
|
let url: string = parserConfig.dev_mode ? parserConfig.dev_url : parserConfig.mobilizon_public_url;
|
|
|
|
let options: any = {
|
|
|
|
method: "post",
|
|
|
|
body: JSON.stringify(body),
|
|
|
|
headers: {
|
|
|
|
"Content-Type": "application/json",
|
|
|
|
"authorization": "Bearer " + parserConfig.bearer_token,
|
|
|
|
}
|
|
|
|
}
|
2022-01-13 14:30:45 +01:00
|
|
|
if(!utilsTools.agendadulibre.doesEventExistsFromJsonScrap(pair.event)){
|
|
|
|
console.log('ajouter');
|
|
|
|
fetchEvent(url, options)
|
|
|
|
}else{
|
|
|
|
console.log('nope');
|
|
|
|
}
|
2022-01-13 12:02:51 +01:00
|
|
|
} else {
|
|
|
|
console.log('---- le fetch est désactivé');
|
|
|
|
}
|
2022-01-13 11:18:44 +01:00
|
|
|
|
|
|
|
|
2022-01-13 12:02:51 +01:00
|
|
|
}
|
|
|
|
})
|
|
|
|
// end running queries
|
|
|
|
});
|
2022-01-13 11:18:44 +01:00
|
|
|
|
2022-01-13 12:02:51 +01:00
|
|
|
}
|
2022-01-13 11:18:44 +01:00
|
|
|
|
2022-01-13 12:02:51 +01:00
|
|
|
const fetchEvent = (theUrl, theOptions) => {
|
|
|
|
fetch(theUrl, theOptions)
|
|
|
|
.then((res: any) => {
|
|
|
|
let status = res.status;
|
|
|
|
console.log('status', status);
|
|
|
|
if (status === 401) {
|
|
|
|
console.error(' /!\\ ------------------ ERROR: Bearer token invalid ------------------')
|
|
|
|
|
|
|
|
} else if (status === 200) {
|
|
|
|
console.log('succès');
|
|
|
|
|
|
|
|
}
|
|
|
|
res.json()
|
|
|
|
})
|
|
|
|
.then((json: any) => console.log(json))
|
|
|
|
.catch((err: any) => console.log(err))
|
2022-01-13 11:18:44 +01:00
|
|
|
}
|
2022-01-13 12:02:51 +01:00
|
|
|
|
|
|
|
// fs.stat(filepath, function (err, stat) {
|
|
|
|
// if (err == null) {
|
|
|
|
runImportEvents();
|
|
|
|
// } else if (err.code === 'ENOENT') {
|
|
|
|
// // file does not exist
|
|
|
|
// console.log('Scrapped json file does not exist. Run a scraper like "ts-node scrapers/adl.ts" before using this importer: ', err.code);
|
|
|
|
// } else {
|
|
|
|
// console.log('Some other error: ', err.code);
|
|
|
|
// }
|
|
|
|
// });
|
|
|
|
|
|
|
|
|