import * as fs from "fs"; import utils from "../utils"; import parserConfig from "../config"; import fetch from 'node-fetch'; let utilsTools = new utils(); console.log('importation depuis le fichier local de l\' agenda du libre'); let filepath = './output/adl_json.json' let filecontent; let counterOfEventsToAdd = 0; const addEventQuery = utilsTools.agendadulibre.addQuery; const runCreationQuery = utilsTools.runCreationQuery; async function runImportEvents() { console.log('File exists'); // build list of existing events in mobilizon database await utilsTools.setupClientPostgresql(); await utilsTools.client.connect(); console.log('✅ OK connecté à postgresql'); console.log(' '); const res = await utilsTools.client.query('SELECT * from events'); console.log('💾 évènements enregistrés dans mobilizon : ', res.rows.length); res.rows.forEach((bdd_event: any) => { if (utilsTools.localMobilizonEventsByTitle.indexOf(utilsTools.agendadulibre.uniqTitleBDD(bdd_event)) === -1) { utilsTools.localMobilizonEventsByTitle.push(utilsTools.agendadulibre.uniqTitleBDD(bdd_event)); } }) console.log('utilsTools.localMobilizonEventsByTitle', utilsTools.localMobilizonEventsByTitle.length); // get json file for ADL await fs.readFile(filepath, 'utf8', function (err, data) { if (err) { return console.log(err); } filecontent = JSON.parse(data) filecontent = filecontent.slice(0,parserConfig.max_new_events_in_scrapping) console.log('events in the scrapped json', filecontent.length); let ii = 0; filecontent.forEach((event: any) => { ii++ // compare events with existing events let eventAlreadyExists = utilsTools.agendadulibre.doesEventExistsFromJsonScrap(event); if (!eventAlreadyExists) { if (parserConfig.debug) { console.log('ajouter l event ', utilsTools.agendadulibre.uniqTitle(event)); } let newQuery = utilsTools.agendadulibre.addQueryFromJsonScrap(event); utilsTools.newEvents.push({ event, newQuery }); } }) // console.log('utilsTools.localMobilizonEventsByTitle', utilsTools.localMobilizonEventsByTitle); console.log('end looking for events'); // import only new events console.log('nouveaux évènements à ajouter: ', utilsTools.newEvents.length); let limiter = parserConfig.limit_persistence_of_new_events; let counter = 0; console.log('utilsTools.localMobilizonEventsByTitle', utilsTools.localMobilizonEventsByTitle); utilsTools.newEvents.forEach((pair: any) => { let graphQLquery = pair.newQuery; if (limiter && counter < parserConfig.max_new_events) { counter++; console.log(counter, ' * ', utilsTools.agendadulibre.uniqTitle(pair.event)); const body = { operationName: "createEvent", query: graphQLquery.query, variables: graphQLquery.variables } if (parserConfig.enableFetch) { let url: string = parserConfig.dev_mode ? parserConfig.dev_url : parserConfig.mobilizon_public_url; let options: any = { method: "post", body: JSON.stringify(body), headers: { "Content-Type": "application/json", "authorization": "Bearer " + parserConfig.bearer_token, } } if(!utilsTools.agendadulibre.doesEventExistsFromJsonScrap(pair.event)){ console.log('ajouter'); fetchEvent(url, options) }else{ console.log('nope'); } } else { console.log('---- le fetch est désactivé'); } } }) // end running queries }); } const fetchEvent = (theUrl, theOptions) => { fetch(theUrl, theOptions) .then((res: any) => { let status = res.status; console.log('status', status); if (status === 401) { console.error(' /!\\ ------------------ ERROR: Bearer token invalid ------------------') } else if (status === 200) { console.log('succès'); } res.json() }) .then((json: any) => console.log(json)) .catch((err: any) => console.log(err)) } // fs.stat(filepath, function (err, stat) { // if (err == null) { runImportEvents(); // } else if (err.code === 'ENOENT') { // // file does not exist // console.log('Scrapped json file does not exist. Run a scraper like "ts-node scrapers/adl.ts" before using this importer: ', err.code); // } else { // console.log('Some other error: ', err.code); // } // });