|
|
|
@ -35,7 +35,7 @@ async function runImportEvents() {
|
|
|
|
|
// get json file for ADL
|
|
|
|
|
await fs.readFile(filepath, 'utf8', function (err, data) { |
|
|
|
|
if (err) { |
|
|
|
|
return console.log(err); |
|
|
|
|
return console.log('readFile ERROR', err); |
|
|
|
|
} |
|
|
|
|
filecontent = JSON.parse(data) |
|
|
|
|
filecontent = filecontent.slice(0, parserConfig.max_new_events_in_scrapping) |
|
|
|
@ -64,8 +64,8 @@ async function runImportEvents() {
|
|
|
|
|
console.log('nouveaux évènements à ajouter: ', utilsTools.newEvents.length); |
|
|
|
|
let limiter = parserConfig.limit_persistence_of_new_events; |
|
|
|
|
let counter = 0; |
|
|
|
|
let counter_max = parserConfig.max_new_events; |
|
|
|
|
|
|
|
|
|
console.log('utilsTools.localMobilizonEventsByTitle', utilsTools.localMobilizonEventsByTitle); |
|
|
|
|
utilsTools.newEvents.forEach((pair: any) => { |
|
|
|
|
let graphQLquery = pair.newQuery; |
|
|
|
|
|
|
|
|
@ -93,7 +93,7 @@ async function runImportEvents() {
|
|
|
|
|
console.log('ajouter'); |
|
|
|
|
// add a little delay between creations
|
|
|
|
|
|
|
|
|
|
fetchEvent(url, options, counter, pair.event) |
|
|
|
|
fetchEvent(url, options, counter, counter_max, pair.event) |
|
|
|
|
} else { |
|
|
|
|
console.log('nope'); |
|
|
|
|
} |
|
|
|
@ -108,43 +108,43 @@ async function runImportEvents() {
|
|
|
|
|
}); |
|
|
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
const fetchEvent = (theUrl, theOptions, counter, event) => { |
|
|
|
|
let timeout = setTimeout( |
|
|
|
|
let timeout; |
|
|
|
|
const fetchEvent = (theUrl, theOptions, counter, counter_max, event) => { |
|
|
|
|
let BearerIsOK = true; |
|
|
|
|
timeout = setTimeout( |
|
|
|
|
function () { |
|
|
|
|
console.log(counter,'/', counter_max,' fetch start - ' + event.title + ' ' + event.start_time); |
|
|
|
|
if(BearerIsOK){ |
|
|
|
|
|
|
|
|
|
fetch(theUrl, theOptions) |
|
|
|
|
.then((res: any) => { |
|
|
|
|
let status = res.status; |
|
|
|
|
console.log('status', status); |
|
|
|
|
console.log(' status', status); |
|
|
|
|
if (status === 401) { |
|
|
|
|
console.error(' /!\\ ------------------ ERROR: Bearer token invalid ------------------') |
|
|
|
|
|
|
|
|
|
clearTimeout(timeout); |
|
|
|
|
|
|
|
|
|
} else if (status === 200) { |
|
|
|
|
console.log('succès - ' + event.title + ' ' + event.start_time); |
|
|
|
|
console.log(' succès - ' + event.title + ' ' + event.start_time); |
|
|
|
|
|
|
|
|
|
} |
|
|
|
|
res.json() |
|
|
|
|
// console.log(' res', res);
|
|
|
|
|
return res.json() |
|
|
|
|
}) |
|
|
|
|
.then((json: any) => console.log(json)) |
|
|
|
|
.catch((err: any) => console.log(err)) |
|
|
|
|
} |
|
|
|
|
.then((json: any) => console.log(counter,'/', counter_max, 'DONE')) |
|
|
|
|
.catch((err: any) => { |
|
|
|
|
console.log(err) |
|
|
|
|
BearerIsOK = false; |
|
|
|
|
}) |
|
|
|
|
|
|
|
|
|
} |
|
|
|
|
}.bind(this) |
|
|
|
|
, |
|
|
|
|
1000 * counter |
|
|
|
|
) |
|
|
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// fs.stat(filepath, function (err, stat) {
|
|
|
|
|
// if (err == null) {
|
|
|
|
|
runImportEvents(); |
|
|
|
|
// } else if (err.code === 'ENOENT') {
|
|
|
|
|
// // file does not exist
|
|
|
|
|
// console.log('Scrapped json file does not exist. Run a scraper like "ts-node scrapers/adl.ts" before using this importer: ', err.code);
|
|
|
|
|
// } else {
|
|
|
|
|
// console.log('Some other error: ', err.code);
|
|
|
|
|
// }
|
|
|
|
|
// });
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|