Compare commits

...

2 Commits

Author SHA1 Message Date
Tykayn d09767dd4e add delay in fetching each second 2022-01-13 14:52:27 +01:00
Tykayn 3f44754e94 add tags from ADL 2022-01-13 14:36:12 +01:00
3 changed files with 49 additions and 26 deletions

View File

@ -23,9 +23,9 @@ const parserConfig = {
// enableFetch: false, // enableFetch: false,
dev_mode: true, // dev mode uses localhost instance of mobilizon running on port 4000 dev_mode: true, // dev mode uses localhost instance of mobilizon running on port 4000
limit_persistence_of_new_events:true, limit_persistence_of_new_events:true,
max_new_events_in_scrapping: 10, max_new_events_in_scrapping: 400,
max_new_events: 2, max_new_events: 5,
bearer_token: "eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJtb2JpbGl6b24iLCJleHAiOjE2NDIwODA2NTAsImlhdCI6MTY0MjA3OTc1MCwiaXNzIjoibW9iaWxpem9uIiwianRpIjoiYmZjNWFkNjMtODA2NS00Y2FhLWJiY2EtYzA2MDZkNDEwNDdlIiwibmJmIjoxNjQyMDc5NzQ5LCJzdWIiOiJVc2VyOjEiLCJ0eXAiOiJhY2Nlc3MifQ.Ar5oUTs0cFOEkznYmhXgrFkGRQv4yJm34lN_gkNSrCTrvXfRBTID4YzIrbzvKOkYvIZmw0dHb7gu1WO_tATBkQ", bearer_token: "eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJtb2JpbGl6b24iLCJleHAiOjE2NDIwODIzOTgsImlhdCI6MTY0MjA4MTQ5OCwiaXNzIjoibW9iaWxpem9uIiwianRpIjoiYWQyZjMxY2YtMDUxZi00MmY0LWI1ZDYtMzc4ZjZiOTgxY2EwIiwibmJmIjoxNjQyMDgxNDk3LCJzdWIiOiJVc2VyOjEiLCJ0eXAiOiJhY2Nlc3MifQ.U0JnKQFkklxDcWPE4utXwj65mR0N1LyT3qwC2RzOiWjLWyCS_0KMKJPm5uvylZ-lhiPigojfZsuQCekYL0qTHg",
ccpl: "https://www.cc-paysdelimours.fr/agenda" ccpl: "https://www.cc-paysdelimours.fr/agenda"

View File

@ -38,7 +38,7 @@ async function runImportEvents() {
return console.log(err); return console.log(err);
} }
filecontent = JSON.parse(data) filecontent = JSON.parse(data)
filecontent = filecontent.slice(0,parserConfig.max_new_events_in_scrapping) filecontent = filecontent.slice(0, parserConfig.max_new_events_in_scrapping)
console.log('events in the scrapped json', filecontent.length); console.log('events in the scrapped json', filecontent.length);
let ii = 0; let ii = 0;
@ -89,10 +89,12 @@ async function runImportEvents() {
"authorization": "Bearer " + parserConfig.bearer_token, "authorization": "Bearer " + parserConfig.bearer_token,
} }
} }
if(!utilsTools.agendadulibre.doesEventExistsFromJsonScrap(pair.event)){ if (!utilsTools.agendadulibre.doesEventExistsFromJsonScrap(pair.event)) {
console.log('ajouter'); console.log('ajouter');
fetchEvent(url, options) // add a little delay between creations
}else{
fetchEvent(url, options, counter, pair.event)
} else {
console.log('nope'); console.log('nope');
} }
} else { } else {
@ -107,22 +109,31 @@ async function runImportEvents() {
} }
const fetchEvent = (theUrl, theOptions) => { const fetchEvent = (theUrl, theOptions, counter, event) => {
fetch(theUrl, theOptions) let timeout = setTimeout(
.then((res: any) => { function () {
let status = res.status;
console.log('status', status);
if (status === 401) {
console.error(' /!\\ ------------------ ERROR: Bearer token invalid ------------------')
} else if (status === 200) { fetch(theUrl, theOptions)
console.log('succès'); .then((res: any) => {
let status = res.status;
console.log('status', status);
if (status === 401) {
console.error(' /!\\ ------------------ ERROR: Bearer token invalid ------------------')
clearTimeout(timeout);
} else if (status === 200) {
console.log('succès - ' + event.title + ' ' + event.start_time);
}
res.json()
})
.then((json: any) => console.log(json))
.catch((err: any) => console.log(err))
}
,
1000 * counter
)
}
res.json()
})
.then((json: any) => console.log(json))
.catch((err: any) => console.log(err))
} }
// fs.stat(filepath, function (err, stat) { // fs.stat(filepath, function (err, stat) {

View File

@ -222,21 +222,33 @@ class utils {
.format("YYYY-MM-DD") + ' ' + event.title .format("YYYY-MM-DD") + ' ' + event.title
}, },
doesEventExistsFromJsonScrap: (event: any): boolean => { doesEventExistsFromJsonScrap: (event: any): boolean => {
console.log('this.agendadulibre.uniqTitle(event)', this.agendadulibre.uniqTitle(event));
const eventAlreadyExists = const eventAlreadyExists =
-1 !== this.localMobilizonEventsByTitle.indexOf(this.agendadulibre.uniqTitle(event)); -1 !== this.localMobilizonEventsByTitle.indexOf(this.agendadulibre.uniqTitle(event));
console.log('eventAlreadyExists', eventAlreadyExists);
return eventAlreadyExists; return eventAlreadyExists;
}, },
addQueryFromJsonScrap: (event: any) => { addQueryFromJsonScrap: (event: any) => {
let tags = event.tags.map(element => {
return element.name
})
tags.push("imported")
let newQuery = { let newQuery = {
operationName: "createEvent", operationName: "createEvent",
query: createEventQueryMobilizon, query: createEventQueryMobilizon,
variables: { variables: {
attributedToId: null, attributedToId: null,
beginsOn: event.start_time, beginsOn: event.start_time,
contacts: [], contacts: [event.contact],
description: "<p>" + event.description + "</p>", description:
"<address>" + event.city
+ "<br/>"+
event.address +
"<br/>"+
event.place_name +
"</address>"+
"<p>" + event.description + "</p>",
draft: false, draft: false,
endsOn: event.end_time, endsOn: event.end_time,
joinOptions: "FREE", joinOptions: "FREE",
@ -246,7 +258,7 @@ class utils {
attendees: [], attendees: [],
commentModeration: "ALLOW_ALL", commentModeration: "ALLOW_ALL",
hideOrganizerWhenGroupEvent: false, hideOrganizerWhenGroupEvent: false,
maximumAttendeeCapacity: 200, // maximumAttendeeCapacity: 200,
offers: [], offers: [],
participationConditions: [], participationConditions: [],
program: "", program: "",