// @ts-ignore import parserConfig from "./config.ts"; import {v4 as uuidv4} from "uuid"; import {Client} from "pg"; import {htmlEscape} from "escape-goat"; const moment = require("moment"); const fs = require("fs"); let createEventQueryMobilizon = "mutation createEvent($organizerActorId: ID!, $attributedToId: ID, $title: String!, $description: String!, $beginsOn: DateTime!, $endsOn: DateTime, $status: EventStatus, $visibility: EventVisibility, $joinOptions: EventJoinOptions, $draft: Boolean, $tags: [String], $picture: MediaInput, $onlineAddress: String, $phoneAddress: String, $category: String, $physicalAddress: AddressInput, $options: EventOptionsInput, $contacts: [Contact]) {\n createEvent(\n organizerActorId: $organizerActorId\n attributedToId: $attributedToId\n title: $title\n description: $description\n beginsOn: $beginsOn\n endsOn: $endsOn\n status: $status\n visibility: $visibility\n joinOptions: $joinOptions\n draft: $draft\n tags: $tags\n picture: $picture\n onlineAddress: $onlineAddress\n phoneAddress: $phoneAddress\n category: $category\n physicalAddress: $physicalAddress\n options: $options\n contacts: $contacts\n ) {\n ...FullEvent\n __typename\n }\n}\n\nfragment FullEvent on Event {\n id\n uuid\n url\n local\n title\n description\n beginsOn\n endsOn\n status\n visibility\n joinOptions\n draft\n picture {\n id\n url\n name\n metadata {\n width\n height\n blurhash\n __typename\n }\n __typename\n }\n publishAt\n onlineAddress\n phoneAddress\n physicalAddress {\n ...AdressFragment\n __typename\n }\n organizerActor {\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n domain\n name\n url\n id\n summary\n __typename\n }\n contacts {\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n name\n summary\n domain\n url\n id\n __typename\n }\n attributedTo {\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n name\n summary\n domain\n url\n id\n __typename\n }\n participantStats {\n going\n notApproved\n participant\n __typename\n }\n tags {\n ...TagFragment\n __typename\n }\n relatedEvents {\n id\n uuid\n title\n beginsOn\n picture {\n id\n url\n name\n metadata {\n width\n height\n blurhash\n __typename\n }\n __typename\n }\n physicalAddress {\n id\n description\n __typename\n }\n organizerActor {\n id\n avatar {\n id\n url\n __typename\n }\n preferredUsername\n domain\n name\n __typename\n }\n __typename\n }\n options {\n ...EventOptions\n __typename\n }\n metadata {\n key\n title\n value\n type\n __typename\n }\n __typename\n}\n\nfragment AdressFragment on Address {\n id\n description\n geom\n street\n locality\n postalCode\n region\n country\n type\n url\n originId\n __typename\n}\n\nfragment TagFragment on Tag {\n id\n slug\n title\n __typename\n}\n\nfragment EventOptions on EventOptions {\n maximumAttendeeCapacity\n remainingAttendeeCapacity\n showRemainingAttendeeCapacity\n anonymousParticipation\n showStartTime\n showEndTime\n offers {\n price\n priceCurrency\n url\n __typename\n }\n participationConditions {\n title\n content\n url\n __typename\n }\n attendees\n program\n commentModeration\n showParticipationPrice\n hideOrganizerWhenGroupEvent\n __typename\n}\n"; /** * utilitaries to manipulate scraped object and prepare queries to import in mobilizon */ class utils { /** * postgres functions */ client: any; createEventQueries: string; makeQuery = () => { let createEventQueries: string = `INSERT INTO events(title, description, organizer_actor_id, inserted_at, updated_at, uuid, url, status, category, options, participants_stats, begins_on, ends_on) VALUES ${this.agendadulibre.queryToAdd} ${this.osmcal.queryToAdd};`; this.createEventQueries = createEventQueries; this.writeFile("event_creation_query.psql", this.createEventQueries, "psql"); } runCreationQuery = async () => { if (this.createEventQueries) { console.log(" "); console.log(" ⚙️⚙️⚙️ "); console.log(" "); console.log(" createEventQueries"); console.log(this.createEventQueries); const res = await this.client.query(this.createEventQueries); console.log("res", res); return res; } else { console.log(" DISABLED createEventQueries"); } }; /** * memorizing properties */ counterOfEventsToAdd = 0; localMobilizonEventsByTitle: Array = []; /** * converters */ geocoderNominatim(coords: any) { // https://nominatim.openstreetmap.org/reverse?lat=&lon=& console.log('https://nominatim.openstreetmap.org/reverse?lat=' + coords[0] + '&lon=' + coords[0]) + '&format=json' } convertRssDateBDD(rssDate: any) { let converted = moment(rssDate) .format("YYYY-MM-DD LTS") .slice(0, -3) .concat(".000000"); // in js format like 2021-03-12T19:00:00Z console.log("converted", converted); // like 2021-01-03 15:31:02.918940 return converted; } convertRssDate(rssDate: string): string { console.log('rssDate', rssDate); if (rssDate) { let converted = new Date(rssDate).toISOString(); // in js format like 2021-03-12T19:00:00Z console.log("converted", converted); // like 2021-01-03 15:31:02.918940 return converted; } else { return ""; } } convertCoordinateLinkOsmCal(coords: any) { this.geocoderNominatim(coords); return `
voir le lieu sur une carte
` } testdateconvert() { let converted = this.convertRssDate("2021-03-12T19:00:00Z"); } /** * file management */ writeFile(fileName: string, data: any, formatData: any = 'json') { let dataToSave = data; if (formatData == 'json') { dataToSave = JSON.stringify(data, null, 4) } // write file to disk fs.writeFile( `./output/${fileName}`, dataToSave, "utf8", (err: any) => { if (err) { console.log(`Error writing file: ${err}`); } else { console.log(`File ${fileName} is written successfully!`); } } ); }; /** ============================== * importation sources */ public osmcal = { queryToAdd: "", counterOfEventsToAdd: 0, getTitle: (event: any) => { return event.name; //+ ' '+ event.location.short }, doesEventExists: (event: any): boolean => { const eventAlreadyExists = -1 !== this.localMobilizonEventsByTitle.indexOf(this.osmcal.getTitle(event)); if (!eventAlreadyExists) { if (parserConfig.debug) { console.log('ajouter l event ', htmlEscape(this.osmcal.getTitle(event))); } this.osmcal.addQuery(event); } return eventAlreadyExists; }, addQuery: (event: any) => { if (this.osmcal.queryToAdd) { this.osmcal.queryToAdd += ` , `; } let title = "'" + htmlEscape(this.osmcal.getTitle(event)) + "'"; let content = "'" + htmlEscape(`${event.date.human}
${+event.url}
`) + "'"; if (event.location) { content += `${event.location.detailed} ` if (event.location.venue) { content += `
${event.location.venue}` } if (event.location.coords) { content += `
` + this.convertCoordinateLinkOsmCal(event.location.coords) } } console.log(' ') console.log(' title', title) let uuid = uuidv4(); let uuidString = "'" + uuid + "'"; let eventUrl = "'" + parserConfig.mobilizon_public_url + "/events/" + uuid + "'"; let begins_on = "'" + this.convertRssDate(event.date.start) + "'"; let ends_on = "'" + this.convertRssDate(event.date.end) + "'"; let baseOptions = '{"offers": [], "program": null, "attendees": [], "show_end_time": true, "show_start_time": true, "comment_moderation": "allow_all", "anonymous_participation": true, "participation_condition": [], "show_participation_price": false, "maximum_attendee_capacity": 0, "remaining_attendee_capacity": 0, "hide_organizer_when_group_event": false, "show_remaining_attendee_capacity": false}'; let baseStats = '{"creator": 1, "rejected": 0, "moderator": 0, "participant": 0, "not_approved": 0, "administrator": 0, "not_confirmed": 0}'; // begins_on , ends_on expecting date format like this: "2020-12-17 23:00:00" this.osmcal.queryToAdd += `( ${title}, ${content}, ${parserConfig.feeder_mobilizon_user_id}, 'now()','now()', ${uuidString}, ${eventUrl}, 'confirmed' , 'meeting', ${baseOptions}, ${baseStats}, ${begins_on} , ${ends_on} )`; this.osmcal.counterOfEventsToAdd++; this.counterOfEventsToAdd++; }, }; public agendadulibre: any = { queryToAdd: [], queryToAddBDD: "", counterOfEventsToAdd: 0, doesEventExists: (event: any) => { if (this.localMobilizonEventsByTitle.length) { const eventAlreadyExists = -1 !== this.localMobilizonEventsByTitle.indexOf(this.agendadulibre.uniqTitle(event)); if (!eventAlreadyExists) { if (parserConfig.debug) { console.log('ajouter l event ', htmlEscape(this.agendadulibre.uniqTitle(event))); } this.agendadulibre.addQuery(event); } return eventAlreadyExists; } else { console.log('aucun évènement dans localMobilizonEventsByTitle'); } return false; }, /** * convert events from data scraping of the agenda du libre, to a string used as a comparison with new events * @param event */ uniqTitle(event: any): string { return event.start_date + ' ' + event.title }, /** * convert events from bdd to a string used as a comparison with new events * @param event */ uniqTitleBDD(event: any): string { return event.begins_on + ' ' + event.title }, doesEventExistsFromJsonScrap: (event: any): boolean => { const eventAlreadyExists = -1 !== this.localMobilizonEventsByTitle.indexOf(htmlEscape(event.start_date + ' ' + event.title)); if (!eventAlreadyExists) { if (parserConfig.debug) { console.log('ajouter l event ', htmlEscape(event.start_date + ' ' + event.title)); } this.agendadulibre.addQuery(event); } return eventAlreadyExists; }, addQueryFromJsonScrap: (event: any) => { console.log('event', event.title); this.agendadulibre.queryToAdd.push( { operationName: "createEvent", query: createEventQueryMobilizon, variables: { attributedToId: null, beginsOn: event.start_time, contacts: [], description: "

" + event.description + "

", draft: false, endsOn: event.end_time, joinOptions: "FREE", onlineAddress: event.url, options: { anonymousParticipation: true, attendees: [], commentModeration: "ALLOW_ALL", hideOrganizerWhenGroupEvent: false, maximumAttendeeCapacity: 200, offers: [], participationConditions: [], program: "", remainingAttendeeCapacity: 0, showEndTime: true, showParticipationPrice: false, showRemainingAttendeeCapacity: false, showStartTime: true }, organizerActorId: "3", phoneAddress: "", status: "CONFIRMED", tags: [ "osm", "openstreetmap", "imported" ], title: event.title, visibility: "PUBLIC" } } ); return this.agendadulibre.queryToAdd; }, addQuery: (event: any) => { console.log('event', event.title); this.agendadulibre.queryToAdd.push( { operationName: "createEvent", query: createEventQueryMobilizon, variables: { attributedToId: null, beginsOn: event.date, contacts: [], description: "

" + event.content + "

", draft: false, endsOn: event.date, joinOptions: "FREE", onlineAddress: event.link, options: { anonymousParticipation: true, attendees: [], commentModeration: "ALLOW_ALL", hideOrganizerWhenGroupEvent: false, maximumAttendeeCapacity: 200, offers: [], participationConditions: [], program: "", remainingAttendeeCapacity: 0, showEndTime: true, showParticipationPrice: false, showRemainingAttendeeCapacity: false, showStartTime: true }, organizerActorId: "3", phoneAddress: "", status: "CONFIRMED", tags: [ "osm", "openstreetmap", "imported" ], title: event.title, visibility: "PUBLIC" } } ); return this.agendadulibre.queryToAdd; }, addQueryBDD: (event: any) => { if (this.agendadulibre.queryToAdd) { this.agendadulibre.queryToAdd += ` , `; } let title = "'" + htmlEscape(event.title) + "'"; let content = "'" + htmlEscape(event.content) + "'"; let uuid = uuidv4(); let uuidString = "'" + uuid + "'"; let eventUrl = "'" + parserConfig.mobilizon_public_url + "/events/" + uuid + "'"; let begins_on = "'" + this.convertRssDate(event.date) + "'"; let ends_on = "'" + this.convertRssDate(event.date) + "'"; let baseOptions = '{"offers": [], "program": null, "attendees": [], "show_end_time": true, "show_start_time": true, "comment_moderation": "allow_all", "anonymous_participation": true, "participation_condition": [], "show_participation_price": false, "maximum_attendee_capacity": 0, "remaining_attendee_capacity": 0, "hide_organizer_when_group_event": false, "show_remaining_attendee_capacity": false}'; let baseStats = '{"creator": 1, "rejected": 0, "moderator": 0, "participant": 0, "not_approved": 0, "administrator": 0, "not_confirmed": 0}'; // begins_on , ends_on expecting date format like this: "2020-12-17 23:00:00" this.agendadulibre.queryToAdd += `( ${title}, ${content}, ${parserConfig.feeder_mobilizon_user_id}, 'now()','now()', ${uuidString}, ${eventUrl}, 'confirmed' , 'meeting', ${baseOptions}, ${baseStats}, ${begins_on} , ${ends_on} )`; this.agendadulibre.counterOfEventsToAdd++; this.counterOfEventsToAdd++; } }; setupClientPostgresql = () => { this.client = new Client({ host: "localhost", user: parserConfig.db_user, password: parserConfig.db_pass, database: parserConfig.db_name, }); } createEventQueriesForApi(EventsToCreate: any[]) { if (EventsToCreate.length) { console.log('we will create events', EventsToCreate.length); } else { console.log('no events to create'); } } } export default utils;