refacto in utils
This commit is contained in:
parent
86aa7bc3ae
commit
b5f108c571
66
index.js
66
index.js
@ -2,10 +2,8 @@
|
|||||||
// imports
|
// imports
|
||||||
import utils from "./utils.ts";
|
import utils from "./utils.ts";
|
||||||
import parserConfig from "./config.ts";
|
import parserConfig from "./config.ts";
|
||||||
import { v4 as uuidv4 } from "uuid";
|
|
||||||
import { htmlEscape } from "escape-goat";
|
import { htmlEscape } from "escape-goat";
|
||||||
import Parser from "rss-parser";
|
import Parser from "rss-parser";
|
||||||
import { Client } from "pg";
|
|
||||||
const fetch = require("node-fetch");
|
const fetch = require("node-fetch");
|
||||||
let parser = new Parser();
|
let parser = new Parser();
|
||||||
|
|
||||||
@ -37,65 +35,20 @@ const doesThisEventAlreadyExistInLocalEvents = (rssEvent) => {
|
|||||||
|
|
||||||
let createEventQueries = "";
|
let createEventQueries = "";
|
||||||
let counterOfEventsToAdd = 0;
|
let counterOfEventsToAdd = 0;
|
||||||
const addEventQuery = (rssEvent) => {
|
const addEventQuery = utilsTools.agendadulibre.addQuery;
|
||||||
if (!createEventQueries) {
|
const runCreationQuery = utilsTools.runCreationQuery;
|
||||||
createEventQueries =
|
|
||||||
"INSERT INTO events(title, description, organizer_actor_id,inserted_at,updated_at, uuid, url, status, category, options,participants_stats, begins_on, ends_on) VALUES ";
|
|
||||||
}
|
|
||||||
if (counterOfEventsToAdd) {
|
|
||||||
createEventQueries += ` , `;
|
|
||||||
}
|
|
||||||
|
|
||||||
let title = "'" + htmlEscape(rssEvent.title) + "'";
|
|
||||||
let content = "'" + htmlEscape(rssEvent.content) + "'";
|
|
||||||
let uuid = uuidv4();
|
|
||||||
let uuidString = "'" + uuid + "'";
|
|
||||||
let eventUrl =
|
|
||||||
"'" + parserConfig.mobilizon_public_url + "/events/" + uuid + "'";
|
|
||||||
let begins_on = "'" + utilsTools.convertRssDate(rssEvent.date) + "'";
|
|
||||||
let ends_on = "'" + utilsTools.convertRssDate(rssEvent.date) + "'";
|
|
||||||
let baseOptions =
|
|
||||||
'{"offers": [], "program": null, "attendees": [], "show_end_time": true, "show_start_time": true, "comment_moderation": "allow_all", "anonymous_participation": true, "participation_condition": [], "show_participation_price": false, "maximum_attendee_capacity": 0, "remaining_attendee_capacity": 0, "hide_organizer_when_group_event": false, "show_remaining_attendee_capacity": false}';
|
|
||||||
let baseStats =
|
|
||||||
'{"creator": 1, "rejected": 0, "moderator": 0, "participant": 0, "not_approved": 0, "administrator": 0, "not_confirmed": 0}';
|
|
||||||
// TODO complete with date conversion
|
|
||||||
// begins_on , ends_on expecting date format like this: "2020-12-17 23:00:00"
|
|
||||||
|
|
||||||
createEventQueries += `( ${title}, ${content}, ${parserConfig.feeder_mobilizon_user_id}, 'now()','now()', ${uuidString}, ${eventUrl}, 'confirmed' , 'meeting', ${baseOptions}, ${baseStats}, ${begins_on} , ${ends_on} )`;
|
|
||||||
|
|
||||||
counterOfEventsToAdd++;
|
|
||||||
};
|
|
||||||
const runCreationQuery = async () => {
|
|
||||||
createEventQueries = createEventQueries + ";";
|
|
||||||
utilsTools.writeFile("event_creation_query.psql", createEventQueries);
|
|
||||||
|
|
||||||
if (createEventQueries) {
|
|
||||||
console.log(" ");
|
|
||||||
console.log(" ⚙️⚙️⚙️ ");
|
|
||||||
console.log(" ");
|
|
||||||
console.log(" createEventQueries");
|
|
||||||
console.log(createEventQueries);
|
|
||||||
|
|
||||||
const res = await client.query(createEventQueries);
|
|
||||||
console.log("res", res);
|
|
||||||
return res;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const client = new Client({
|
|
||||||
host: "localhost",
|
|
||||||
user: parserConfig.db_user,
|
|
||||||
password: parserConfig.db_pass,
|
|
||||||
database: parserConfig.db_name,
|
|
||||||
});
|
|
||||||
|
|
||||||
(async () => {
|
(async () => {
|
||||||
console.log("⌛ interroger la BDD mobilizon postgresql");
|
console.log("⌛ interroger la BDD mobilizon postgresql");
|
||||||
|
|
||||||
await client.connect();
|
utilsTools.setupClientPostgresql()
|
||||||
|
|
||||||
|
await utilsTools.client.connect();
|
||||||
console.log("✅ OK connecté à postgresql");
|
console.log("✅ OK connecté à postgresql");
|
||||||
console.log(" ");
|
console.log(" ");
|
||||||
const res = await client.query("SELECT * from events");
|
const res = await utilsTools.client.query("SELECT * from events");
|
||||||
console.log("💾 évènements enregistrés dans mobilizon : ", res.rows.length); // Hello world!
|
console.log("💾 évènements enregistrés dans mobilizon : ", res.rows.length); // Hello world!
|
||||||
|
|
||||||
utilsTools.postgresEventsExisting = res.rows;
|
utilsTools.postgresEventsExisting = res.rows;
|
||||||
@ -142,7 +95,7 @@ const client = new Client({
|
|||||||
arrayOfEvents.forEach((eventFound) => {
|
arrayOfEvents.forEach((eventFound) => {
|
||||||
console.log(eventFound.name);
|
console.log(eventFound.name);
|
||||||
// console.log('item', item)
|
// console.log('item', item)
|
||||||
utilsTools.osmCal.doesEventExists(eventFound);
|
utilsTools.osmcal.doesEventExists(eventFound);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@ -153,7 +106,8 @@ const client = new Client({
|
|||||||
"➕ rajouter les évènements manquants à la base mobilizon".blue
|
"➕ rajouter les évènements manquants à la base mobilizon".blue
|
||||||
);
|
);
|
||||||
|
|
||||||
await runCreationQuery();
|
utilsTools.makeQuery();
|
||||||
|
utilsTools.runCreationQuery();
|
||||||
// @ts-ignore
|
// @ts-ignore
|
||||||
console.log(" ---------- ".green);
|
console.log(" ---------- ".green);
|
||||||
// @ts-ignore
|
// @ts-ignore
|
||||||
@ -163,7 +117,7 @@ const client = new Client({
|
|||||||
// @ts-ignore
|
// @ts-ignore
|
||||||
console.log("✅ ça c'est fait ".green);
|
console.log("✅ ça c'est fait ".green);
|
||||||
}
|
}
|
||||||
await client.end();
|
await utilsTools.client.end();
|
||||||
})();
|
})();
|
||||||
|
|
||||||
exports = () => {
|
exports = () => {
|
||||||
|
File diff suppressed because one or more lines are too long
149
utils.ts
149
utils.ts
@ -1,50 +1,121 @@
|
|||||||
|
// @ts-ignore
|
||||||
|
import parserConfig from "./config.ts";
|
||||||
|
import {v4 as uuidv4} from "uuid";
|
||||||
|
import {Client} from "pg";
|
||||||
|
import {htmlEscape} from "escape-goat";
|
||||||
|
|
||||||
const moment = require("moment");
|
const moment = require("moment");
|
||||||
|
|
||||||
const fs = require("fs");
|
const fs = require("fs");
|
||||||
|
|
||||||
class utils {
|
class utils {
|
||||||
convertRssDate(rssDate) {
|
client;
|
||||||
let converted = moment(rssDate)
|
makeQuery = () => {
|
||||||
.format("YYYY-MM-DD LTS")
|
this.createEventQueries = `INSERT INTO events(title, description, organizer_actor_id,inserted_at,updated_at, uuid, url, status, category, options,participants_stats, begins_on, ends_on) VALUES ${this.agendadulibre.queryToAdd} ${this.osmcal.queryToAdd};`;
|
||||||
.slice(0, -3)
|
this.writeFile("event_creation_query.psql", this.createEventQueries);
|
||||||
.concat(".000000"); // in js format like 2021-03-12T19:00:00Z
|
}
|
||||||
|
runCreationQuery = async () => {
|
||||||
|
if (this.createEventQueries) {
|
||||||
|
console.log(" ");
|
||||||
|
console.log(" ⚙️⚙️⚙️ ");
|
||||||
|
console.log(" ");
|
||||||
|
console.log(" createEventQueries");
|
||||||
|
console.log(this.createEventQueries);
|
||||||
|
|
||||||
console.log("converted", converted);
|
const res = await this.client.query(this.createEventQueries);
|
||||||
// like 2021-01-03 15:31:02.918940
|
console.log("res", res);
|
||||||
return converted;
|
return res;
|
||||||
}
|
|
||||||
|
|
||||||
testdateconvert() {
|
|
||||||
let converted = this.convertRssDate("2021-03-12T19:00:00Z");
|
|
||||||
console.log("converted", converted);
|
|
||||||
}
|
|
||||||
|
|
||||||
postgresEventsExisting = [];
|
|
||||||
writeFile = (fileName, data) => {
|
|
||||||
// write file to disk
|
|
||||||
fs.writeFile(
|
|
||||||
`./sources_examples/${fileName}`,
|
|
||||||
JSON.stringify(data),
|
|
||||||
"utf8",
|
|
||||||
(err) => {
|
|
||||||
if (err) {
|
|
||||||
console.log(`Error writing file: ${err}`);
|
|
||||||
} else {
|
} else {
|
||||||
console.log(`File ${fileName} is written successfully!`);
|
console.log(" DISABLED createEventQueries");
|
||||||
}
|
}
|
||||||
}
|
};
|
||||||
);
|
createEventQueries = "";
|
||||||
};
|
counterOfEventsToAdd = 0;
|
||||||
osmCal = {
|
|
||||||
queryToAdd: "",
|
|
||||||
doesEventExists: (event) => {},
|
convertRssDate(rssDate) {
|
||||||
addQuery: (event) => {},
|
let converted = moment(rssDate)
|
||||||
};
|
.format("YYYY-MM-DD LTS")
|
||||||
agendadulibre = {
|
.slice(0, -3)
|
||||||
queryToAdd: "",
|
.concat(".000000"); // in js format like 2021-03-12T19:00:00Z
|
||||||
doesEventExists: (event) => {},
|
|
||||||
addQuery: (event) => {},
|
console.log("converted", converted);
|
||||||
};
|
// like 2021-01-03 15:31:02.918940
|
||||||
|
return converted;
|
||||||
|
}
|
||||||
|
|
||||||
|
testdateconvert() {
|
||||||
|
let converted = this.convertRssDate("2021-03-12T19:00:00Z");
|
||||||
|
console.log("converted", converted);
|
||||||
|
}
|
||||||
|
|
||||||
|
postgresEventsExisting = [];
|
||||||
|
writeFile = (fileName, data) => {
|
||||||
|
// write file to disk
|
||||||
|
fs.writeFile(
|
||||||
|
`./sources_examples/${fileName}`,
|
||||||
|
JSON.stringify(data),
|
||||||
|
"utf8",
|
||||||
|
(err) => {
|
||||||
|
if (err) {
|
||||||
|
console.log(`Error writing file: ${err}`);
|
||||||
|
} else {
|
||||||
|
console.log(`File ${fileName} is written successfully!`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
);
|
||||||
|
};
|
||||||
|
osmcal = {
|
||||||
|
queryToAdd: "",
|
||||||
|
counterOfEventsToAdd: 0,
|
||||||
|
doesEventExists: (event) => {
|
||||||
|
},
|
||||||
|
addQuery: (event) => {
|
||||||
|
|
||||||
|
this.agendadulibre.counterOfEventsToAdd++;
|
||||||
|
this.counterOfEventsToAdd++;
|
||||||
|
|
||||||
|
},
|
||||||
|
};
|
||||||
|
agendadulibre = {
|
||||||
|
queryToAdd: "",
|
||||||
|
counterOfEventsToAdd: 0,
|
||||||
|
doesEventExists: (event) => {
|
||||||
|
},
|
||||||
|
addQuery: (rssEvent) => {
|
||||||
|
if (this.agendadulibre.queryToAdd) {
|
||||||
|
this.agendadulibre.queryToAdd += ` , `;
|
||||||
|
}
|
||||||
|
|
||||||
|
let title = "'" + htmlEscape(rssEvent.title) + "'";
|
||||||
|
let content = "'" + htmlEscape(rssEvent.content) + "'";
|
||||||
|
let uuid = uuidv4();
|
||||||
|
let uuidString = "'" + uuid + "'";
|
||||||
|
let eventUrl =
|
||||||
|
"'" + parserConfig.mobilizon_public_url + "/events/" + uuid + "'";
|
||||||
|
let begins_on = "'" + this.convertRssDate(rssEvent.date) + "'";
|
||||||
|
let ends_on = "'" + this.convertRssDate(rssEvent.date) + "'";
|
||||||
|
let baseOptions =
|
||||||
|
'{"offers": [], "program": null, "attendees": [], "show_end_time": true, "show_start_time": true, "comment_moderation": "allow_all", "anonymous_participation": true, "participation_condition": [], "show_participation_price": false, "maximum_attendee_capacity": 0, "remaining_attendee_capacity": 0, "hide_organizer_when_group_event": false, "show_remaining_attendee_capacity": false}';
|
||||||
|
let baseStats =
|
||||||
|
'{"creator": 1, "rejected": 0, "moderator": 0, "participant": 0, "not_approved": 0, "administrator": 0, "not_confirmed": 0}';
|
||||||
|
// begins_on , ends_on expecting date format like this: "2020-12-17 23:00:00"
|
||||||
|
|
||||||
|
this.agendadulibre.queryToAdd += `( ${title}, ${content}, ${parserConfig.feeder_mobilizon_user_id}, 'now()','now()', ${uuidString}, ${eventUrl}, 'confirmed' , 'meeting', ${baseOptions}, ${baseStats}, ${begins_on} , ${ends_on} )`;
|
||||||
|
|
||||||
|
this.agendadulibre.counterOfEventsToAdd++;
|
||||||
|
this.counterOfEventsToAdd++;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
setupClientPostgresql = () => {
|
||||||
|
this.client = new Client({
|
||||||
|
host: "localhost",
|
||||||
|
user: parserConfig.db_user,
|
||||||
|
password: parserConfig.db_pass,
|
||||||
|
database: parserConfig.db_name,
|
||||||
|
});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export default utils;
|
export default utils;
|
||||||
|
Loading…
Reference in New Issue
Block a user