multi-account-post-schedule.../helpers/wiki_osm.mjs

293 lines
9.9 KiB
JavaScript
Raw Normal View History

2022-08-04 14:16:43 +02:00
// https://www.mediawiki.org/wiki/Manual:Random_page
2022-08-08 16:17:04 +02:00
/**
* Post de page aléatoire du wiki osm avec le compte curator
*/
2022-08-04 14:16:43 +02:00
import fetch from "node-fetch"
import rp from "request-promise";
import $ from "cheerio";
2022-08-04 16:30:18 +02:00
import fs from "fs";
import path from 'path';
2022-08-08 15:47:09 +02:00
import {randomIntFromInterval, sendPostMastodon} from "./utils.js";
2022-08-04 16:30:18 +02:00
const __dirname = path.resolve();
2022-08-08 16:17:04 +02:00
// select one line of table to override description
2022-08-08 16:30:55 +02:00
// const selectionOverrideOfSectionTable = 2;
// const selectionOverrideOfLineTable = 7;
2022-08-08 15:25:09 +02:00
2022-08-08 16:17:04 +02:00
// select randomly a line
2022-08-08 16:30:55 +02:00
const selectionOverrideOfSectionTable = '';
const selectionOverrideOfLineTable = '';
2022-08-08 15:47:09 +02:00
2022-08-08 18:44:45 +02:00
let foundExistingWikiPageInFrench = false;
2022-08-08 18:45:38 +02:00
const reallySendPost = true;
2022-08-04 23:07:59 +02:00
2022-08-04 16:30:18 +02:00
function getRandomWikiOSMPage() {
2022-08-04 14:16:43 +02:00
// get all FR tags
// const url = "https://wiki.openstreetmap.org/w/index.php?title=Special:Categories&offset=FR&limit=500"
const url = "https://wiki.openstreetmap.org/wiki/Special:AllPages?from=&to=&namespace=202&limit=500"
// get a random page wich is not a discussion
// const url = "https://wiki.openstreetmap.org/wiki/Special:Random?namespace=all-discussions%3B1%3B2%3B3%3B5%3B7%3B9%3B11%3B120%3B121%3B122%3B123%3B201%3B203&invert=1&limit=10&days=1&enhanced=1&namespace__202_color=c1&urlversion=2"
rp(url)
2022-08-04 16:30:18 +02:00
.then(function (html) {
2022-08-04 14:16:43 +02:00
//success!
const wikiUrls = [];
const filteredHtml = $('.mw-spcontent ul li a', html)
console.log("filteredHtml.length", filteredHtml.length)
// .filter(elem => {
// return (elem.attribs.title.indexOf('description for') === -1)
// })
for (let i = 0; i < filteredHtml.length; i++) {
// const contentCurated = filteredHtml[i]
const contentCurated = filteredHtml[i].attribs.href
wikiUrls.push(contentCurated);
}
console.log(wikiUrls);
return wikiUrls;
})
2022-08-04 16:30:18 +02:00
.catch(function (err) {
2022-08-04 14:16:43 +02:00
//handle error
console.error(err)
});
}
2022-08-04 16:30:18 +02:00
export default function getElementCartographique() {
if(foundExistingWikiPageInFrench){
console.log("already found foundExistingWikiPageInFrench")
return;
}
2022-08-04 14:16:43 +02:00
const url = "https://wiki.openstreetmap.org/wiki/FR:%C3%89l%C3%A9ments_cartographiques"
const titleLink = "#toc a"
const listElementSelector = "#toc a"
// prendre un lien dans la table des matières de la page, extraire l'id du lien.
// prendre une ligne de tableau au hasard correspondant à l'ID de la section
const titleSelector = ""
2022-08-05 21:30:04 +02:00
return fs.readFile(__dirname + '/assets/documents/elements_carto_osm.html', 'utf8', function (err, html) {
2022-08-04 16:30:18 +02:00
if (err) {
return console.log(err);
}
// console.log(data);
//success!
const wikiUrls = [];
2022-08-08 14:47:40 +02:00
// const listOfLinks = $(titleLink, html)
// console.log("filteredHtml.length", listOfLinks.length)
// const selectedTocLink = getRandomElementOfArray(listOfLinks).attribs.href;
// console.log("selectedTocLink", selectedTocLink)
2022-08-04 16:30:18 +02:00
const listOfTableRows = $('.wikitable', html);
let keys = Object.keys(listOfTableRows);
2022-08-08 14:47:40 +02:00
2022-08-08 15:25:09 +02:00
let randNumber = 0;
2022-08-08 15:47:09 +02:00
if (selectionOverrideOfSectionTable) {
2022-08-08 15:25:09 +02:00
randNumber = selectionOverrideOfSectionTable
2022-08-08 15:47:09 +02:00
} else {
2022-08-08 15:25:09 +02:00
randNumber = randomIntFromInterval(0, keys.length)
2022-08-08 14:47:40 +02:00
}
console.log("rand", randNumber)
const foundLine = $(listOfTableRows[randNumber]).find('tr');
// console.log("randomTable",foundTable)
// on regarde les lignes du tableau
keys = Object.keys(foundLine);
2022-08-08 15:47:09 +02:00
if (selectionOverrideOfLineTable) {
2022-08-08 15:25:09 +02:00
randNumber = selectionOverrideOfLineTable
2022-08-08 15:47:09 +02:00
} else {
2022-08-08 15:25:09 +02:00
randNumber = randomIntFromInterval(0, keys.length)
}
console.log("lignes de tableau", keys.length)
console.log("ligne sélectionnée", randNumber)
let cells = $(foundLine[randNumber]).find('td')
keys = Object.keys(cells);
2022-08-08 15:47:09 +02:00
// console.log("cellules trouvées", keys.length)
2022-08-04 16:30:18 +02:00
2022-08-08 15:47:09 +02:00
console.log("\n")
2022-08-08 18:25:37 +02:00
let configPost = {
key: '',
value: '',
2022-08-08 15:25:09 +02:00
description: '',
long_desc: '',
}
2022-08-04 23:07:59 +02:00
cells.each((i, element) => {
2022-08-08 15:25:09 +02:00
// console.log("cell element", i, $(element).text().trim())
2022-08-04 23:07:59 +02:00
if (i === 0) {
2022-08-08 18:25:37 +02:00
configPost.key = $(element).text().trim();
}
2022-08-04 23:07:59 +02:00
if (i === 1) {
2022-08-08 18:25:37 +02:00
configPost.value = $(element).text().trim();
}
2022-08-04 23:07:59 +02:00
if (i === 3) {
2022-08-08 18:25:37 +02:00
configPost.description = $(element).text().replace(' ', '').replace(' ', '').trim();
}
})
2022-08-08 18:25:37 +02:00
if (!configPost.key) {
2022-08-08 15:47:09 +02:00
console.error("ERROR no key found", $(cells).html())
console.error('no key BOOOOOOOOOOH');
getElementCartographique();
return;
2022-08-05 21:30:04 +02:00
}
2022-08-08 15:47:09 +02:00
2022-08-05 21:30:04 +02:00
// aller chercher le lien
// si la clé est une description avec des espaces, faire un lien vers la Key
2022-08-08 18:25:37 +02:00
let pageLink = `FR:Tag:${configPost.key}=${configPost.value}`
2022-08-05 21:30:04 +02:00
2022-08-08 18:25:37 +02:00
if (configPost.value.indexOf(' ') !== -1) {
pageLink = `FR:Key:${configPost.key}`
2022-08-05 21:30:04 +02:00
}
2022-08-08 18:25:37 +02:00
configPost.link = `https://wiki.openstreetmap.org/wiki/${pageLink}`
2022-08-05 21:30:04 +02:00
2022-08-08 18:25:37 +02:00
const url = configPost.link
2022-08-05 21:30:04 +02:00
// get a random page wich is not a discussion
// certaines valeurs ont des espaces, donc pas utilisable en url
if (configPost.value.indexOf(' ') === -1 || configPost.value.indexOf('<') === -1 || configPost.value.indexOf('>') === -1) {
2022-08-05 21:30:04 +02:00
console.log("url", url)
rp(url).then(function (html) {
//success!
const descriptionStrophe = '';
const filteredHtml = $('.mw-parser-output', html).find('p')
2022-08-08 15:25:09 +02:00
console.log("filteredHtml", filteredHtml.length)
2022-08-08 18:25:37 +02:00
configPost.long_desc = $(filteredHtml).text().substring(0,250)
console.log("filteredHtml", configPost.long_desc)
2022-08-05 21:30:04 +02:00
// image de description:
2022-08-08 15:25:09 +02:00
// let imgSelector = ".description a.image img"
2022-08-08 15:47:09 +02:00
console.log("✅ cette page existe bien en Français sur le wiki OSM")
foundExistingWikiPageInFrench = true;
2022-08-08 18:25:37 +02:00
sendMessageWikiTagOfTheDay(makePostMessageFromObj(configPost))
return;
2022-08-08 15:25:09 +02:00
2022-08-08 15:47:09 +02:00
}, (err) => {
console.error(err.statusCode)
if (err.statusCode == 404) {
2022-08-08 18:25:37 +02:00
console.log("\n Cette page n'existe pas encore, c'est le moment de la créer.", `https://wiki.openstreetmap.org/w/index.php?title=FR:${configPost.key}=${configPost.value}`) //FR:Tag:boundary%3Dborder_zone&action=edit)
2022-08-08 15:25:09 +02:00
console.log("Essayez de voir si la version en Anglais existe.")
2022-08-08 18:25:37 +02:00
let englishPage = `https://wiki.openstreetmap.org/wiki/Tag:${configPost.key}=${configPost.value}`
let englishKey = `https://wiki.openstreetmap.org/wiki/Key:${configPost.key}`
2022-08-08 15:47:09 +02:00
console.log("-> ", englishPage)
checkExistenceOfWebPage(englishPage).catch(err=> {
checkExistenceOfWebPage( englishKey)
});
2022-08-08 16:30:55 +02:00
console.log("bon spa tout ça on essaie une autre ligne")
getElementCartographique()
return;
2022-08-08 15:25:09 +02:00
}
2022-08-08 15:47:09 +02:00
2022-08-05 21:30:04 +02:00
})
} else {
2022-08-08 18:25:37 +02:00
console.log("result", configPost)
rp(configPost.link).then((result) => {
console.log("oui cette page existe!", url)
sendMessageWikiTagOfTheDay(makePostMessageFromObj(result))
}, (err) => {
console.log("hé non. WTF ?")
// getElementCartographique()
})
2022-08-05 21:30:04 +02:00
}
2022-08-04 23:07:59 +02:00
2022-08-08 18:25:37 +02:00
return configPost;
2022-08-04 16:30:18 +02:00
});
2022-08-04 14:16:43 +02:00
}
2022-08-08 15:47:09 +02:00
/**
* check if a page anwsers to a GET request
* @param url
* @returns {*}
*/
function checkExistenceOfWebPage(url) {
if(foundExistingWikiPageInFrench){
console.log("already found foundExistingWikiPageInFrench, no need to check other page", url)
return;
}
2022-08-08 15:47:09 +02:00
return rp(url).then((html) => {
console.log("oui cette page existe!", url)
}, (err) => {
console.log("hé non. WTF ?")
})
}
2022-08-05 15:19:45 +02:00
function makePostMessageFromObj(result) {
return `
2022-08-08 16:30:55 +02:00
# [Le tag OSM du jour : ${result.key}=${result.value} ](${result.link}) 🗺🏷
2022-08-08 18:25:37 +02:00
${result.description ? result.description.trim() : ''}
> ${result.long_desc ? result.long_desc.trim() : ''}
2022-08-05 15:19:45 +02:00
#osm #openstreetmap #wiki #rtfw
`
}
2022-08-04 14:16:43 +02:00
function getQuery() {
// var url = "https://wiki.openstreetmap.org/w/api.php";
var url = "https://fr.wikipedia.org/w/api.php";
var params = {
action: "query",
format: "json",
list: "random",
enhanced: "1",
hiderobot: "1",
urlversion: "2",
invert: "1",
rnnamespace: "all-discussions%3B1%3B2%3B3%3B5%3B7%3B9%3B11%3B120%3B121%3B122%3B123%3B201%3B203", // WIKI et FR:
lang: "FR",
rnlimit: "5"
};
url = url + "?&origin=*";
Object.keys(params).forEach(function (key) {
url += "&" + key + "=" + params[key];
});
return fetch(url)
.then(function (response) {
return response.json();
})
.then(function (response) {
const randoms = response.query.random;
for (let ii = 0; ii < randoms.length; ii++) {
// console.log(randoms[ii].title);
console.log("randoms[ii]", randoms[ii])
}
})
.catch(function (error) {
console.log(error);
});
2022-08-05 15:19:45 +02:00
}
// run
const res = getElementCartographique()
2022-08-08 15:47:09 +02:00
// console.log("res", res)
2022-08-08 15:25:09 +02:00
2022-08-08 15:47:09 +02:00
function sendMessageWikiTagOfTheDay(message) {
2022-08-08 15:25:09 +02:00
let configPost = {
author: 'curator',
2022-08-08 16:17:04 +02:00
image: 'osm_default.jpg',
2022-08-08 15:25:09 +02:00
message,
// reallySendPost: true,
reallySendPost: reallySendPost,
2022-08-08 15:25:09 +02:00
}
sendPostMastodon(configPost)
}