// https://www.mediawiki.org/wiki/Manual:Random_page /** * Post de page aléatoire du wiki osm avec le compte curator */ import fetch from "node-fetch" import rp from "request-promise"; import $ from "cheerio"; import fs from "fs"; import path from 'path'; import sendPostMastodon, {downloadImage, randomIntFromInterval, slugify} from './utils.mjs' import {getSourceOfDescriptorPageContent} from "./osm_get_description_picture.mjs"; const __dirname = path.resolve(); // select one line of table to override description // const selectionOverrideOfSectionTable = 19; // const selectionOverrideOfLineTable = 7; // select randomly a line const selectionOverrideOfSectionTable = ''; const selectionOverrideOfLineTable = ''; let foundExistingWikiPageInFrench = false; function getRandomWikiOSMPage() { // get all FR tags // const url = "https://wiki.openstreetmap.org/w/index.php?title=Special:Categories&offset=FR&limit=500" const url = "https://wiki.openstreetmap.org/wiki/Special:AllPages?from=&to=&namespace=202&limit=500" // get a random page wich is not a discussion // const url = "https://wiki.openstreetmap.org/wiki/Special:Random?namespace=all-discussions%3B1%3B2%3B3%3B5%3B7%3B9%3B11%3B120%3B121%3B122%3B123%3B201%3B203&invert=1&limit=10&days=1&enhanced=1&namespace__202_color=c1&urlversion=2" rp(url) .then(function (html) { //success! const wikiUrls = []; const filteredHtml = $('.mw-spcontent ul li a', html) console.log("filteredHtml.length", filteredHtml.length) // .filter(elem => { // return (elem.attribs.title.indexOf('description for') === -1) // }) for (let i = 0; i < filteredHtml.length; i++) { // const contentCurated = filteredHtml[i] const contentCurated = filteredHtml[i].attribs.href wikiUrls.push(contentCurated); } console.log(wikiUrls); return wikiUrls; }) .catch(function (err) { //handle error console.error(err) }); } export default function getElementCartographique() { if(foundExistingWikiPageInFrench){ console.log("already found foundExistingWikiPageInFrench") return; } const url = "https://wiki.openstreetmap.org/wiki/FR:%C3%89l%C3%A9ments_cartographiques" const titleLink = "#toc a" const listElementSelector = "#toc a" // prendre un lien dans la table des matières de la page, extraire l'id du lien. // prendre une ligne de tableau au hasard correspondant à l'ID de la section const titleSelector = "" return fs.readFile(__dirname + '/assets/documents/elements_cartographiques.html', 'utf8', function (err, html) { if (err) { return console.log(err); } // console.log(data); //success! const wikiUrls = []; // const listOfLinks = $(titleLink, html) // console.log("filteredHtml.length", listOfLinks.length) // const selectedTocLink = getRandomElementOfArray(listOfLinks).attribs.href; // console.log("selectedTocLink", selectedTocLink) const listOfTableRows = $('.wikitable', html); let keys = Object.keys(listOfTableRows); let randNumber = 0; if (selectionOverrideOfSectionTable) { randNumber = selectionOverrideOfSectionTable } else { randNumber = randomIntFromInterval(0, keys.length) } console.log("rand", randNumber) const foundLine = $(listOfTableRows[randNumber]).find('tr'); // console.log("randomTable",foundTable) // on regarde les lignes du tableau keys = Object.keys(foundLine); if (selectionOverrideOfLineTable) { randNumber = selectionOverrideOfLineTable } else { randNumber = randomIntFromInterval(0, keys.length) } console.log("lignes de tableau", keys.length) console.log("ligne sélectionnée", randNumber) let cells = $(foundLine[randNumber]).find('td') keys = Object.keys(cells); // console.log("cellules trouvées", keys.length) console.log("\n") let configPost = { key: '', value: '', description: '', long_desc: '', } cells.each((i, element) => { // console.log("cell element", i, $(element).text().trim()) if (i === 0) { configPost.key = $(element).text().trim(); } if (i === 1) { configPost.value = $(element).text().trim(); } if (i === 3) { configPost.description = $(element).text().replace(' ', '').replace(' ', '').trim(); } }) if (!configPost.key) { console.error("ERROR no key found", $(cells).html()) console.error('no key BOOOOOOOOOOH'); getElementCartographique(); return; } // aller chercher le lien // si la clé est une description avec des espaces, faire un lien vers la Key let pageLink = `FR:Tag:${configPost.key}=${configPost.value}` if (configPost.value.indexOf(' ') !== -1) { pageLink = `FR:Key:${configPost.key}` } configPost.link = `https://wiki.openstreetmap.org/wiki/${pageLink}` const url = configPost.link // get a random page wich is not a discussion // certaines valeurs ont des espaces, donc pas utilisable en url if (configPost.value.indexOf(' ') === -1 || configPost.value.indexOf('<') === -1 || configPost.value.indexOf('>') === -1) { console.log("url", url) rp(url).then(function (html) { //success! const descriptionStrophe = ''; const filteredHtml = $('.mw-parser-output', html).find('p') // trouver l'image de description configPost.download_description_src = getSourceOfDescriptorPageContent(html) console.log("filteredHtml", filteredHtml.length) configPost.long_desc = $(filteredHtml).text().substring(0,250) console.log("filteredHtml", configPost.long_desc) // image de description: // let imgSelector = ".description a.image img" console.log("✅ cette page existe bien en Français sur le wiki OSM") foundExistingWikiPageInFrench = true; sendMessageWikiTagOfTheDay(makePostMessageFromObj(configPost), configPost.download_description_src) return; }, (err) => { console.error(err.statusCode) if (err.statusCode == 404) { console.log("\n Cette page n'existe pas encore, c'est le moment de la créer.", `https://wiki.openstreetmap.org/w/index.php?title=FR:${configPost.key}=${configPost.value}`) //FR:Tag:boundary%3Dborder_zone&action=edit) console.log("Essayez de voir si la version en Anglais existe.") let englishPage = `https://wiki.openstreetmap.org/wiki/Tag:${configPost.key}=${configPost.value}` let englishKey = `https://wiki.openstreetmap.org/wiki/Key:${configPost.key}` console.log("-> ", englishPage) checkExistenceOfWebPage(englishPage).catch(err=> { checkExistenceOfWebPage( englishKey) }); console.log("bon spa tout ça on essaie une autre ligne") getElementCartographique() return; } }) } else { console.log("result", configPost) rp(configPost.link).then((result) => { console.log("oui cette page existe!", url) sendMessageWikiTagOfTheDay(makePostMessageFromObj(result)) }, (err) => { console.log("hé non. WTF ?") // getElementCartographique() }) } return configPost; }); } /** * check if a page anwsers to a GET request * @param url * @returns {*} */ function checkExistenceOfWebPage(url) { if(foundExistingWikiPageInFrench){ console.log("already found foundExistingWikiPageInFrench, no need to check other page", url) return; } return rp(url).then((html) => { console.log("oui cette page existe!", url) }, (err) => { console.log("hé non. WTF ?") }) } function makePostMessageFromObj(result) { return ` # Le tag OSM du jour : ${result.key}=${result.value} 🗺️🏷️ ${result.link} ${result.description ? result.description.trim() : ''} ${result.long_desc ? result.long_desc.trim() : ''} #osm #openstreetmap #wiki #rtfw ` } function getQuery() { // var url = "https://wiki.openstreetmap.org/w/api.php"; var url = "https://fr.wikipedia.org/w/api.php"; var params = { action: "query", format: "json", list: "random", enhanced: "1", hiderobot: "1", urlversion: "2", invert: "1", rnnamespace: "all-discussions%3B1%3B2%3B3%3B5%3B7%3B9%3B11%3B120%3B121%3B122%3B123%3B201%3B203", // WIKI et FR: lang: "FR", rnlimit: "5" }; url = url + "?&origin=*"; Object.keys(params).forEach(function (key) { url += "&" + key + "=" + params[key]; }); return fetch(url) .then(function (response) { return response.json(); }) .then(function (response) { const randoms = response.query.random; for (let ii = 0; ii < randoms.length; ii++) { // console.log(randoms[ii].title); console.log("randoms[ii]", randoms[ii]) } }) .catch(function (error) { console.log(error); }); } // run const res = getElementCartographique() // console.log("res", res) function sendMessageWikiTagOfTheDay(message, download_description_src) { let configPost = { author: 'curator', // document.querySelector('.d_image img').attributes['src'] image: '/home/tykayn/www/multi-account-post-schedule-mastodon/assets/osm_post_' + randomIntFromInterval(1 , 5) + '.jpg', message, } if(download_description_src) { let filePathImage = `${process.cwd()}/assets/blog_posts_medias/osm_wiki_description_page.jpg` if (download_description_src) { console.log("firstPictureSource found", download_description_src) // check if picture already exist console.log('on récupère l image de description : ', filePathImage) downloadImage(download_description_src, filePathImage) .then((res) => { // suite du poste avec upload d'image console.log('média téléchargé, on envoie le post') configPost.image = filePathImage; sendPostMastodon(configPost) }, (err) => { console.log('pas dimage trouvée pour l URL ', download_description_src, err) sendPostMastodon(configPost) } ) .catch((err) => { console.log('erreur avec cette URL ', download_description_src, err) sendPostMastodon(configPost) }) } }else{ console.log('no image description') sendPostMastodon(configPost) } }