multi-account-post-schedule-mastodon/helpers/wiki_osm.mjs

// https://www.mediawiki.org/wiki/Manual:Random_page
/**
 * Post de page aléatoire du wiki osm avec le compte curator
 */
import fetch from "node-fetch"
import rp from "request-promise";
import $ from "cheerio";
import fs from "fs";
import path from 'path';
import sendPostMastodon, {randomIntFromInterval} from './utils.mjs'

const __dirname = path.resolve();

// select one line of table to override description
// const selectionOverrideOfSectionTable = 2;
// const selectionOverrideOfLineTable = 7;

// select randomly a line
const selectionOverrideOfSectionTable = '';
const selectionOverrideOfLineTable = '';

let foundExistingWikiPageInFrench = false;
let reallySendPost = true;
// reallySendPost = false;

function getRandomWikiOSMPage() {

    // get all FR tags
    // const url = "https://wiki.openstreetmap.org/w/index.php?title=Special:Categories&offset=FR&limit=500"
    const url = "https://wiki.openstreetmap.org/wiki/Special:AllPages?from=&to=&namespace=202&limit=500"
    // get a random page wich is not a discussion
    // const url = "https://wiki.openstreetmap.org/wiki/Special:Random?namespace=all-discussions%3B1%3B2%3B3%3B5%3B7%3B9%3B11%3B120%3B121%3B122%3B123%3B201%3B203&invert=1&limit=10&days=1&enhanced=1&namespace__202_color=c1&urlversion=2"
    rp(url)
        .then(function (html) {
            //success!
            const wikiUrls = [];
            const filteredHtml = $('.mw-spcontent ul li a', html)
            console.log("filteredHtml.length", filteredHtml.length)
            //     .filter(elem => {
            //     return (elem.attribs.title.indexOf('description for') === -1)
            // })
            for (let i = 0; i < filteredHtml.length; i++) {
                // const contentCurated = filteredHtml[i]
                const contentCurated = filteredHtml[i].attribs.href
                wikiUrls.push(contentCurated);
            }
            console.log(wikiUrls);
            return wikiUrls;
        })
        .catch(function (err) {
            //handle error
            console.error(err)
        });
}

export default function getElementCartographique() {
    if(foundExistingWikiPageInFrench){
        console.log("already found foundExistingWikiPageInFrench")
        return;
    }
    const url = "https://wiki.openstreetmap.org/wiki/FR:%C3%89l%C3%A9ments_cartographiques"
    const titleLink = "#toc a"
    const listElementSelector = "#toc a"
    // prendre un lien dans la table des matières de la page, extraire l'id du lien.
    // prendre une ligne de tableau au hasard correspondant à l'ID de la section
    const titleSelector = ""


    return fs.readFile(__dirname + '/assets/documents/elements_cartographiques.html', 'utf8', function (err, html) {
        if (err) {
            return console.log(err);
        }
        // console.log(data);


        //success!
        const wikiUrls = [];
        // const listOfLinks = $(titleLink, html)
        // console.log("filteredHtml.length", listOfLinks.length)
        // const selectedTocLink = getRandomElementOfArray(listOfLinks).attribs.href;
        // console.log("selectedTocLink", selectedTocLink)

        const listOfTableRows = $('.wikitable', html);
        let keys = Object.keys(listOfTableRows);

        let randNumber = 0;
        if (selectionOverrideOfSectionTable) {
            randNumber = selectionOverrideOfSectionTable
        } else {
            randNumber = randomIntFromInterval(0, keys.length)
        }

        console.log("rand", randNumber)
        const foundLine = $(listOfTableRows[randNumber]).find('tr');

        // console.log("randomTable",foundTable)

        // on regarde les lignes du tableau
        keys = Object.keys(foundLine);
        if (selectionOverrideOfLineTable) {
            randNumber = selectionOverrideOfLineTable
        } else {
            randNumber = randomIntFromInterval(0, keys.length)
        }

        console.log("lignes de tableau", keys.length)
        console.log("ligne sélectionnée", randNumber)
        let cells = $(foundLine[randNumber]).find('td')

        keys = Object.keys(cells);
        // console.log("cellules trouvées", keys.length)

        console.log("\n")
        let configPost = {
            key: '',
            value: '',
            description: '',
            long_desc: '',
        }
        cells.each((i, element) => {
            // console.log("cell element", i, $(element).text().trim())
            if (i === 0) {
                configPost.key = $(element).text().trim();
            }
            if (i === 1) {
                configPost.value = $(element).text().trim();
            }
            if (i === 3) {
                configPost.description = $(element).text().replace('  ', '').replace('                            ', '').trim();
            }
        })

        if (!configPost.key) {
            console.error("ERROR no key found", $(cells).html())
            console.error('no key BOOOOOOOOOOH');
            getElementCartographique();
            return;
        }

// aller chercher le lien
        // si la clé est une description avec des espaces, faire un lien vers la Key
        let pageLink = `FR:Tag:${configPost.key}=${configPost.value}`

        if (configPost.value.indexOf(' ') !== -1) {
            pageLink = `FR:Key:${configPost.key}`
        }
        configPost.link = `https://wiki.openstreetmap.org/wiki/${pageLink}`

        const url = configPost.link
        // get a random page wich is not a discussion

        // certaines valeurs ont des espaces, donc pas utilisable en url
        if (configPost.value.indexOf(' ') === -1 || configPost.value.indexOf('<') === -1 || configPost.value.indexOf('>') === -1) {

            console.log("url", url)
            rp(url).then(function (html) {
                //success!
                const descriptionStrophe = '';
                const filteredHtml = $('.mw-parser-output', html).find('p')

                console.log("filteredHtml", filteredHtml.length)
                configPost.long_desc = $(filteredHtml).text().substring(0,250)
                console.log("filteredHtml", configPost.long_desc)

                // image de description:
                // let imgSelector = ".description a.image img"
                console.log("✅ cette page existe bien en Français sur le wiki OSM")
                foundExistingWikiPageInFrench = true;
                sendMessageWikiTagOfTheDay(makePostMessageFromObj(configPost))
                return;

            }, (err) => {
                console.error(err.statusCode)
                if (err.statusCode == 404) {
                    console.log("\n Cette page n'existe pas encore, c'est le moment de la créer.", `https://wiki.openstreetmap.org/w/index.php?title=FR:${configPost.key}=${configPost.value}`) //FR:Tag:boundary%3Dborder_zone&action=edit)
                    console.log("Essayez de voir si la version en Anglais existe.")
                    let englishPage = `https://wiki.openstreetmap.org/wiki/Tag:${configPost.key}=${configPost.value}`
                    let englishKey = `https://wiki.openstreetmap.org/wiki/Key:${configPost.key}`
                    console.log("-> ", englishPage)
                    checkExistenceOfWebPage(englishPage).catch(err=> {
                        checkExistenceOfWebPage( englishKey)
                    });
                    console.log("bon spa tout ça on essaie une autre ligne")
                    getElementCartographique()
                    return;
                }

            })
        } else {
            console.log("result", configPost)
            rp(configPost.link).then((result) => {
                console.log("oui cette page existe!", url)
                sendMessageWikiTagOfTheDay(makePostMessageFromObj(result))
            }, (err) => {
                console.log("hé non. WTF ?")
                // getElementCartographique()

            })


        }


        return configPost;

    });

}

/**
 * check if a page anwsers to a GET request
 * @param url
 * @returns {*}
 */
function checkExistenceOfWebPage(url) {
    if(foundExistingWikiPageInFrench){
        console.log("already found foundExistingWikiPageInFrench, no need to check other page", url)
        return;
    }
    return rp(url).then((html) => {
        console.log("oui cette page existe!", url)
    }, (err) => {
        console.log("hé non. WTF ?")
    })
}

function makePostMessageFromObj(result) {
    return `
# [Le tag OSM du jour : ${result.key}=${result.value} ](${result.link}) 🗺️🏷️
${result.description ? result.description.trim() : ''}
> ${result.long_desc ? result.long_desc.trim() : ''}
#osm #openstreetmap #wiki #rtfw
`
}

function getQuery() {
    // var url = "https://wiki.openstreetmap.org/w/api.php";
    var url = "https://fr.wikipedia.org/w/api.php";
    var params = {
        action: "query",
        format: "json",
        list: "random",
        enhanced: "1",
        hiderobot: "1",
        urlversion: "2",
        invert: "1",
        rnnamespace: "all-discussions%3B1%3B2%3B3%3B5%3B7%3B9%3B11%3B120%3B121%3B122%3B123%3B201%3B203", // WIKI et FR:
        lang: "FR",
        rnlimit: "5"
    };

    url = url + "?&origin=*";
    Object.keys(params).forEach(function (key) {
        url += "&" + key + "=" + params[key];
    });

    return fetch(url)
        .then(function (response) {
            return response.json();
        })
        .then(function (response) {
            const randoms = response.query.random;

            for (let ii = 0; ii < randoms.length; ii++) {
                // console.log(randoms[ii].title);
                console.log("randoms[ii]", randoms[ii])
            }
        })
        .catch(function (error) {
            console.log(error);
        });


}


// run

const res = getElementCartographique()
// console.log("res", res)


function sendMessageWikiTagOfTheDay(message) {


    let configPost = {
        author: 'curator',
        image: 'osm_post_' + randomIntFromInterval(1 , 5) + '.jpg',
        message,
        // reallySendPost: true,
        reallySendPost: reallySendPost,
    }
    sendPostMastodon(configPost)
}