multi-account-post-schedule-mastodon/helpers/wiki_osm.mjs

// https://www.mediawiki.org/wiki/Manual:Random_page
/**
 * Post de page aléatoire du wiki osm avec le compte curator
 */
import fetch from "node-fetch"
import rp from "request-promise";
import $ from "cheerio";
import fs from "fs";
import path from 'path';
import sendPostMastodon, {downloadImage, randomIntFromInterval} from './libs/utils.mjs'
import {getSourceOfDescriptorPageContent} from "./osm_get_description_picture.mjs";

const __dirname = path.resolve();

// select one line of table to override description
// const selectionOverrideOfSectionTable = 19;
// const selectionOverrideOfLineTable = 7;

// select randomly a line
const selectionOverrideOfSectionTable = '';
const selectionOverrideOfLineTable = '';

let foundExistingWikiPageInFrench = false;

function getRandomWikiOSMPage() {

  // get all FR tags
  // const url = "https://wiki.openstreetmap.org/w/index.php?title=Special:Categories&offset=FR&limit=500"
  const url = "https://wiki.openstreetmap.org/wiki/Special:AllPages?from=&to=&namespace=202&limit=500"
  // get a random page wich is not a discussion
  // const url = "https://wiki.openstreetmap.org/wiki/Special:Random?namespace=all-discussions%3B1%3B2%3B3%3B5%3B7%3B9%3B11%3B120%3B121%3B122%3B123%3B201%3B203&invert=1&limit=10&days=1&enhanced=1&namespace__202_color=c1&urlversion=2"
  rp(url)
      .then(function (html) {
        //success!
        const wikiUrls = [];
        const filteredHtml = $('.mw-spcontent ul li a', html)
        console.log("filteredHtml.length", filteredHtml.length)
        //     .filter(elem => {
        //     return (elem.attribs.title.indexOf('description for') === -1)
        // })
        for (let i = 0; i < filteredHtml.length; i++) {
          // const contentCurated = filteredHtml[i]
          const contentCurated = filteredHtml[i].attribs.href
          wikiUrls.push(contentCurated);
        }
        console.log(wikiUrls);
        return wikiUrls;
      })
      .catch(function (err) {
        //handle error
        console.error(err)
      });
}

export default function getElementCartographique() {
  if (foundExistingWikiPageInFrench) {
    console.log("already found foundExistingWikiPageInFrench")
    return;
  }
  const url = "https://wiki.openstreetmap.org/wiki/FR:%C3%89l%C3%A9ments_cartographiques"
  const titleLink = "#toc a"
  const listElementSelector = "#toc a"
  // prendre un lien dans la table des matières de la page, extraire l'id du lien.
  // prendre une ligne de tableau au hasard correspondant à l'ID de la section
  const titleSelector = ""


  return fs.readFile(__dirname + '/assets/documents/elements_cartographiques.html', 'utf8', function (err, html) {
    if (err) {
      return console.log(err);
    }
    // console.log(data);


    //success!
    const wikiUrls = [];
    // const listOfLinks = $(titleLink, html)
    // console.log("filteredHtml.length", listOfLinks.length)
    // const selectedTocLink = getRandomElementOfArray(listOfLinks).attribs.href;
    // console.log("selectedTocLink", selectedTocLink)

    const listOfTableRows = $('.wikitable', html);
    let keys = Object.keys(listOfTableRows);

    let randNumber = 0;
    if (selectionOverrideOfSectionTable) {
      randNumber = selectionOverrideOfSectionTable
    } else {
      randNumber = randomIntFromInterval(0, keys.length)
    }

    console.log("rand", randNumber)
    const foundLine = $(listOfTableRows[randNumber]).find('tr');

    // console.log("randomTable",foundTable)

    // on regarde les lignes du tableau
    keys = Object.keys(foundLine);
    if (selectionOverrideOfLineTable) {
      randNumber = selectionOverrideOfLineTable
    } else {
      randNumber = randomIntFromInterval(0, keys.length)
    }

    console.log("lignes de tableau", keys.length)
    console.log("ligne sélectionnée", randNumber)
    let cells = $(foundLine[randNumber]).find('td')

    keys = Object.keys(cells);
    // console.log("cellules trouvées", keys.length)

    console.log("\n")
    let configPost = {
      key: '',
      value: '',
      description: '',
      long_desc: '',
    }
    cells.each((i, element) => {
      // console.log("cell element", i, $(element).text().trim())
      if (i === 0) {
        configPost.key = $(element).text().trim();
      }
      if (i === 1) {
        configPost.value = $(element).text().trim();
      }
      if (i === 3) {
        configPost.description = $(element).text().replace('  ', '').replace('                            ', '').trim();
      }
    })

    if (!configPost.key) {
      console.error("ERROR no key found", $(cells).html())
      console.error('no key BOOOOOOOOOOH');
      getElementCartographique();
      return;
    }

// aller chercher le lien
    // si la clé est une description avec des espaces, faire un lien vers la Key
    let pageLink = `FR:Tag:${configPost.key}=${configPost.value}`

    if (configPost.value.indexOf(' ') !== -1) {
      pageLink = `FR:Key:${configPost.key}`
    }
    configPost.link = `https://wiki.openstreetmap.org/wiki/${pageLink}`

    const url = configPost.link
    // get a random page wich is not a discussion

    // certaines valeurs ont des espaces, donc pas utilisable en url
    if (configPost.value.indexOf(' ') === -1 || configPost.value.indexOf('<') === -1 || configPost.value.indexOf('>') === -1) {

      console.log("url", url)
      rp(url).then(function (html) {
        //success!
        const descriptionStrophe = '';
        const filteredHtml = $('.mw-parser-output', html).find('p')

        // trouver l'image de description
        configPost.download_description_src = getSourceOfDescriptorPageContent(html)

        console.log("filteredHtml", filteredHtml.length)
        configPost.long_desc = $(filteredHtml).text().substring(0, 250)
        console.log("filteredHtml", configPost.long_desc)

        // image de description:
        // let imgSelector = ".description a.image img"
        console.log("✅ cette page existe bien en Français sur le wiki OSM")
        foundExistingWikiPageInFrench = true;
        sendMessageWikiTagOfTheDay(makePostMessageFromObj(configPost), configPost.download_description_src)


      }, (err) => {
        console.error(err.statusCode)
        if (err.statusCode == 404) {
          console.log("\n Cette page n'existe pas encore, c'est le moment de la créer.", `https://wiki.openstreetmap.org/w/index.php?title=FR:${configPost.key}=${configPost.value}`) //FR:Tag:boundary%3Dborder_zone&action=edit)
          console.log("Essayez de voir si la version en Anglais existe.")
          let englishPage = `https://wiki.openstreetmap.org/wiki/Tag:${configPost.key}=${configPost.value}`
          let englishKey = `https://wiki.openstreetmap.org/wiki/Key:${configPost.key}`
          console.log("-> ", englishPage)
          checkExistenceOfWebPage(englishPage).catch(err => {
            checkExistenceOfWebPage(englishKey)
          });
          console.log("bon spa tout ça on essaie une autre ligne")
          getElementCartographique()

        }

      })
    } else {
      console.log("result", configPost)
      rp(configPost.link).then((result) => {
        console.log("oui cette page existe!", url)
        sendMessageWikiTagOfTheDay(makePostMessageFromObj(result))
      }, (err) => {
        console.log("hé non. WTF ?")
        // getElementCartographique()

      })


    }


    return configPost;

  });

}

/**
 * check if a page anwsers to a GET request
 * @param url
 * @returns {*}
 */
function checkExistenceOfWebPage(url) {
  if (foundExistingWikiPageInFrench) {
    console.log("already found foundExistingWikiPageInFrench, no need to check other page", url)
    return;
  }
  return rp(url).then((html) => {
    console.log("oui cette page existe!", url)
  }, (err) => {
    console.log("hé non. WTF ?")
  })
}

function makePostMessageFromObj(post_obj) {
  return `
# Le tag OSM du jour : ${post_obj.key}=${post_obj.value}  🗺️🏷️
${post_obj.link}

${post_obj.description ? post_obj.description.trim() : ''}
 ${post_obj.long_desc ? post_obj.long_desc.trim() : ''}
#osm #openstreetmap #wiki #rtfw
`
}

function getQuery() {
  // var url = "https://wiki.openstreetmap.org/w/api.php";
  var url = "https://fr.wikipedia.org/w/api.php";
  var params = {
    action: "query",
    format: "json",
    list: "random",
    enhanced: "1",
    hiderobot: "1",
    urlversion: "2",
    invert: "1",
    rnnamespace: "all-discussions%3B1%3B2%3B3%3B5%3B7%3B9%3B11%3B120%3B121%3B122%3B123%3B201%3B203", // WIKI et FR:
    lang: "FR",
    rnlimit: "5"
  };

  url = url + "?&origin=*";
  Object.keys(params).forEach(function (key) {
    url += "&" + key + "=" + params[key];
  });

  return fetch(url)
      .then(function (response) {
        return response.json();
      })
      .then(function (response) {
        const randoms = response.query.random;

        for (let ii = 0; ii < randoms.length; ii++) {
          // console.log(randoms[ii].title);
          console.log("randoms[ii]", randoms[ii])
        }
      })
      .catch(function (error) {
        console.log(error);
      });


}


// run

const res = getElementCartographique()

// console.log("res", res)


function sendMessageWikiTagOfTheDay(message, download_description_src) {


  let configPost = {
    author: 'curator',
    // document.querySelector('.d_image img').attributes['src']
    // image: '/home/tykayn/www/multi-account-post-schedule-mastodon/assets/osm_post_' + randomIntFromInterval(1 , 5) + '.jpg',
    image: `${process.cwd()}/assets/blog_posts_medias/assets/osm_post_${randomIntFromInterval(1, 5)}.jpg`,
    message,
  }
  if (download_description_src) {
    let filePathImage = `${process.cwd()}/assets/blog_posts_medias/osm_wiki_description_page.jpg`
    if (download_description_src) {
      console.log("firstPictureSource found", download_description_src)

      // check if picture already exist
      console.log('on récupère l image de description : ', filePathImage)
      downloadImage(download_description_src, filePathImage)
          .then((res) => {
                // suite du poste avec upload d'image

                console.log('média téléchargé, on envoie le post')
                configPost.image = filePathImage;

                sendPostMastodon(configPost)

              },
              (err) => {
                console.log('pas dimage trouvée pour l URL ', download_description_src, err)
                sendPostMastodon(configPost)
              }
          )
          .catch((err) => {
            console.log('erreur avec cette URL ', download_description_src, err)
            sendPostMastodon(configPost)
          })
    }

  } else {
    console.log('no image description')
    sendPostMastodon(configPost)
  }
}