From 9b67221aa58756cf36f85201bcc7121851d41163 Mon Sep 17 00:00:00 2001 From: Tykayn Date: Mon, 10 Feb 2025 22:34:56 +0100 Subject: [PATCH] up wiki posting --- .idea/dbnavigator.xml | 398 ++++++++++++++++++ .idea/inspectionProfiles/Project_Default.xml | 21 + .../bash_scripts/agendadulibre_coming_osm.sh | 2 +- assets/bash_scripts/osm_wiki.sh | 2 +- helpers/libs/utils.mjs | 48 ++- helpers/wiki_fr.mjs | 52 ++- helpers/wiki_osm_random_from_elements.mjs | 20 +- 7 files changed, 490 insertions(+), 53 deletions(-) create mode 100644 .idea/dbnavigator.xml create mode 100644 .idea/inspectionProfiles/Project_Default.xml diff --git a/.idea/dbnavigator.xml b/.idea/dbnavigator.xml new file mode 100644 index 0000000..218ff5d --- /dev/null +++ b/.idea/dbnavigator.xml @@ -0,0 +1,398 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 0000000..dee35bc --- /dev/null +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,21 @@ + + + + \ No newline at end of file diff --git a/assets/bash_scripts/agendadulibre_coming_osm.sh b/assets/bash_scripts/agendadulibre_coming_osm.sh index f76e55f..dddb76f 100644 --- a/assets/bash_scripts/agendadulibre_coming_osm.sh +++ b/assets/bash_scripts/agendadulibre_coming_osm.sh @@ -1,6 +1,6 @@ #!/bin/bash nvm use 16 #cd /home/tykayn/www/multi-account-post-schedule-mastodon -wget --no-check-certificate --no-directories -O ./../documents/agendadulibre_openstreetmap.json "https://www.agendadulibre.org/events.json?city=&near%5Blocation%5D=&near%5Bdistance%5D=®ion=&tag=openstreetmap" +wget --no-check-certificate --no-directories -O $HOME/www/multi-account-post-schedule-mastodon/documents/agendadulibre_openstreetmap.json "https://www.agendadulibre.org/events.json?city=&near%5Blocation%5D=&near%5Bdistance%5D=®ion=&tag=openstreetmap" node helpers/agendadulibre_coming_osm.mjs --force | tee -a /home/tykayn/log_autopost.txt 2>&1 \ No newline at end of file diff --git a/assets/bash_scripts/osm_wiki.sh b/assets/bash_scripts/osm_wiki.sh index b9130f1..c2c355b 100644 --- a/assets/bash_scripts/osm_wiki.sh +++ b/assets/bash_scripts/osm_wiki.sh @@ -1,4 +1,4 @@ #!/bin/bash nvm use 16 cd /home/tykayn/www/multi-account-post-schedule-mastodon -node helpers/wiki_osm.mjs --force | tee -a /home/tykayn/log_autopost.txt 2>&1 +$HOME/.nvm/versions/node/v20.6.1/bin/node helpers/wiki_osm_random_from_elements.mjs --force | tee -a /home/tykayn/log_autopost.txt 2>&1 diff --git a/helpers/libs/utils.mjs b/helpers/libs/utils.mjs index 77fdcba..e368d84 100644 --- a/helpers/libs/utils.mjs +++ b/helpers/libs/utils.mjs @@ -6,7 +6,7 @@ import https from 'https' import moment from 'moment' import Parser from 'rss-parser' -import {load} from 'cheerio' +import { load } from 'cheerio' let local_node_env_conf = dotenv.config() const myArgs = process.argv.slice(2) @@ -74,7 +74,6 @@ export default function sendPostMastodon(config) { ...config, } - // console.log("sendPostMastodon config", config) if (!config.reallySendPost) { @@ -84,6 +83,11 @@ export default function sendPostMastodon(config) { } else { console.log(' ') + + console.log('>>>>>>>>>>>> pas d envoi réel du post <<<<<<<<<<<<<<<<<<') + console.log('') + console.log(config) + if (process.env.INSTANCE_MASTODON && tokenForAuthorIsPresentInDotEnv(config.author)) { let visibility = 'public' @@ -137,7 +141,7 @@ export default function sendPostMastodon(config) { var id console.log('envoi du média', config.image) // upload new media - return masto.post('media', {file: fs.createReadStream(config.image)}) + return masto.post('media', { file: fs.createReadStream(config.image) }) .then(resp => { id = resp.data.id params.media_ids = [id] @@ -264,7 +268,7 @@ function clearLink(linkString) { export function downloadImage(url, filepath) { return new Promise((resolve, reject) => { const options = { - headers: {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 Edg/107.0.1418.52'} + headers: { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 Edg/107.0.1418.52' } } https.get(url, options, (res) => { @@ -294,7 +298,7 @@ export function CropPicture(pictureName, width = 500, height = 300) { return sharp(pictureName) - .extract({left: 0, top: 0, width, height}) + .extract({ left: 0, top: 0, width, height }) .toFile('thumb_' + pictureName, function (err) { if (err) console.log(err) }) @@ -333,14 +337,14 @@ export function findPictureAndSendPost(postContent, configPost) { console.log('on envoie le média et l image : ', filePathForDownloadedImage) downloadImage(firstPictureSource, filePathForDownloadedImage) .then((res) => { - // suite du poste avec upload d'image + // suite du poste avec upload d'image - console.log('média téléchargé, on envoie le post') - configPost.image = filePathForDownloadedImage + console.log('média téléchargé, on envoie le post') + configPost.image = filePathForDownloadedImage - sendPostMastodon(configPost) + sendPostMastodon(configPost) - }, + }, (err) => { console.log('pas dimage trouvée pour l URL ', firstPictureSource, err) sendPostMastodon(configPost) @@ -416,18 +420,18 @@ export function convertHTMLtoMD(htmlContent) { export function translateNode($) { const elementsToTranslate = [ - {tag: 'h1', mdTag: '##'}, - {tag: 'h2', mdTag: '###'}, - {tag: 'h3', mdTag: '####'}, - {tag: 'h4', mdTag: '#####'}, - {tag: 'h5', mdTag: '######'}, - {tag: 'p', mdTag: '\n\n'}, - {tag: 'ul', preserveChildren: true, transformChild: ($child) => '\n- ' + $.text($child)}, - {tag: 'ol', preserveChildren: true, transformChild: ($child, idx) => `\n${idx + 1}. ${$.text($child)}`}, - {tag: 'a', parseAttr: ('href', link => `[${link}](${link})`)}, - {tag: 'strong', mdFormat: '$&'}, - {tag: 'em', mdFormat: '_$&_'}, - {tag: 'code', mdFormat: '`$&`'}, + { tag: 'h1', mdTag: '##' }, + { tag: 'h2', mdTag: '###' }, + { tag: 'h3', mdTag: '####' }, + { tag: 'h4', mdTag: '#####' }, + { tag: 'h5', mdTag: '######' }, + { tag: 'p', mdTag: '\n\n' }, + { tag: 'ul', preserveChildren: true, transformChild: ($child) => '\n- ' + $.text($child) }, + { tag: 'ol', preserveChildren: true, transformChild: ($child, idx) => `\n${idx + 1}. ${$.text($child)}` }, + { tag: 'a', parseAttr: ('href', link => `[${link}](${link})`) }, + { tag: 'strong', mdFormat: '$&' }, + { tag: 'em', mdFormat: '_$&_' }, + { tag: 'code', mdFormat: '`$&`' }, ] elementsToTranslate.forEach(element => { diff --git a/helpers/wiki_fr.mjs b/helpers/wiki_fr.mjs index c95ebfb..8b9ae36 100644 --- a/helpers/wiki_fr.mjs +++ b/helpers/wiki_fr.mjs @@ -5,11 +5,11 @@ * nécessite d'avoir le fichier .env rempli. */ import fs from 'fs'; -import path, {dirname} from 'path'; +import path, { dirname } from 'path'; import axios from 'axios'; -import {fileURLToPath} from 'url'; -import {getSourceOfDescriptorPageContent} from "./osm_get_description_picture.mjs"; -import sendPostMastodon, {downloadImage, randomIntFromInterval} from "./libs/utils.mjs"; +import { fileURLToPath } from 'url'; +import { getSourceOfDescriptorPageContent } from "./osm_get_description_picture.mjs"; +import sendPostMastodon, { downloadImage, randomIntFromInterval } from "./libs/utils.mjs"; const __dirname = dirname(fileURLToPath(import.meta.url)); @@ -35,6 +35,7 @@ const params = { prop: 'text', section: 0 }; +const remplacedTitle = title.replace(' ', '_') console.log('url de l article: ', `https://wiki.openstreetmap.org/wiki/` + title) let message = '' @@ -49,8 +50,9 @@ export function sendMessageWikiOSMOfTheDay(message, download_description_src) { image: `${process.cwd()}/assets/blog_posts_medias/assets/osm_post_${randomIntFromInterval(1, 5)}.jpg`, message, } + let filePathImage = `${process.cwd()}/assets/blog_posts_medias/osm_wiki_description_page.jpg` if (download_description_src) { - let filePathImage = `${process.cwd()}/assets/blog_posts_medias/osm_wiki_description_page.jpg` + // let filePathImage = '' if (download_description_src) { console.log("firstPictureSource found", download_description_src) @@ -58,15 +60,15 @@ export function sendMessageWikiOSMOfTheDay(message, download_description_src) { console.log('on récupère l image de description : ', filePathImage) downloadImage(download_description_src, filePathImage) .then((res) => { - // suite du poste avec upload d'image + // suite du poste avec upload d'image - console.log('média téléchargé, on envoie le post') - configPost.image = filePathImage; + console.log('média téléchargé, on envoie le post') + configPost.image = filePathImage; - sendPostMastodon(configPost) + sendPostMastodon(configPost) - }, + }, (err) => { console.log('pas dimage trouvée pour l URL ', download_description_src, err) sendPostMastodon(configPost) @@ -80,42 +82,54 @@ export function sendMessageWikiOSMOfTheDay(message, download_description_src) { } else { console.log('::::::::::::: no image description') + filePathImage = `${process.cwd()}/assets/blog_posts_medias/osm_wiki_description_page.jpg` console.log('configPost.image', configPost.image) sendPostMastodon(configPost) } } -axios.get(wikiApiUrl, {params}) +axios.get(wikiApiUrl, { params }) .then(response => { // console.log('wikiApiUrl', wikiApiUrl) let keys = Object.keys(response.data) - // console.log('keys', keys) + console.log('keys', keys) const articleContent = response.data.parse.text['*']; let resultpicture = getSourceOfDescriptorPageContent(articleContent) - // console.log('resultpicture', resultpicture) + console.log('resultpicture', resultpicture) if (resultpicture === "pas d'image de description dans le HTML") { download_description_src = `${process.cwd()}/assets/blog_posts_medias/osm_wiki_description_page.jpg` } let firstParagraph; if (articleContent.includes('

')) { - firstParagraph = articleContent.split('

')[1].split('

')[0]; + firstParagraph = articleContent.split('

')[1]?.split('

')[0]; } else { firstParagraph = articleContent.substring(0, 200); } // Sanitizer le texte const sanitizedText = firstParagraph.replace(/<\/?[^>]+(>|$)/g, ''); - const sanitizedTextWithoutEntities = sanitizedText.replace(/&#(\d+);/g, (match, code) => { - return String.fromCharCode(code); - }); + const sanitizedTextWithoutEntities = sanitizedText + // const sanitizedTextWithoutEntities = sanitizedText.replace(/&#(\d+);/g, (match, code) => { + // return String.fromCharCode(code); + // }); console.log('image', download_description_src) console.log('-----------------') // Générer le message - message = `Le page #OSM du jour : ${title}. - https://wiki.openstreetmap.org/wiki/${title} + const formules = [ + "La page #OSM du jour", + "Aujourd'hui sur le wiki #OSM", + "Le wiki #OSM a du talent", + "Lisons le fabuleux wiki #OSM", + "Ho la belle documentation #OSM", + "Le wiki #OSM vous présente" + ]; + const formule = formules[randomIntFromInterval(0, formules.length - 1)]; + + message = `${formule} : ${title}. + [https://wiki.openstreetmap.org/wiki/${remplacedTitle}](https://wiki.openstreetmap.org/wiki/${remplacedTitle}) ${sanitizedTextWithoutEntities} diff --git a/helpers/wiki_osm_random_from_elements.mjs b/helpers/wiki_osm_random_from_elements.mjs index 4c43e08..6bc849a 100644 --- a/helpers/wiki_osm_random_from_elements.mjs +++ b/helpers/wiki_osm_random_from_elements.mjs @@ -9,8 +9,8 @@ import rp from "request-promise"; import $ from "cheerio"; import fs from "fs"; import path from 'path'; -import sendPostMastodon, {downloadImage, randomIntFromInterval} from './libs/utils.mjs' -import {getSourceOfDescriptorPageContent} from "./osm_get_description_picture.mjs"; +import sendPostMastodon, { downloadImage, randomIntFromInterval } from './libs/utils.mjs' +import { getSourceOfDescriptorPageContent } from "./osm_get_description_picture.mjs"; const __dirname = path.resolve(); @@ -138,7 +138,7 @@ export default function getElementCartographique() { return; } -// aller chercher le lien + // aller chercher le lien // si la clé est une description avec des espaces, faire un lien vers la Key let pageLink = `FR:Tag:${configPost.key}=${configPost.value}` @@ -284,8 +284,6 @@ export function sendMessageWikiOSMOfTheDay(message, download_description_src) { let configPost = { author: 'curator', - // document.querySelector('.d_image img').attributes['src'] - // image: '/home/tykayn/www/multi-account-post-schedule-mastodon/assets/osm_post_' + randomIntFromInterval(1 , 5) + '.jpg', image: `${process.cwd()}/assets/blog_posts_medias/assets/osm_post_${randomIntFromInterval(1, 5)}.jpg`, message, } @@ -298,14 +296,14 @@ export function sendMessageWikiOSMOfTheDay(message, download_description_src) { console.log('on récupère l image de description : ', filePathImage) downloadImage(download_description_src, filePathImage) .then((res) => { - // suite du poste avec upload d'image + // suite du poste avec upload d'image - console.log('média téléchargé, on envoie le post') - configPost.image = filePathImage; + console.log('média téléchargé, on envoie le post') + configPost.image = filePathImage; - sendPostMastodon(configPost) + sendPostMastodon(configPost) - }, + }, (err) => { console.log('pas dimage trouvée pour l URL ', download_description_src, err) sendPostMastodon(configPost) @@ -315,6 +313,8 @@ export function sendMessageWikiOSMOfTheDay(message, download_description_src) { console.log('erreur avec cette URL ', download_description_src, err) sendPostMastodon(configPost) }) + } else { + console.log(' pas de download_description_src', download_description_src) } } else {