page fr osm wiki
This commit is contained in:
parent
400e1b2b42
commit
6b8e111d77
@ -6,43 +6,44 @@ import https from 'https'
|
|||||||
import moment from 'moment'
|
import moment from 'moment'
|
||||||
|
|
||||||
import Parser from 'rss-parser'
|
import Parser from 'rss-parser'
|
||||||
import { load } from 'cheerio'
|
import {load} from 'cheerio'
|
||||||
|
|
||||||
let local_node_env_conf = dotenv.config()
|
let local_node_env_conf = dotenv.config()
|
||||||
const myArgs = process.argv.slice(2)
|
const myArgs = process.argv.slice(2)
|
||||||
export const reallySendPost = hasCliArgument('--force')
|
export const reallySendPost = hasCliArgument('--force')
|
||||||
|
console.log('reallySendPost', reallySendPost)
|
||||||
export const folderBlogPostsPreview = process.cwd() + '/assets/blog_posts_medias/'
|
export const folderBlogPostsPreview = process.cwd() + '/assets/blog_posts_medias/'
|
||||||
|
|
||||||
export function randomIntFromInterval (min, max) { // min and max included
|
export function randomIntFromInterval(min, max) { // min and max included
|
||||||
return Math.floor(Math.random() * (max - min + 1) + min) | 1
|
return Math.floor(Math.random() * (max - min + 1) + min) | 1
|
||||||
}
|
}
|
||||||
|
|
||||||
export function getRandomElementOfArray (listItems) {
|
export function getRandomElementOfArray(listItems) {
|
||||||
return listItems[Math.floor(Math.random() * listItems.length)]
|
return listItems[Math.floor(Math.random() * listItems.length)]
|
||||||
}
|
}
|
||||||
|
|
||||||
let nowDate = new Date()
|
let nowDate = new Date()
|
||||||
|
|
||||||
export let defaultConfigMasto = {
|
export let defaultConfigMasto = {
|
||||||
author: 'curator',
|
author: 'curator',
|
||||||
visibility: 'public',
|
visibility: 'public',
|
||||||
language: 'fr',
|
language: 'fr',
|
||||||
sensitive: false,
|
sensitive: false,
|
||||||
disable_slugify: false,
|
disable_slugify: false,
|
||||||
reallySendPost,
|
reallySendPost,
|
||||||
image: '',
|
image: '',
|
||||||
folder_image: process.cwd() + '/assets/blog_posts_medias/',
|
folder_image: process.cwd() + '/assets/blog_posts_medias/',
|
||||||
message: 'Hey coucou! on est le' + nowDate,
|
message: 'Hey coucou! on est le' + nowDate,
|
||||||
scheduled_at: '',
|
scheduled_at: '',
|
||||||
scheduled_at_bool: false,
|
scheduled_at_bool: false,
|
||||||
content_type: 'text/markdown',
|
content_type: 'text/markdown',
|
||||||
website: 'qzine',
|
website: 'qzine',
|
||||||
slug: 'default_post_title',
|
slug: 'default_post_title',
|
||||||
postObject: {},
|
postObject: {},
|
||||||
}
|
}
|
||||||
|
|
||||||
export function tokenForAuthorIsPresentInDotEnv (author) {
|
export function tokenForAuthorIsPresentInDotEnv(author) {
|
||||||
return process.env['TOKEN_' + author.toUpperCase()]
|
return process.env['TOKEN_' + author.toUpperCase()]
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -50,13 +51,13 @@ export function tokenForAuthorIsPresentInDotEnv (author) {
|
|||||||
* @param userNickName
|
* @param userNickName
|
||||||
* @returns {Mastodon}
|
* @returns {Mastodon}
|
||||||
*/
|
*/
|
||||||
export function createMastoFetcherWithAuthorLogin (userNickName) {
|
export function createMastoFetcherWithAuthorLogin(userNickName) {
|
||||||
let accessToken = process.env['TOKEN_' + userNickName.toUpperCase()]
|
let accessToken = process.env['TOKEN_' + userNickName.toUpperCase()]
|
||||||
const masto = new Masto({
|
const masto = new Masto({
|
||||||
access_token: accessToken,
|
access_token: accessToken,
|
||||||
api_url: process.env.INSTANCE_MASTODON + '/api/v1/',
|
api_url: process.env.INSTANCE_MASTODON + '/api/v1/',
|
||||||
})
|
})
|
||||||
return masto
|
return masto
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -64,125 +65,125 @@ export function createMastoFetcherWithAuthorLogin (userNickName) {
|
|||||||
* @param config
|
* @param config
|
||||||
* @returns {*}
|
* @returns {*}
|
||||||
*/
|
*/
|
||||||
export default function sendPostMastodon (config) {
|
export default function sendPostMastodon(config) {
|
||||||
|
|
||||||
// console.log('send post', config.postObject.post_guid , config.postObject.guid )
|
// console.log('send post', config.postObject.post_guid , config.postObject.guid )
|
||||||
// override defaults with input argument
|
// override defaults with input argument
|
||||||
config = {
|
config = {
|
||||||
...defaultConfigMasto,
|
...defaultConfigMasto,
|
||||||
...config,
|
...config,
|
||||||
}
|
}
|
||||||
|
|
||||||
// console.log("sendPostMastodon config", config)
|
// console.log("sendPostMastodon config", config)
|
||||||
|
|
||||||
if (!config.reallySendPost) {
|
if (!config.reallySendPost) {
|
||||||
|
|
||||||
console.log('\n\n =========== le message ne sera PAS réellement posté sur le compte @' + config.author + '@' + process.env.INSTANCE_MASTODON + ' =========== \n')
|
console.log('\n\n =========== le message ne sera PAS réellement posté sur le compte @' + config.author + '@' + process.env.INSTANCE_MASTODON + ' =========== \n')
|
||||||
// console.log('configPost.folder_image', config.folder_image)
|
// console.log('configPost.folder_image', config.folder_image)
|
||||||
console.log('config', config.message)
|
console.log('config', config.message)
|
||||||
} else {
|
} else {
|
||||||
|
|
||||||
console.log(' ')
|
console.log(' ')
|
||||||
if (process.env.INSTANCE_MASTODON && tokenForAuthorIsPresentInDotEnv(config.author)) {
|
if (process.env.INSTANCE_MASTODON && tokenForAuthorIsPresentInDotEnv(config.author)) {
|
||||||
|
|
||||||
let visibility = 'public'
|
let visibility = 'public'
|
||||||
let language = 'fr'
|
let language = 'fr'
|
||||||
let sensitive = false
|
let sensitive = false
|
||||||
|
|
||||||
let accessToken = process.env['TOKEN_' + config.author.toUpperCase()]
|
let accessToken = process.env['TOKEN_' + config.author.toUpperCase()]
|
||||||
const masto = new Masto({
|
const masto = new Masto({
|
||||||
access_token: accessToken,
|
access_token: accessToken,
|
||||||
api_url: process.env.INSTANCE_MASTODON + '/api/v1/',
|
api_url: process.env.INSTANCE_MASTODON + '/api/v1/',
|
||||||
})
|
})
|
||||||
|
|
||||||
let params = {
|
let params = {
|
||||||
status: config.message,
|
status: config.message,
|
||||||
visibility,
|
visibility,
|
||||||
language,
|
language,
|
||||||
sensitive
|
sensitive
|
||||||
}
|
}
|
||||||
if (config.cw) {
|
if (config.cw) {
|
||||||
params['spoiler_text'] = config.cw
|
params['spoiler_text'] = config.cw
|
||||||
}
|
}
|
||||||
if (config.scheduled_at && config.scheduled_at_bool) {
|
if (config.scheduled_at && config.scheduled_at_bool) {
|
||||||
let dateschedule = new Date(config.scheduled_at)
|
let dateschedule = new Date(config.scheduled_at)
|
||||||
params['scheduled_at'] = dateschedule.toISOString()
|
params['scheduled_at'] = dateschedule.toISOString()
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* envoi sans fichier joint
|
* envoi sans fichier joint
|
||||||
*/
|
*/
|
||||||
if (!config.image) {
|
if (!config.image) {
|
||||||
console.log('pas d image dans la config')
|
console.log('pas d image dans la config')
|
||||||
|
|
||||||
if (config.reallySendPost) {
|
if (config.reallySendPost) {
|
||||||
|
|
||||||
masto.post('statuses', params).then(rep => {
|
masto.post('statuses', params).then(rep => {
|
||||||
|
|
||||||
console.log('posté, yay!')
|
console.log('posté, yay!')
|
||||||
}, err => {
|
}, err => {
|
||||||
console.error(err)
|
console.error(err)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* envoi avec fichier,
|
* envoi avec fichier,
|
||||||
* on doit d'abord faire un upload du fichier,
|
* on doit d'abord faire un upload du fichier,
|
||||||
* puis relier son id de média au nouveau post.
|
* puis relier son id de média au nouveau post.
|
||||||
*/
|
*/
|
||||||
else if (config.image) {
|
else if (config.image) {
|
||||||
|
|
||||||
var id
|
var id
|
||||||
console.log('envoi du média', config.image)
|
console.log('envoi du média', config.image)
|
||||||
// upload new media
|
// upload new media
|
||||||
return masto.post('media', { file: fs.createReadStream(config.image) })
|
return masto.post('media', {file: fs.createReadStream(config.image)})
|
||||||
.then(resp => {
|
.then(resp => {
|
||||||
id = resp.data.id
|
id = resp.data.id
|
||||||
params.media_ids = [id]
|
params.media_ids = [id]
|
||||||
console.log('\n ✅ image, id', id)
|
console.log('\n ✅ image, id', id)
|
||||||
masto.post('statuses', params).then(rep => {
|
masto.post('statuses', params).then(rep => {
|
||||||
// console.log('rep', rep)
|
// console.log('rep', rep)
|
||||||
console.log('\n ✅ posté avec une nouvelle image, WOOT')
|
console.log('\n ✅ posté avec une nouvelle image, WOOT')
|
||||||
}, err => {
|
}, err => {
|
||||||
console.error(err)
|
console.error(err)
|
||||||
|
|
||||||
console.log('erreur T_T')
|
console.log('erreur T_T')
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// }
|
// }
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
console.error(`pas de token pour l'auteur "${config.author}" ou pas d'instance mastodon définie`)
|
console.error(`pas de token pour l'auteur "${config.author}" ou pas d'instance mastodon définie`)
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Slugify a string
|
// Slugify a string
|
||||||
export function slugify (str) {
|
export function slugify(str) {
|
||||||
str = str.replace(/^\s+|\s+$/g, '')
|
str = str.replace(/^\s+|\s+$/g, '')
|
||||||
|
|
||||||
// Make the string lowercase
|
// Make the string lowercase
|
||||||
str = str.toLowerCase()
|
str = str.toLowerCase()
|
||||||
|
|
||||||
// Remove accents, swap ñ for n, etc
|
// Remove accents, swap ñ for n, etc
|
||||||
var from = 'ÁÄÂÀÃÅČÇĆĎÉĚËÈÊẼĔȆÍÌÎÏŇÑÓÖÒÔÕØŘŔŠŤÚŮÜÙÛÝŸŽáäâàãåčçćďéěëèêẽĕȇíìîïňñóöòôõøðřŕšťúůüùûýÿžþÞĐđßÆa·/_,:;'
|
var from = 'ÁÄÂÀÃÅČÇĆĎÉĚËÈÊẼĔȆÍÌÎÏŇÑÓÖÒÔÕØŘŔŠŤÚŮÜÙÛÝŸŽáäâàãåčçćďéěëèêẽĕȇíìîïňñóöòôõøðřŕšťúůüùûýÿžþÞĐđßÆa·/_,:;'
|
||||||
var to = 'AAAAAACCCDEEEEEEEEIIIINNOOOOOORRSTUUUUUYYZaaaaaacccdeeeeeeeeiiiinnooooooorrstuuuuuyyzbBDdBAa------'
|
var to = 'AAAAAACCCDEEEEEEEEIIIINNOOOOOORRSTUUUUUYYZaaaaaacccdeeeeeeeeiiiinnooooooorrstuuuuuyyzbBDdBAa------'
|
||||||
for (var i = 0, l = from.length; i < l; i++) {
|
for (var i = 0, l = from.length; i < l; i++) {
|
||||||
str = str.replace(new RegExp(from.charAt(i), 'g'), to.charAt(i))
|
str = str.replace(new RegExp(from.charAt(i), 'g'), to.charAt(i))
|
||||||
}
|
}
|
||||||
|
|
||||||
// Remove invalid chars
|
// Remove invalid chars
|
||||||
str = str.replace(/[^a-z0-9 -]/g, '')
|
str = str.replace(/[^a-z0-9 -]/g, '')
|
||||||
// Collapse whitespace and replace by -
|
// Collapse whitespace and replace by -
|
||||||
.replace(/\s+/g, '-')
|
.replace(/\s+/g, '-')
|
||||||
// Collapse dashes
|
// Collapse dashes
|
||||||
.replace(/-+/g, '-')
|
.replace(/-+/g, '-')
|
||||||
|
|
||||||
return str
|
return str
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -190,13 +191,13 @@ export function slugify (str) {
|
|||||||
* lister les noms de fichier que l'on peut publier dans un dossier.
|
* lister les noms de fichier que l'on peut publier dans un dossier.
|
||||||
* retourne un tableau
|
* retourne un tableau
|
||||||
*/
|
*/
|
||||||
export function listFilesOfFolder (folderPath) {
|
export function listFilesOfFolder(folderPath) {
|
||||||
let filesNames = []
|
let filesNames = []
|
||||||
fs.readdirSync(folderPath).map(fileName => {
|
fs.readdirSync(folderPath).map(fileName => {
|
||||||
return filesNames.push(fileName)
|
return filesNames.push(fileName)
|
||||||
})
|
})
|
||||||
|
|
||||||
return filesNames
|
return filesNames
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -204,14 +205,14 @@ export function listFilesOfFolder (folderPath) {
|
|||||||
* crée un dossier d'assets, avec ses sous dossiers not_published et published si ils manquent.
|
* crée un dossier d'assets, avec ses sous dossiers not_published et published si ils manquent.
|
||||||
* une fois que l'on prendra une image dans le dossier non publié, on la déplacera dans le dossier des images publées.
|
* une fois que l'on prendra une image dans le dossier non publié, on la déplacera dans le dossier des images publées.
|
||||||
*/
|
*/
|
||||||
export function initializeFolderForPictures (folderName) {
|
export function initializeFolderForPictures(folderName) {
|
||||||
try {
|
try {
|
||||||
if (!fs.existsSync(folderName)) {
|
if (!fs.existsSync(folderName)) {
|
||||||
fs.mkdirSync(folderName)
|
fs.mkdirSync(folderName)
|
||||||
}
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error(err)
|
console.error(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -219,36 +220,36 @@ export function initializeFolderForPictures (folderName) {
|
|||||||
* @param htmlContent
|
* @param htmlContent
|
||||||
* @returns {string}
|
* @returns {string}
|
||||||
*/
|
*/
|
||||||
export function findFirstImageInContent (htmlContent = '') {
|
export function findFirstImageInContent(htmlContent = '') {
|
||||||
let result = ''
|
let result = ''
|
||||||
let foundPictures = htmlContent.match(/<img\s[^>]*?src\s*=\s*['\"]([^'\"]*?)['\"][^>]*?>/)
|
let foundPictures = htmlContent.match(/<img\s[^>]*?src\s*=\s*['\"]([^'\"]*?)['\"][^>]*?>/)
|
||||||
let first = ''
|
let first = ''
|
||||||
if (foundPictures && foundPictures[0]) {
|
if (foundPictures && foundPictures[0]) {
|
||||||
|
|
||||||
first = foundPictures[0]
|
first = foundPictures[0]
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
console.log('pas d image trouvée dans le contenu ', htmlContent)
|
console.log('pas d image trouvée dans le contenu ', htmlContent)
|
||||||
}
|
}
|
||||||
if (first) {
|
if (first) {
|
||||||
result = first.match(/src\=\"(.*)\"/i)
|
result = first.match(/src\=\"(.*)\"/i)
|
||||||
if (result.length && result[0]) {
|
if (result.length && result[0]) {
|
||||||
result = result[0].split('"')
|
result = result[0].split('"')
|
||||||
result = result[1]
|
result = result[1]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
result = clearLink(result)
|
result = clearLink(result)
|
||||||
console.log('clearLink', result)
|
console.log('clearLink', result)
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
function clearLink (linkString) {
|
function clearLink(linkString) {
|
||||||
linkString = linkString.replace('http:', 'https:')
|
linkString = linkString.replace('http:', 'https:')
|
||||||
linkString = linkString.replace('https://www.ailesse.info/~tykayn/bazar/kotlife', 'https://www.tykayn.fr/wp-content/uploads/i/kotlife')
|
linkString = linkString.replace('https://www.ailesse.info/~tykayn/bazar/kotlife', 'https://www.tykayn.fr/wp-content/uploads/i/kotlife')
|
||||||
linkString = linkString.replace('https://blog.artlemoine.com/public/i', 'https://www.tykayn.fr/wp-content/uploads/i')
|
linkString = linkString.replace('https://blog.artlemoine.com/public/i', 'https://www.tykayn.fr/wp-content/uploads/i')
|
||||||
linkString = linkString.replace('https://www.ailesse.com/%7Etykayn/bazar', 'https://www.tykayn.fr/wp-content/uploads/i/bazar')
|
linkString = linkString.replace('https://www.ailesse.com/%7Etykayn/bazar', 'https://www.tykayn.fr/wp-content/uploads/i/bazar')
|
||||||
|
|
||||||
return linkString
|
return linkString
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -260,25 +261,25 @@ function clearLink (linkString) {
|
|||||||
* @param filepath
|
* @param filepath
|
||||||
* @returns {Promise<unknown>}
|
* @returns {Promise<unknown>}
|
||||||
*/
|
*/
|
||||||
export function downloadImage (url, filepath) {
|
export function downloadImage(url, filepath) {
|
||||||
return new Promise((resolve, reject) => {
|
return new Promise((resolve, reject) => {
|
||||||
const options = {
|
const options = {
|
||||||
headers: { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 Edg/107.0.1418.52' }
|
headers: {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 Edg/107.0.1418.52'}
|
||||||
}
|
}
|
||||||
|
|
||||||
https.get(url, options, (res) => {
|
https.get(url, options, (res) => {
|
||||||
if (res.statusCode === 200) {
|
if (res.statusCode === 200) {
|
||||||
res.pipe(fs.createWriteStream(filepath))
|
res.pipe(fs.createWriteStream(filepath))
|
||||||
.on('error', reject)
|
.on('error', reject)
|
||||||
.once('close', () => resolve(filepath))
|
.once('close', () => resolve(filepath))
|
||||||
} else {
|
} else {
|
||||||
// Consume response data to free up memory
|
// Consume response data to free up memory
|
||||||
res.resume()
|
res.resume()
|
||||||
reject(new Error(`Request Failed With a Status Code: ${res.statusCode}; \n ${res.statusMessage} `))
|
reject(new Error(`Request Failed With a Status Code: ${res.statusCode}; \n ${res.statusMessage} `))
|
||||||
|
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -289,29 +290,29 @@ export function downloadImage (url, filepath) {
|
|||||||
* @returns {Promise<Object>}
|
* @returns {Promise<Object>}
|
||||||
* @constructor
|
* @constructor
|
||||||
*/
|
*/
|
||||||
export function CropPicture (pictureName, width = 500, height = 300) {
|
export function CropPicture(pictureName, width = 500, height = 300) {
|
||||||
|
|
||||||
return sharp(pictureName)
|
return sharp(pictureName)
|
||||||
|
|
||||||
.extract({ left: 0, top: 0, width, height })
|
.extract({left: 0, top: 0, width, height})
|
||||||
.toFile('thumb_' + pictureName, function (err) {
|
.toFile('thumb_' + pictureName, function (err) {
|
||||||
if (err) console.log(err)
|
if (err) console.log(err)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* prendre un post parmi tous ceux du blog, dans ceux qui ont été publiés
|
* prendre un post parmi tous ceux du blog, dans ceux qui ont été publiés
|
||||||
* @returns {*}
|
* @returns {*}
|
||||||
*/
|
*/
|
||||||
export function getRandomLinkGeneral (tkpostsjson) {
|
export function getRandomLinkGeneral(tkpostsjson) {
|
||||||
let filteredLinks = []
|
let filteredLinks = []
|
||||||
if (tkpostsjson[0].post_status) {
|
if (tkpostsjson[0].post_status) {
|
||||||
|
|
||||||
filteredLinks = tkpostsjson.filter(elem => elem.post_status === 'publish')
|
filteredLinks = tkpostsjson.filter(elem => elem.post_status === 'publish')
|
||||||
} else if (tkpostsjson[0].status) {
|
} else if (tkpostsjson[0].status) {
|
||||||
filteredLinks = tkpostsjson.filter(elem => elem.status === 'publish')
|
filteredLinks = tkpostsjson.filter(elem => elem.status === 'publish')
|
||||||
}
|
}
|
||||||
return getRandomElementOfArray(filteredLinks)
|
return getRandomElementOfArray(filteredLinks)
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -319,44 +320,44 @@ export function getRandomLinkGeneral (tkpostsjson) {
|
|||||||
* @param postContent
|
* @param postContent
|
||||||
* @param configPost
|
* @param configPost
|
||||||
*/
|
*/
|
||||||
export function findPictureAndSendPost (postContent, configPost) {
|
export function findPictureAndSendPost(postContent, configPost) {
|
||||||
|
|
||||||
let firstPictureSource = findFirstImageInContent(postContent)
|
let firstPictureSource = findFirstImageInContent(postContent)
|
||||||
|
|
||||||
let filePathForDownloadedImage = `${configPost.folder_image}_${configPost.website}_media_post_${slugify(configPost.slug)}.jpg`
|
let filePathForDownloadedImage = `${configPost.folder_image}_${configPost.website}_media_post_${slugify(configPost.slug)}.jpg`
|
||||||
|
|
||||||
if (firstPictureSource) {
|
if (firstPictureSource) {
|
||||||
console.log('firstPictureSource found', firstPictureSource)
|
console.log('firstPictureSource found', firstPictureSource)
|
||||||
|
|
||||||
// check if picture already exist
|
// check if picture already exist
|
||||||
console.log('on envoie le média et l image : ', filePathForDownloadedImage)
|
console.log('on envoie le média et l image : ', filePathForDownloadedImage)
|
||||||
downloadImage(firstPictureSource, filePathForDownloadedImage)
|
downloadImage(firstPictureSource, filePathForDownloadedImage)
|
||||||
.then((res) => {
|
.then((res) => {
|
||||||
// suite du poste avec upload d'image
|
// suite du poste avec upload d'image
|
||||||
|
|
||||||
console.log('média téléchargé, on envoie le post')
|
console.log('média téléchargé, on envoie le post')
|
||||||
configPost.image = filePathForDownloadedImage
|
configPost.image = filePathForDownloadedImage
|
||||||
|
|
||||||
sendPostMastodon(configPost)
|
sendPostMastodon(configPost)
|
||||||
|
|
||||||
},
|
},
|
||||||
(err) => {
|
(err) => {
|
||||||
console.log('pas dimage trouvée pour l URL ', firstPictureSource, err)
|
console.log('pas dimage trouvée pour l URL ', firstPictureSource, err)
|
||||||
sendPostMastodon(configPost)
|
sendPostMastodon(configPost)
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
.catch((err) => {
|
.catch((err) => {
|
||||||
console.log('erreur avec cette URL ', firstPictureSource, err)
|
console.log('erreur avec cette URL ', firstPictureSource, err)
|
||||||
sendPostMastodon(configPost)
|
sendPostMastodon(configPost)
|
||||||
})
|
})
|
||||||
} else {
|
} else {
|
||||||
|
|
||||||
// no image provided
|
// no image provided
|
||||||
|
|
||||||
console.log('pas d\'image dans le corps du texte', configPost.image)
|
console.log('pas d\'image dans le corps du texte', configPost.image)
|
||||||
// on envoie avec l'image par défaut
|
// on envoie avec l'image par défaut
|
||||||
sendPostMastodon(configPost)
|
sendPostMastodon(configPost)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -364,163 +365,163 @@ export function findPictureAndSendPost (postContent, configPost) {
|
|||||||
* @param argument
|
* @param argument
|
||||||
* @returns {boolean}
|
* @returns {boolean}
|
||||||
*/
|
*/
|
||||||
export function hasCliArgument (argument) {
|
export function hasCliArgument(argument) {
|
||||||
return myArgs.indexOf(argument) !== -1
|
return myArgs.indexOf(argument) !== -1
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let parser = new Parser()
|
let parser = new Parser()
|
||||||
|
|
||||||
export function diffDaysBetweenTwoDates (date1, date2) {
|
export function diffDaysBetweenTwoDates(date1, date2) {
|
||||||
|
|
||||||
const a = moment(date1)
|
const a = moment(date1)
|
||||||
const b = moment(date2)
|
const b = moment(date2)
|
||||||
return a.diff(b, 'days')
|
return a.diff(b, 'days')
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export function filterRegionAgendaDuLibreEvents (events_list, filter_critera) {
|
export function filterRegionAgendaDuLibreEvents(events_list, filter_critera) {
|
||||||
let selection = []
|
let selection = []
|
||||||
events_list.forEach(item => {
|
events_list.forEach(item => {
|
||||||
if (item.region_id == filter_critera) {
|
if (item.region_id == filter_critera) {
|
||||||
selection.push(item)
|
selection.push(item)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
return selection
|
return selection
|
||||||
}
|
}
|
||||||
|
|
||||||
moment.locale('fr')
|
moment.locale('fr')
|
||||||
|
|
||||||
export function groupEventsByDay (events_list) {
|
export function groupEventsByDay(events_list) {
|
||||||
let selection = {}
|
let selection = {}
|
||||||
events_list.forEach(item => {
|
events_list.forEach(item => {
|
||||||
|
|
||||||
let formattedDay = moment(item.start_time).format('dddd DD')
|
let formattedDay = moment(item.start_time).format('dddd DD')
|
||||||
|
|
||||||
if (!selection[formattedDay]) {
|
if (!selection[formattedDay]) {
|
||||||
selection[formattedDay] = []
|
selection[formattedDay] = []
|
||||||
}
|
}
|
||||||
selection[formattedDay].push(item)
|
selection[formattedDay].push(item)
|
||||||
})
|
})
|
||||||
return selection
|
return selection
|
||||||
}
|
}
|
||||||
|
|
||||||
export function convertHTMLtoMD (htmlContent) {
|
export function convertHTMLtoMD(htmlContent) {
|
||||||
const $ = load(htmlContent)
|
const $ = load(htmlContent)
|
||||||
|
|
||||||
translateNode($)
|
translateNode($)
|
||||||
|
|
||||||
return $.html()
|
return $.html()
|
||||||
}
|
}
|
||||||
|
|
||||||
export function translateNode ($) {
|
export function translateNode($) {
|
||||||
const elementsToTranslate = [
|
const elementsToTranslate = [
|
||||||
{ tag: 'h1', mdTag: '##' },
|
{tag: 'h1', mdTag: '##'},
|
||||||
{ tag: 'h2', mdTag: '###' },
|
{tag: 'h2', mdTag: '###'},
|
||||||
{ tag: 'h3', mdTag: '####' },
|
{tag: 'h3', mdTag: '####'},
|
||||||
{ tag: 'h4', mdTag: '#####' },
|
{tag: 'h4', mdTag: '#####'},
|
||||||
{ tag: 'h5', mdTag: '######' },
|
{tag: 'h5', mdTag: '######'},
|
||||||
{ tag: 'p', mdTag: '\n\n' },
|
{tag: 'p', mdTag: '\n\n'},
|
||||||
{ tag: 'ul', preserveChildren: true, transformChild: ($child) => '\n- ' + $.text($child) },
|
{tag: 'ul', preserveChildren: true, transformChild: ($child) => '\n- ' + $.text($child)},
|
||||||
{ tag: 'ol', preserveChildren: true, transformChild: ($child, idx) => `\n${idx + 1}. ${$.text($child)}` },
|
{tag: 'ol', preserveChildren: true, transformChild: ($child, idx) => `\n${idx + 1}. ${$.text($child)}`},
|
||||||
{ tag: 'a', parseAttr: ('href', link => `[${link}](${link})`) },
|
{tag: 'a', parseAttr: ('href', link => `[${link}](${link})`)},
|
||||||
{ tag: 'strong', mdFormat: '$&' },
|
{tag: 'strong', mdFormat: '$&'},
|
||||||
{ tag: 'em', mdFormat: '_$&_' },
|
{tag: 'em', mdFormat: '_$&_'},
|
||||||
{ tag: 'code', mdFormat: '`$&`' },
|
{tag: 'code', mdFormat: '`$&`'},
|
||||||
]
|
]
|
||||||
|
|
||||||
elementsToTranslate.forEach(element => {
|
elementsToTranslate.forEach(element => {
|
||||||
$(element.tag).each((_, el) => {
|
$(element.tag).each((_, el) => {
|
||||||
const $el = $(el)
|
const $el = $(el)
|
||||||
if (element.parseAttr) {
|
if (element.parseAttr) {
|
||||||
const attrVal = $el.attr(element.parseAttr[0])
|
const attrVal = $el.attr(element.parseAttr[0])
|
||||||
if (attrVal) {
|
if (attrVal) {
|
||||||
$el.replaceWith(element.parseAttr[1](attrVal))
|
$el.replaceWith(element.parseAttr[1](attrVal))
|
||||||
} else {
|
} else {
|
||||||
$el.remove()
|
$el.remove()
|
||||||
}
|
}
|
||||||
} else if (element.transformChild) {
|
} else if (element.transformChild) {
|
||||||
$el.contents().filter(() => !!this.parent()).each((_, childEl) => {
|
$el.contents().filter(() => !!this.parent()).each((_, childEl) => {
|
||||||
const transformedText = element.transformChild($(childEl), $el.index())
|
const transformedText = element.transformChild($(childEl), $el.index())
|
||||||
$el.before('\n' + transformedText)
|
$el.before('\n' + transformedText)
|
||||||
})
|
})
|
||||||
$el.remove()
|
$el.remove()
|
||||||
} else {
|
} else {
|
||||||
$el.replaceWith(element.mdTag + $el.text() + element.mdFormat)
|
$el.replaceWith(element.mdTag + $el.text() + element.mdFormat)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
export function getRequestOptions (host, port, path) {
|
export function getRequestOptions(host, port, path) {
|
||||||
const options = {
|
const options = {
|
||||||
host: host,
|
host: host,
|
||||||
port: port,
|
port: port,
|
||||||
path: path,
|
path: path,
|
||||||
method: 'GET',
|
method: 'GET',
|
||||||
headers: {
|
headers: {
|
||||||
Accept: 'application/activity+json',
|
Accept: 'application/activity+json',
|
||||||
'User-Agent': 'MyApp/1.0.0 (contact@myemail.org)',
|
'User-Agent': 'MyApp/1.0.0 (contact@myemail.org)',
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
return options
|
return options
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function sendGetRequest (options) {
|
export async function sendGetRequest(options) {
|
||||||
return new Promise((resolve, reject) => {
|
return new Promise((resolve, reject) => {
|
||||||
const req = http.request(options, (res) => {
|
const req = http.request(options, (res) => {
|
||||||
let body = Buffer.alloc(0)
|
let body = Buffer.alloc(0)
|
||||||
|
|
||||||
res.on('data', (chunk) => {
|
res.on('data', (chunk) => {
|
||||||
body = Buffer.concat([body, chunk])
|
body = Buffer.concat([body, chunk])
|
||||||
})
|
})
|
||||||
|
|
||||||
res.on('end', () => {
|
res.on('end', () => {
|
||||||
resolve(JSON.parse(body.toString()))
|
resolve(JSON.parse(body.toString()))
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
req.on('error', (err) => {
|
req.on('error', (err) => {
|
||||||
reject(err)
|
reject(err)
|
||||||
})
|
})
|
||||||
|
|
||||||
req.end()
|
req.end()
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
const splitTextIntoChunks = (text, limit) => {
|
const splitTextIntoChunks = (text, limit) => {
|
||||||
const words = text.trim().split(/\s+/g)
|
const words = text.trim().split(/\s+/g)
|
||||||
const chunks = []
|
const chunks = []
|
||||||
let currentChunk = []
|
let currentChunk = []
|
||||||
let wordCount = 0
|
let wordCount = 0
|
||||||
|
|
||||||
for (const word of words) {
|
for (const word of words) {
|
||||||
if (wordCount + word.length > limit) {
|
if (wordCount + word.length > limit) {
|
||||||
chunks.push(currentChunk.join(' '))
|
chunks.push(currentChunk.join(' '))
|
||||||
currentChunk = [word]
|
currentChunk = [word]
|
||||||
wordCount = word.length
|
wordCount = word.length
|
||||||
} else {
|
} else {
|
||||||
currentChunk.push(word)
|
currentChunk.push(word)
|
||||||
wordCount += word.length + 1
|
wordCount += word.length + 1
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (currentChunk.length > 0) {
|
if (currentChunk.length > 0) {
|
||||||
const joinedWords = currentChunk.join(' ')
|
const joinedWords = currentChunk.join(' ')
|
||||||
chunks.push(joinedWords.length <= limit ? joinedWords : joinedWords.substr(0, limit) + '...')
|
chunks.push(joinedWords.length <= limit ? joinedWords : joinedWords.substr(0, limit) + '...')
|
||||||
}
|
}
|
||||||
|
|
||||||
return chunks
|
return chunks
|
||||||
}
|
}
|
||||||
|
|
||||||
export function splitLongDescription (text, limit) {
|
export function splitLongDescription(text, limit) {
|
||||||
let chunks = splitTextIntoChunks(text, limit)
|
let chunks = splitTextIntoChunks(text, limit)
|
||||||
if (chunks) {
|
if (chunks) {
|
||||||
if (chunks[0]) {
|
if (chunks[0]) {
|
||||||
return chunks[0]
|
return chunks[0]
|
||||||
}
|
}
|
||||||
return ''
|
return ''
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -1,12 +1,8 @@
|
|||||||
import fetch from "node-fetch"
|
|
||||||
import rp from "request-promise";
|
|
||||||
import $ from "cheerio";
|
import $ from "cheerio";
|
||||||
import fs from "fs";
|
|
||||||
import path from 'path';
|
|
||||||
|
|
||||||
let url = 'https://wiki.openstreetmap.org/wiki/FR:Key:building:material';
|
let url = 'https://wiki.openstreetmap.org/wiki/FR:Key:building:material';
|
||||||
|
|
||||||
let pictureDescrFinder = '.d_image img'
|
let pictureDescrFinder = '.d_image img'
|
||||||
|
|
||||||
// rp(url).then(function (html) {
|
// rp(url).then(function (html) {
|
||||||
// getSourceOfDescriptorPageContent(html)
|
// getSourceOfDescriptorPageContent(html)
|
||||||
@ -30,7 +26,7 @@ export function getSourceOfDescriptorPageContent(HTMLcontent) {
|
|||||||
}
|
}
|
||||||
console.log(sourcesSet)
|
console.log(sourcesSet)
|
||||||
console.log(selectedPicture)
|
console.log(selectedPicture)
|
||||||
return selectedPicture;
|
return `https://wiki.openstreetmap.org${selectedPicture.trim()}`;
|
||||||
} else {
|
} else {
|
||||||
console.log("pas d'image de description dans le HTML")
|
console.log("pas d'image de description dans le HTML")
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,7 +1,15 @@
|
|||||||
|
/**
|
||||||
|
* Envoi d'une page du wiki au hasard parmi les pages en Français répertoriées dans le json all_wiki_osm.json
|
||||||
|
* Cette liste d'environ 3000 pages du wiki contient toutes les pages francophones, pas seulement les tags.
|
||||||
|
* Utiliser l'argument --force pour réellement envoyer le post avec le compte Curator.
|
||||||
|
* nécessite d'avoir le fichier .env rempli.
|
||||||
|
*/
|
||||||
import fs from 'fs';
|
import fs from 'fs';
|
||||||
import path, {dirname} from 'path';
|
import path, {dirname} from 'path';
|
||||||
import axios from 'axios';
|
import axios from 'axios';
|
||||||
import {fileURLToPath} from 'url';
|
import {fileURLToPath} from 'url';
|
||||||
|
import {getSourceOfDescriptorPageContent} from "./osm_get_description_picture.mjs";
|
||||||
|
import sendPostMastodon, {downloadImage, randomIntFromInterval} from "./libs/utils.mjs";
|
||||||
|
|
||||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||||
|
|
||||||
@ -13,10 +21,22 @@ const randomIndex = Math.floor(Math.random() * wikiArticles['elements'].length);
|
|||||||
console.log('wikiArticles[\'elements\']', wikiArticles['elements'].length)
|
console.log('wikiArticles[\'elements\']', wikiArticles['elements'].length)
|
||||||
const selectedArticle = wikiArticles['elements'][randomIndex];
|
const selectedArticle = wikiArticles['elements'][randomIndex];
|
||||||
|
|
||||||
|
console.log('selectedArticle', selectedArticle)
|
||||||
// Récupérer le titre et la description de l'article
|
// Récupérer le titre et la description de l'article
|
||||||
const title = selectedArticle.title;
|
const title = selectedArticle.title;
|
||||||
const pageId = selectedArticle.pageid;
|
const pageId = selectedArticle.pageid;
|
||||||
|
|
||||||
|
function makePostMessageFromObj(post_obj) {
|
||||||
|
return `
|
||||||
|
# Le tag OSM du jour : ${post_obj.key}=${post_obj.value} 🗺️🏷️
|
||||||
|
${post_obj.link}
|
||||||
|
|
||||||
|
${post_obj.description ? post_obj.description.trim() : ''}
|
||||||
|
${post_obj.long_desc ? post_obj.long_desc.trim() : ''}
|
||||||
|
#osm #openstreetmap #wiki #rtfw
|
||||||
|
`
|
||||||
|
}
|
||||||
|
|
||||||
// Récupérer le contenu de l'article via l'API de MediaWiki
|
// Récupérer le contenu de l'article via l'API de MediaWiki
|
||||||
const wikiApiUrl = `https://wiki.openstreetmap.org/w/api.php`;
|
const wikiApiUrl = `https://wiki.openstreetmap.org/w/api.php`;
|
||||||
const params = {
|
const params = {
|
||||||
@ -26,10 +46,23 @@ const params = {
|
|||||||
prop: 'text',
|
prop: 'text',
|
||||||
section: 0
|
section: 0
|
||||||
};
|
};
|
||||||
|
// console.log('params', params)
|
||||||
|
|
||||||
|
let message = ''
|
||||||
|
let download_description_src = ''
|
||||||
axios.get(wikiApiUrl, {params})
|
axios.get(wikiApiUrl, {params})
|
||||||
.then(response => {
|
.then(response => {
|
||||||
|
// console.log('wikiApiUrl', wikiApiUrl)
|
||||||
|
let keys = Object.keys(response.data)
|
||||||
|
// console.log('keys', keys)
|
||||||
const articleContent = response.data.parse.text['*'];
|
const articleContent = response.data.parse.text['*'];
|
||||||
|
|
||||||
|
let resultpicture = getSourceOfDescriptorPageContent(articleContent)
|
||||||
|
|
||||||
|
// console.log('resultpicture', resultpicture)
|
||||||
|
if (resultpicture === "pas d'image de description dans le HTML") {
|
||||||
|
download_description_src = `${process.cwd()}/assets/blog_posts_medias/osm_wiki_description_page.jpg`
|
||||||
|
}
|
||||||
const firstParagraph = articleContent.split('<p>')[1].split('</p>')[0];
|
const firstParagraph = articleContent.split('<p>')[1].split('</p>')[0];
|
||||||
|
|
||||||
// Sanitizer le texte
|
// Sanitizer le texte
|
||||||
@ -38,13 +71,67 @@ axios.get(wikiApiUrl, {params})
|
|||||||
return String.fromCharCode(code);
|
return String.fromCharCode(code);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
console.log('image', download_description_src)
|
||||||
|
console.log('-----------------')
|
||||||
// Générer le message
|
// Générer le message
|
||||||
const message = `Je vous recommande de lire l'article "${title}" sur le wiki d'OpenStreetMap :
|
message = `Le page #OSM du jour : ${title}.
|
||||||
|
https://wiki.openstreetmap.org/wiki/${title}
|
||||||
|
|
||||||
${sanitizedTextWithoutEntities}
|
${sanitizedTextWithoutEntities}
|
||||||
Lire la suite sur : https://wiki.openstreetmap.org/wiki/${title}`;
|
|
||||||
|
#openstreetmap #wiki #RTFW
|
||||||
|
`;
|
||||||
|
|
||||||
console.log(message);
|
console.log(message);
|
||||||
|
|
||||||
})
|
})
|
||||||
.catch(error => {
|
.catch(error => {
|
||||||
console.error(error);
|
console.error(error);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
||||||
|
export function sendMessageWikiOSMOfTheDay(message, download_description_src) {
|
||||||
|
|
||||||
|
|
||||||
|
let configPost = {
|
||||||
|
author: 'curator',
|
||||||
|
// document.querySelector('.d_image img').attributes['src']
|
||||||
|
// image: '/home/tykayn/www/multi-account-post-schedule-mastodon/assets/osm_post_' + randomIntFromInterval(1 , 5) + '.jpg',
|
||||||
|
image: `${process.cwd()}/assets/blog_posts_medias/assets/osm_post_${randomIntFromInterval(1, 5)}.jpg`,
|
||||||
|
message,
|
||||||
|
}
|
||||||
|
if (download_description_src) {
|
||||||
|
let filePathImage = `${process.cwd()}/assets/blog_posts_medias/osm_wiki_description_page.jpg`
|
||||||
|
if (download_description_src) {
|
||||||
|
console.log("firstPictureSource found", download_description_src)
|
||||||
|
|
||||||
|
// check if picture already exist
|
||||||
|
console.log('on récupère l image de description : ', filePathImage)
|
||||||
|
downloadImage(download_description_src, filePathImage)
|
||||||
|
.then((res) => {
|
||||||
|
// suite du poste avec upload d'image
|
||||||
|
|
||||||
|
console.log('média téléchargé, on envoie le post')
|
||||||
|
configPost.image = filePathImage;
|
||||||
|
|
||||||
|
sendPostMastodon(configPost)
|
||||||
|
|
||||||
|
},
|
||||||
|
(err) => {
|
||||||
|
console.log('pas dimage trouvée pour l URL ', download_description_src, err)
|
||||||
|
sendPostMastodon(configPost)
|
||||||
|
}
|
||||||
|
)
|
||||||
|
.catch((err) => {
|
||||||
|
console.log('erreur avec cette URL ', download_description_src, err)
|
||||||
|
sendPostMastodon(configPost)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
} else {
|
||||||
|
console.log('no image description')
|
||||||
|
sendPostMastodon(configPost)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
sendMessageWikiOSMOfTheDay(message, download_description_src)
|
||||||
@ -1,6 +1,8 @@
|
|||||||
// https://www.mediawiki.org/wiki/Manual:Random_page
|
// https://www.mediawiki.org/wiki/Manual:Random_page
|
||||||
/**
|
/**
|
||||||
* Post de page aléatoire du wiki osm avec le compte curator
|
* Post de page aléatoire du wiki osm avec le compte curator
|
||||||
|
* utiliser l'argument --force pour réellement envoyer le post avec le compte Curator.
|
||||||
|
* nécessite d'avoir le fichier .env rempli.
|
||||||
*/
|
*/
|
||||||
import fetch from "node-fetch"
|
import fetch from "node-fetch"
|
||||||
import rp from "request-promise";
|
import rp from "request-promise";
|
||||||
@ -24,188 +26,188 @@ let foundExistingWikiPageInFrench = false;
|
|||||||
|
|
||||||
function getRandomWikiOSMPage() {
|
function getRandomWikiOSMPage() {
|
||||||
|
|
||||||
// get all FR tags
|
// get all FR tags
|
||||||
// const url = "https://wiki.openstreetmap.org/w/index.php?title=Special:Categories&offset=FR&limit=500"
|
// const url = "https://wiki.openstreetmap.org/w/index.php?title=Special:Categories&offset=FR&limit=500"
|
||||||
const url = "https://wiki.openstreetmap.org/wiki/Special:AllPages?from=&to=&namespace=202&limit=500"
|
const url = "https://wiki.openstreetmap.org/wiki/Special:AllPages?from=&to=&namespace=202&limit=500"
|
||||||
// get a random page wich is not a discussion
|
// get a random page wich is not a discussion
|
||||||
// const url = "https://wiki.openstreetmap.org/wiki/Special:Random?namespace=all-discussions%3B1%3B2%3B3%3B5%3B7%3B9%3B11%3B120%3B121%3B122%3B123%3B201%3B203&invert=1&limit=10&days=1&enhanced=1&namespace__202_color=c1&urlversion=2"
|
// const url = "https://wiki.openstreetmap.org/wiki/Special:Random?namespace=all-discussions%3B1%3B2%3B3%3B5%3B7%3B9%3B11%3B120%3B121%3B122%3B123%3B201%3B203&invert=1&limit=10&days=1&enhanced=1&namespace__202_color=c1&urlversion=2"
|
||||||
rp(url)
|
rp(url)
|
||||||
.then(function (html) {
|
.then(function (html) {
|
||||||
//success!
|
//success!
|
||||||
const wikiUrls = [];
|
const wikiUrls = [];
|
||||||
const filteredHtml = $('.mw-spcontent ul li a', html)
|
const filteredHtml = $('.mw-spcontent ul li a', html)
|
||||||
console.log("filteredHtml.length", filteredHtml.length)
|
console.log("filteredHtml.length", filteredHtml.length)
|
||||||
// .filter(elem => {
|
// .filter(elem => {
|
||||||
// return (elem.attribs.title.indexOf('description for') === -1)
|
// return (elem.attribs.title.indexOf('description for') === -1)
|
||||||
// })
|
// })
|
||||||
for (let i = 0; i < filteredHtml.length; i++) {
|
for (let i = 0; i < filteredHtml.length; i++) {
|
||||||
// const contentCurated = filteredHtml[i]
|
// const contentCurated = filteredHtml[i]
|
||||||
const contentCurated = filteredHtml[i].attribs.href
|
const contentCurated = filteredHtml[i].attribs.href
|
||||||
wikiUrls.push(contentCurated);
|
wikiUrls.push(contentCurated);
|
||||||
}
|
}
|
||||||
console.log(wikiUrls);
|
console.log(wikiUrls);
|
||||||
return wikiUrls;
|
return wikiUrls;
|
||||||
})
|
})
|
||||||
.catch(function (err) {
|
.catch(function (err) {
|
||||||
//handle error
|
//handle error
|
||||||
console.error(err)
|
console.error(err)
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
export default function getElementCartographique() {
|
export default function getElementCartographique() {
|
||||||
if (foundExistingWikiPageInFrench) {
|
if (foundExistingWikiPageInFrench) {
|
||||||
console.log("already found foundExistingWikiPageInFrench")
|
console.log("already found foundExistingWikiPageInFrench")
|
||||||
return;
|
return;
|
||||||
}
|
|
||||||
const url = "https://wiki.openstreetmap.org/wiki/FR:%C3%89l%C3%A9ments_cartographiques"
|
|
||||||
const titleLink = "#toc a"
|
|
||||||
const listElementSelector = "#toc a"
|
|
||||||
// prendre un lien dans la table des matières de la page, extraire l'id du lien.
|
|
||||||
// prendre une ligne de tableau au hasard correspondant à l'ID de la section
|
|
||||||
const titleSelector = ""
|
|
||||||
|
|
||||||
|
|
||||||
return fs.readFile(__dirname + '/assets/documents/elements_cartographiques.html', 'utf8', function (err, html) {
|
|
||||||
if (err) {
|
|
||||||
return console.log(err);
|
|
||||||
}
|
}
|
||||||
// console.log(data);
|
const url = "https://wiki.openstreetmap.org/wiki/FR:%C3%89l%C3%A9ments_cartographiques"
|
||||||
|
const titleLink = "#toc a"
|
||||||
|
const listElementSelector = "#toc a"
|
||||||
|
// prendre un lien dans la table des matières de la page, extraire l'id du lien.
|
||||||
|
// prendre une ligne de tableau au hasard correspondant à l'ID de la section
|
||||||
|
const titleSelector = ""
|
||||||
|
|
||||||
|
|
||||||
//success!
|
return fs.readFile(__dirname + '/assets/documents/elements_cartographiques.html', 'utf8', function (err, html) {
|
||||||
const wikiUrls = [];
|
if (err) {
|
||||||
// const listOfLinks = $(titleLink, html)
|
return console.log(err);
|
||||||
// console.log("filteredHtml.length", listOfLinks.length)
|
}
|
||||||
// const selectedTocLink = getRandomElementOfArray(listOfLinks).attribs.href;
|
// console.log(data);
|
||||||
// console.log("selectedTocLink", selectedTocLink)
|
|
||||||
|
|
||||||
const listOfTableRows = $('.wikitable', html);
|
|
||||||
let keys = Object.keys(listOfTableRows);
|
|
||||||
|
|
||||||
let randNumber = 0;
|
//success!
|
||||||
if (selectionOverrideOfSectionTable) {
|
const wikiUrls = [];
|
||||||
randNumber = selectionOverrideOfSectionTable
|
// const listOfLinks = $(titleLink, html)
|
||||||
} else {
|
// console.log("filteredHtml.length", listOfLinks.length)
|
||||||
randNumber = randomIntFromInterval(0, keys.length)
|
// const selectedTocLink = getRandomElementOfArray(listOfLinks).attribs.href;
|
||||||
}
|
// console.log("selectedTocLink", selectedTocLink)
|
||||||
|
|
||||||
console.log("rand", randNumber)
|
const listOfTableRows = $('.wikitable', html);
|
||||||
const foundLine = $(listOfTableRows[randNumber]).find('tr');
|
let keys = Object.keys(listOfTableRows);
|
||||||
|
|
||||||
// console.log("randomTable",foundTable)
|
let randNumber = 0;
|
||||||
|
if (selectionOverrideOfSectionTable) {
|
||||||
|
randNumber = selectionOverrideOfSectionTable
|
||||||
|
} else {
|
||||||
|
randNumber = randomIntFromInterval(0, keys.length)
|
||||||
|
}
|
||||||
|
|
||||||
// on regarde les lignes du tableau
|
console.log("rand", randNumber)
|
||||||
keys = Object.keys(foundLine);
|
const foundLine = $(listOfTableRows[randNumber]).find('tr');
|
||||||
if (selectionOverrideOfLineTable) {
|
|
||||||
randNumber = selectionOverrideOfLineTable
|
|
||||||
} else {
|
|
||||||
randNumber = randomIntFromInterval(0, keys.length)
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log("lignes de tableau", keys.length)
|
// console.log("randomTable",foundTable)
|
||||||
console.log("ligne sélectionnée", randNumber)
|
|
||||||
let cells = $(foundLine[randNumber]).find('td')
|
|
||||||
|
|
||||||
keys = Object.keys(cells);
|
// on regarde les lignes du tableau
|
||||||
// console.log("cellules trouvées", keys.length)
|
keys = Object.keys(foundLine);
|
||||||
|
if (selectionOverrideOfLineTable) {
|
||||||
|
randNumber = selectionOverrideOfLineTable
|
||||||
|
} else {
|
||||||
|
randNumber = randomIntFromInterval(0, keys.length)
|
||||||
|
}
|
||||||
|
|
||||||
console.log("\n")
|
console.log("lignes de tableau", keys.length)
|
||||||
let configPost = {
|
console.log("ligne sélectionnée", randNumber)
|
||||||
key: '',
|
let cells = $(foundLine[randNumber]).find('td')
|
||||||
value: '',
|
|
||||||
description: '',
|
|
||||||
long_desc: '',
|
|
||||||
}
|
|
||||||
cells.each((i, element) => {
|
|
||||||
// console.log("cell element", i, $(element).text().trim())
|
|
||||||
if (i === 0) {
|
|
||||||
configPost.key = $(element).text().trim();
|
|
||||||
}
|
|
||||||
if (i === 1) {
|
|
||||||
configPost.value = $(element).text().trim();
|
|
||||||
}
|
|
||||||
if (i === 3) {
|
|
||||||
configPost.description = $(element).text().replace(' ', '').replace(' ', '').trim();
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
if (!configPost.key) {
|
keys = Object.keys(cells);
|
||||||
console.error("ERROR no key found", $(cells).html())
|
// console.log("cellules trouvées", keys.length)
|
||||||
console.error('no key BOOOOOOOOOOH');
|
|
||||||
getElementCartographique();
|
console.log("\n")
|
||||||
return;
|
let configPost = {
|
||||||
}
|
key: '',
|
||||||
|
value: '',
|
||||||
|
description: '',
|
||||||
|
long_desc: '',
|
||||||
|
}
|
||||||
|
cells.each((i, element) => {
|
||||||
|
// console.log("cell element", i, $(element).text().trim())
|
||||||
|
if (i === 0) {
|
||||||
|
configPost.key = $(element).text().trim();
|
||||||
|
}
|
||||||
|
if (i === 1) {
|
||||||
|
configPost.value = $(element).text().trim();
|
||||||
|
}
|
||||||
|
if (i === 3) {
|
||||||
|
configPost.description = $(element).text().replace(' ', '').replace(' ', '').trim();
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
if (!configPost.key) {
|
||||||
|
console.error("ERROR no key found", $(cells).html())
|
||||||
|
console.error('no key BOOOOOOOOOOH');
|
||||||
|
getElementCartographique();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
// aller chercher le lien
|
// aller chercher le lien
|
||||||
// si la clé est une description avec des espaces, faire un lien vers la Key
|
// si la clé est une description avec des espaces, faire un lien vers la Key
|
||||||
let pageLink = `FR:Tag:${configPost.key}=${configPost.value}`
|
let pageLink = `FR:Tag:${configPost.key}=${configPost.value}`
|
||||||
|
|
||||||
if (configPost.value.indexOf(' ') !== -1) {
|
if (configPost.value.indexOf(' ') !== -1) {
|
||||||
pageLink = `FR:Key:${configPost.key}`
|
pageLink = `FR:Key:${configPost.key}`
|
||||||
}
|
}
|
||||||
configPost.link = `https://wiki.openstreetmap.org/wiki/${pageLink}`
|
configPost.link = `https://wiki.openstreetmap.org/wiki/${pageLink}`
|
||||||
|
|
||||||
const url = configPost.link
|
const url = configPost.link
|
||||||
// get a random page wich is not a discussion
|
// get a random page wich is not a discussion
|
||||||
|
|
||||||
// certaines valeurs ont des espaces, donc pas utilisable en url
|
// certaines valeurs ont des espaces, donc pas utilisable en url
|
||||||
if (configPost.value.indexOf(' ') === -1 || configPost.value.indexOf('<') === -1 || configPost.value.indexOf('>') === -1) {
|
if (configPost.value.indexOf(' ') === -1 || configPost.value.indexOf('<') === -1 || configPost.value.indexOf('>') === -1) {
|
||||||
|
|
||||||
console.log("url", url)
|
console.log("url", url)
|
||||||
rp(url).then(function (html) {
|
rp(url).then(function (html) {
|
||||||
//success!
|
//success!
|
||||||
const descriptionStrophe = '';
|
const descriptionStrophe = '';
|
||||||
const filteredHtml = $('.mw-parser-output', html).find('p')
|
const filteredHtml = $('.mw-parser-output', html).find('p')
|
||||||
|
|
||||||
// trouver l'image de description
|
// trouver l'image de description
|
||||||
configPost.download_description_src = getSourceOfDescriptorPageContent(html)
|
configPost.download_description_src = getSourceOfDescriptorPageContent(html)
|
||||||
|
|
||||||
console.log("filteredHtml", filteredHtml.length)
|
console.log("filteredHtml", filteredHtml.length)
|
||||||
configPost.long_desc = $(filteredHtml).text().substring(0, 250)
|
configPost.long_desc = $(filteredHtml).text().substring(0, 250)
|
||||||
console.log("filteredHtml", configPost.long_desc)
|
console.log("filteredHtml", configPost.long_desc)
|
||||||
|
|
||||||
// image de description:
|
// image de description:
|
||||||
// let imgSelector = ".description a.image img"
|
// let imgSelector = ".description a.image img"
|
||||||
console.log("✅ cette page existe bien en Français sur le wiki OSM")
|
console.log("✅ cette page existe bien en Français sur le wiki OSM")
|
||||||
foundExistingWikiPageInFrench = true;
|
foundExistingWikiPageInFrench = true;
|
||||||
sendMessageWikiTagOfTheDay(makePostMessageFromObj(configPost), configPost.download_description_src)
|
sendMessageWikiOSMOfTheDay(makePostMessageFromObj(configPost), configPost.download_description_src)
|
||||||
|
|
||||||
|
|
||||||
}, (err) => {
|
}, (err) => {
|
||||||
console.error(err.statusCode)
|
console.error(err.statusCode)
|
||||||
if (err.statusCode == 404) {
|
if (err.statusCode == 404) {
|
||||||
console.log("\n Cette page n'existe pas encore, c'est le moment de la créer.", `https://wiki.openstreetmap.org/w/index.php?title=FR:${configPost.key}=${configPost.value}`) //FR:Tag:boundary%3Dborder_zone&action=edit)
|
console.log("\n Cette page n'existe pas encore, c'est le moment de la créer.", `https://wiki.openstreetmap.org/w/index.php?title=FR:${configPost.key}=${configPost.value}`) //FR:Tag:boundary%3Dborder_zone&action=edit)
|
||||||
console.log("Essayez de voir si la version en Anglais existe.")
|
console.log("Essayez de voir si la version en Anglais existe.")
|
||||||
let englishPage = `https://wiki.openstreetmap.org/wiki/Tag:${configPost.key}=${configPost.value}`
|
let englishPage = `https://wiki.openstreetmap.org/wiki/Tag:${configPost.key}=${configPost.value}`
|
||||||
let englishKey = `https://wiki.openstreetmap.org/wiki/Key:${configPost.key}`
|
let englishKey = `https://wiki.openstreetmap.org/wiki/Key:${configPost.key}`
|
||||||
console.log("-> ", englishPage)
|
console.log("-> ", englishPage)
|
||||||
checkExistenceOfWebPage(englishPage).catch(err => {
|
checkExistenceOfWebPage(englishPage).catch(err => {
|
||||||
checkExistenceOfWebPage(englishKey)
|
checkExistenceOfWebPage(englishKey)
|
||||||
});
|
});
|
||||||
console.log("bon spa tout ça on essaie une autre ligne")
|
console.log("bon spa tout ça on essaie une autre ligne")
|
||||||
getElementCartographique()
|
getElementCartographique()
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
console.log("result", configPost)
|
||||||
|
rp(configPost.link).then((result) => {
|
||||||
|
console.log("oui cette page existe!", url)
|
||||||
|
sendMessageWikiOSMOfTheDay(makePostMessageFromObj(result))
|
||||||
|
}, (err) => {
|
||||||
|
console.log("hé non. WTF ?")
|
||||||
|
// getElementCartographique()
|
||||||
|
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
})
|
|
||||||
} else {
|
|
||||||
console.log("result", configPost)
|
|
||||||
rp(configPost.link).then((result) => {
|
|
||||||
console.log("oui cette page existe!", url)
|
|
||||||
sendMessageWikiTagOfTheDay(makePostMessageFromObj(result))
|
|
||||||
}, (err) => {
|
|
||||||
console.log("hé non. WTF ?")
|
|
||||||
// getElementCartographique()
|
|
||||||
|
|
||||||
})
|
return configPost;
|
||||||
|
|
||||||
|
});
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
return configPost;
|
|
||||||
|
|
||||||
});
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -215,19 +217,19 @@ export default function getElementCartographique() {
|
|||||||
* @returns {*}
|
* @returns {*}
|
||||||
*/
|
*/
|
||||||
function checkExistenceOfWebPage(url) {
|
function checkExistenceOfWebPage(url) {
|
||||||
if (foundExistingWikiPageInFrench) {
|
if (foundExistingWikiPageInFrench) {
|
||||||
console.log("already found foundExistingWikiPageInFrench, no need to check other page", url)
|
console.log("already found foundExistingWikiPageInFrench, no need to check other page", url)
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
return rp(url).then((html) => {
|
return rp(url).then((html) => {
|
||||||
console.log("oui cette page existe!", url)
|
console.log("oui cette page existe!", url)
|
||||||
}, (err) => {
|
}, (err) => {
|
||||||
console.log("hé non. WTF ?")
|
console.log("hé non. WTF ?")
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
function makePostMessageFromObj(post_obj) {
|
function makePostMessageFromObj(post_obj) {
|
||||||
return `
|
return `
|
||||||
# Le tag OSM du jour : ${post_obj.key}=${post_obj.value} 🗺️🏷️
|
# Le tag OSM du jour : ${post_obj.key}=${post_obj.value} 🗺️🏷️
|
||||||
${post_obj.link}
|
${post_obj.link}
|
||||||
|
|
||||||
@ -238,93 +240,85 @@ ${post_obj.description ? post_obj.description.trim() : ''}
|
|||||||
}
|
}
|
||||||
|
|
||||||
function getQuery() {
|
function getQuery() {
|
||||||
// var url = "https://wiki.openstreetmap.org/w/api.php";
|
// var url = "https://wiki.openstreetmap.org/w/api.php";
|
||||||
var url = "https://fr.wikipedia.org/w/api.php";
|
var url = "https://fr.wikipedia.org/w/api.php";
|
||||||
var params = {
|
var params = {
|
||||||
action: "query",
|
action: "query",
|
||||||
format: "json",
|
format: "json",
|
||||||
list: "random",
|
list: "random",
|
||||||
enhanced: "1",
|
enhanced: "1",
|
||||||
hiderobot: "1",
|
hiderobot: "1",
|
||||||
urlversion: "2",
|
urlversion: "2",
|
||||||
invert: "1",
|
invert: "1",
|
||||||
rnnamespace: "all-discussions%3B1%3B2%3B3%3B5%3B7%3B9%3B11%3B120%3B121%3B122%3B123%3B201%3B203", // WIKI et FR:
|
rnnamespace: "all-discussions%3B1%3B2%3B3%3B5%3B7%3B9%3B11%3B120%3B121%3B122%3B123%3B201%3B203", // WIKI et FR:
|
||||||
lang: "FR",
|
lang: "FR",
|
||||||
rnlimit: "5"
|
rnlimit: "5"
|
||||||
};
|
};
|
||||||
|
|
||||||
url = url + "?&origin=*";
|
url = url + "?&origin=*";
|
||||||
Object.keys(params).forEach(function (key) {
|
Object.keys(params).forEach(function (key) {
|
||||||
url += "&" + key + "=" + params[key];
|
url += "&" + key + "=" + params[key];
|
||||||
});
|
});
|
||||||
|
|
||||||
return fetch(url)
|
return fetch(url)
|
||||||
.then(function (response) {
|
.then(function (response) {
|
||||||
return response.json();
|
return response.json();
|
||||||
})
|
})
|
||||||
.then(function (response) {
|
.then(function (response) {
|
||||||
const randoms = response.query.random;
|
const randoms = response.query.random;
|
||||||
|
|
||||||
for (let ii = 0; ii < randoms.length; ii++) {
|
for (let ii = 0; ii < randoms.length; ii++) {
|
||||||
// console.log(randoms[ii].title);
|
// console.log(randoms[ii].title);
|
||||||
console.log("randoms[ii]", randoms[ii])
|
console.log("randoms[ii]", randoms[ii])
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.catch(function (error) {
|
.catch(function (error) {
|
||||||
console.log(error);
|
console.log(error);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function sendMessageWikiOSMOfTheDay(message, download_description_src) {
|
||||||
// run
|
|
||||||
|
|
||||||
const res = getElementCartographique()
|
|
||||||
|
|
||||||
// console.log("res", res)
|
|
||||||
|
|
||||||
|
|
||||||
function sendMessageWikiTagOfTheDay(message, download_description_src) {
|
let configPost = {
|
||||||
|
author: 'curator',
|
||||||
|
// document.querySelector('.d_image img').attributes['src']
|
||||||
let configPost = {
|
// image: '/home/tykayn/www/multi-account-post-schedule-mastodon/assets/osm_post_' + randomIntFromInterval(1 , 5) + '.jpg',
|
||||||
author: 'curator',
|
image: `${process.cwd()}/assets/blog_posts_medias/assets/osm_post_${randomIntFromInterval(1, 5)}.jpg`,
|
||||||
// document.querySelector('.d_image img').attributes['src']
|
message,
|
||||||
// image: '/home/tykayn/www/multi-account-post-schedule-mastodon/assets/osm_post_' + randomIntFromInterval(1 , 5) + '.jpg',
|
|
||||||
image: `${process.cwd()}/assets/blog_posts_medias/assets/osm_post_${randomIntFromInterval(1, 5)}.jpg`,
|
|
||||||
message,
|
|
||||||
}
|
|
||||||
if (download_description_src) {
|
|
||||||
let filePathImage = `${process.cwd()}/assets/blog_posts_medias/osm_wiki_description_page.jpg`
|
|
||||||
if (download_description_src) {
|
|
||||||
console.log("firstPictureSource found", download_description_src)
|
|
||||||
|
|
||||||
// check if picture already exist
|
|
||||||
console.log('on récupère l image de description : ', filePathImage)
|
|
||||||
downloadImage(download_description_src, filePathImage)
|
|
||||||
.then((res) => {
|
|
||||||
// suite du poste avec upload d'image
|
|
||||||
|
|
||||||
console.log('média téléchargé, on envoie le post')
|
|
||||||
configPost.image = filePathImage;
|
|
||||||
|
|
||||||
sendPostMastodon(configPost)
|
|
||||||
|
|
||||||
},
|
|
||||||
(err) => {
|
|
||||||
console.log('pas dimage trouvée pour l URL ', download_description_src, err)
|
|
||||||
sendPostMastodon(configPost)
|
|
||||||
}
|
|
||||||
)
|
|
||||||
.catch((err) => {
|
|
||||||
console.log('erreur avec cette URL ', download_description_src, err)
|
|
||||||
sendPostMastodon(configPost)
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
if (download_description_src) {
|
||||||
|
let filePathImage = `${process.cwd()}/assets/blog_posts_medias/osm_wiki_description_page.jpg`
|
||||||
|
if (download_description_src) {
|
||||||
|
console.log("firstPictureSource found", download_description_src)
|
||||||
|
|
||||||
} else {
|
// check if picture already exist
|
||||||
console.log('no image description')
|
console.log('on récupère l image de description : ', filePathImage)
|
||||||
sendPostMastodon(configPost)
|
downloadImage(download_description_src, filePathImage)
|
||||||
}
|
.then((res) => {
|
||||||
|
// suite du poste avec upload d'image
|
||||||
|
|
||||||
|
console.log('média téléchargé, on envoie le post')
|
||||||
|
configPost.image = filePathImage;
|
||||||
|
|
||||||
|
sendPostMastodon(configPost)
|
||||||
|
|
||||||
|
},
|
||||||
|
(err) => {
|
||||||
|
console.log('pas dimage trouvée pour l URL ', download_description_src, err)
|
||||||
|
sendPostMastodon(configPost)
|
||||||
|
}
|
||||||
|
)
|
||||||
|
.catch((err) => {
|
||||||
|
console.log('erreur avec cette URL ', download_description_src, err)
|
||||||
|
sendPostMastodon(configPost)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
} else {
|
||||||
|
console.log('no image description')
|
||||||
|
sendPostMastodon(configPost)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user