refacto functions of utils in split concerns, add wips to recommend people

This commit is contained in:
Tykayn 2023-03-04 22:53:12 +01:00 committed by tykayn
parent 5f7231ca68
commit db339d4f52
20 changed files with 227 additions and 127 deletions

3
.gitignore vendored
View File

@ -3,7 +3,8 @@
.yarn .yarn
node_modules node_modules
mastodon_multi_accounts.db mastodon_multi_accounts.db
#assets/documents/*.json assets/documents/abonnements_tykayn.json
assets/documents/following_accounts.csv
#assets/documents/*.rss #assets/documents/*.rss
#assets/documents/*.html #assets/documents/*.html
assets/pictures/meme/not_published/* assets/pictures/meme/not_published/*

View File

@ -3,7 +3,7 @@ import sendPostMastodon, {
getArticlesFromDaysInRssFeed, getArticlesFromDaysInRssFeed,
makeTitleContentFromRss, makeTitleContentFromRss,
parseRssFeed, parseRssFeed,
} from "./utils.mjs"; } from "./libs/utils.mjs";
const afis_rss_feed_url = 'https://afis.org/rss.xml'; const afis_rss_feed_url = 'https://afis.org/rss.xml';
const days_back_in_rss_feed = 7; const days_back_in_rss_feed = 7;

View File

@ -1,7 +1,8 @@
import sendPostMastodon from "./utils.mjs"; import sendPostMastodon from "./libs/utils.mjs";
import * as coming_events_json from './../assets/documents/agendadulibre_coming_openstreetmap.json' assert { type: "json" }; import * as coming_events_json from './../assets/documents/agendadulibre_coming_openstreetmap.json' assert { type: "json" };
import {getArticlesFromDaysInJson} from "./utils.mjs";
import {getArticlesFromDaysInJson} from './libs/rss_management.mjs'
const days_filter = 7; const days_filter = 7;

View File

@ -5,7 +5,7 @@ import sendPostMastodon, {
findFirstImageInContent, findFirstImageInContent,
findPictureAndSendPost, findPictureAndSendPost,
getRandomElementOfArray getRandomElementOfArray
} from './utils.mjs' } from './libs/utils.mjs'
const __dirname = path.resolve(); const __dirname = path.resolve();
const tkpostsjson = JSON.parse(fs.readFileSync(__dirname + "/assets/documents/cipherbliss_tkwp_posts.json", 'utf-8')) const tkpostsjson = JSON.parse(fs.readFileSync(__dirname + "/assets/documents/cipherbliss_tkwp_posts.json", 'utf-8'))

View File

@ -1,7 +1,9 @@
import * as coming_events_json from './../assets/documents/idf_agendadulibre_coming.json' assert { type: "json" } ; import * as coming_events_json from './../assets/documents/idf_agendadulibre_coming.json' assert { type: "json" } ;
import sendPostMastodon, {getArticlesFromDaysInJson, filterRegionAgendaDuLibreEvents, groupEventsByDay} from "./utils.mjs"; import sendPostMastodon, { filterRegionAgendaDuLibreEvents, groupEventsByDay} from "./libs/utils.mjs";
import moment from 'moment'; import moment from 'moment';
import {diffDaysBetweenTwoDates} from "./utils.mjs";
import {getArticlesFromDaysInJson,diffDaysBetweenTwoDates} from './libs/rss_management.mjs'

View File

@ -1,4 +1,4 @@
import {createMastoFetcherWithAuthorLogin} from "./utils.mjs"; import {createMastoFetcherWithAuthorLogin} from "./libs/utils.mjs";
import moment from 'moment' import moment from 'moment'
moment.locale('fr') moment.locale('fr')
@ -19,6 +19,7 @@ console.log('get mentions on all accounts managed')
export async function getMentionsOfAccount(accountToFetch) { export async function getMentionsOfAccount(accountToFetch) {
console.log('\n fetching account: ', accountToFetch)
const masto = createMastoFetcherWithAuthorLogin(accountToFetch) const masto = createMastoFetcherWithAuthorLogin(accountToFetch)
let results = {} let results = {}
@ -54,8 +55,9 @@ export async function getMentionsOfAccount(accountToFetch) {
const accounts = [ const accounts = [
'qzine', 'qzine',
// 'tykayn', 'tykayn',
// 'curator', 'curator',
'kurator',
// 'modominem', // 'modominem',
'voixdunucleaire', 'voixdunucleaire',
// 'voicesofnuclear', // 'voicesofnuclear',

View File

View File

@ -0,0 +1,84 @@
import Parser from 'rss-parser';
import {diffDaysBetweenTwoDates} from "./utils";
/**
* fetch and return a parsed rss feed
* @param url
* @returns {Promise<*>}
*/
export async function parseRssFeed(url){
let parsedFeed = await parser.parseURL(url);
console.log(parsedFeed.title);
parsedFeed.items.forEach((item) => {
console.log(item.title);
});
return parsedFeed
}
/**
* get a subset of rss parsed items
* @param parsedRssFeed
* @param days
* @returns {T[]}
*/
export function getArticlesFromDaysInRssFeed(parsedRssFeed , days=7) {
// return parsedRssFeed.items.splice(0,days);
return parsedRssFeed.items.splice(0,6);
}
export function makeTitleContentFromRss(parsedFeed) {
let content = ''
let counter = 0;
if (!parsedFeed.length) {
return '';
}
parsedFeed.forEach((item) => {
// if (!counter) {
// console.log('first item', item)
// }
let cleanedTitle= item.title.replace('[','').trim()
cleanedTitle= cleanedTitle.replace(']','').trim()
content += `\n* [${cleanedTitle}](${item.link}) `;
content += `\n ${item.isoDate.substring(0,10)} : ${item.contentSnippet.split('\n')[0]} \n`;
counter++;
});
return content;
}
/**
* get a subset of articles in a specified range number of days in the past
* @param json_content
* @param nbOfDays
* @returns {[]}
*/
export function getArticlesFromDaysInJson(json_content,nbOfDays){
let today = new Date()
let selectedEvents = [];
console.log('getArticlesFromDaysInJson: filtre nbOfDays', nbOfDays)
console.log('events ', json_content.length)
json_content.forEach((item) => {
let differenceDays = diffDaysBetweenTwoDates(new Date(item.start_time) ,today);
if( differenceDays < nbOfDays){
console.log('on garde', differenceDays,item.city, item.title)
selectedEvents.push(item)
}
// else{
// console.log('on ne garde pas', differenceDays,item.city, item.title)
// }
})
return selectedEvents
}

View File

@ -5,6 +5,7 @@ import fs from 'fs';
import https from 'https'; import https from 'https';
let local_node_env_conf = dotenv.config() let local_node_env_conf = dotenv.config()
const myArgs = process.argv.slice(2); const myArgs = process.argv.slice(2);
export const reallySendPost = hasCliArgument('--force'); export const reallySendPost = hasCliArgument('--force');
@ -57,6 +58,7 @@ export function createMastoFetcherWithAuthorLogin(userNickName){
}); });
return masto; return masto;
} }
/** /**
* send post to mastodon with config * send post to mastodon with config
* @param config * @param config
@ -160,8 +162,7 @@ export default function sendPostMastodon(config) {
} }
// Slugify a string // Slugify a string
export function slugify(str) export function slugify(str) {
{
str = str.replace(/^\s+|\s+$/g, ''); str = str.replace(/^\s+|\s+$/g, '');
// Make the string lowercase // Make the string lowercase
@ -250,6 +251,7 @@ function clearLink(linkString){
return linkString return linkString
} }
/** /**
* usage: * usage:
* downloadImage('https://upload.wikimedia.org/wikipedia/en/thumb/7/7d/Lenna_%28test_image%29.png/440px-Lenna_%28test_image%29.png', 'lena.png') * downloadImage('https://upload.wikimedia.org/wikipedia/en/thumb/7/7d/Lenna_%28test_image%29.png/440px-Lenna_%28test_image%29.png', 'lena.png')
@ -297,6 +299,7 @@ export function CropPicture(pictureName, width = 500, height = 300) {
if (err) console.log(err); if (err) console.log(err);
}); });
} }
/** /**
* prendre un post parmi tous ceux du blog, dans ceux qui ont été publiés * prendre un post parmi tous ceux du blog, dans ceux qui ont été publiés
* @returns {*} * @returns {*}
@ -360,7 +363,6 @@ export function findPictureAndSendPost(postContent, configPost){
} }
/** /**
* find cli argument * find cli argument
* @param argument * @param argument
@ -371,85 +373,11 @@ export function hasCliArgument(argument){
} }
import Parser from 'rss-parser';
import moment from "moment"; import moment from "moment";
let parser = new Parser(); let parser = new Parser();
/**
* fetch and return a parsed rss feed
* @param url
* @returns {Promise<*>}
*/
export async function parseRssFeed(url){
let parsedFeed = await parser.parseURL(url);
console.log(parsedFeed.title);
parsedFeed.items.forEach((item) => {
console.log(item.title);
});
return parsedFeed
}
/**
* get a subset of rss parsed items
* @param parsedRssFeed
* @param days
* @returns {T[]}
*/
export function getArticlesFromDaysInRssFeed(parsedRssFeed , days=7) {
// return parsedRssFeed.items.splice(0,days);
return parsedRssFeed.items.splice(0,6);
}
export function makeTitleContentFromRss(parsedFeed) {
let content = ''
let counter = 0;
if (!parsedFeed.length) {
return '';
}
parsedFeed.forEach((item) => {
// if (!counter) {
// console.log('first item', item)
// }
let cleanedTitle= item.title.replace('[','').trim()
cleanedTitle= cleanedTitle.replace(']','').trim()
content += `\n* [${cleanedTitle}](${item.link}) `;
content += `\n ${item.isoDate.substring(0,10)} : ${item.contentSnippet.split('\n')[0]} \n`;
counter++;
});
return content;
}
export function getArticlesFromDaysInJson(json_content,nbOfDays){
let today = new Date()
let selectedEvents = [];
console.log('getArticlesFromDaysInJson: filtre nbOfDays', nbOfDays)
console.log('events ', json_content.length)
json_content.forEach((item) => {
let differenceDays = diffDaysBetweenTwoDates(new Date(item.start_time) ,today);
if( differenceDays < nbOfDays){
console.log('on garde', differenceDays,item.city, item.title)
selectedEvents.push(item)
}
// else{
// console.log('on ne garde pas', differenceDays,item.city, item.title)
// }
})
return selectedEvents
}
export function diffDaysBetweenTwoDates(date1, date2) { export function diffDaysBetweenTwoDates(date1, date2) {
const a = moment(date1); const a = moment(date1);
@ -459,7 +387,6 @@ export function diffDaysBetweenTwoDates(date1,date2) {
} }
export function filterRegionAgendaDuLibreEvents(events_list, filter_critera) { export function filterRegionAgendaDuLibreEvents(events_list, filter_critera) {
let selection = [] let selection = []
events_list.forEach(item => { events_list.forEach(item => {

View File

@ -1,6 +1,6 @@
import fs from "fs"; import fs from "fs";
import path from 'path'; import path from 'path';
import {getRandomElementOfArray, listFilesOfFolder} from './utils.mjs' import {getRandomElementOfArray, listFilesOfFolder} from './libs/utils.mjs'
import Masto from "mastodon"; import Masto from "mastodon";
const __dirname = path.resolve(); const __dirname = path.resolve();

View File

@ -4,7 +4,7 @@ import sendPostMastodon, {
randomIntFromInterval, randomIntFromInterval,
getRandomElementOfArray, getRandomElementOfArray,
findPictureAndSendPost, getRandomLinkGeneral findPictureAndSendPost, getRandomLinkGeneral
} from './utils.mjs' } from './libs/utils.mjs'
const folderBlogPostsPreview = process.cwd() + '/assets/blog_posts_medias/' const folderBlogPostsPreview = process.cwd() + '/assets/blog_posts_medias/'

View File

@ -4,7 +4,7 @@ import {
randomIntFromInterval, randomIntFromInterval,
findPictureAndSendPost, getRandomLinkGeneral, hasCliArgument, slugify findPictureAndSendPost, getRandomLinkGeneral, hasCliArgument, slugify
} from './utils.mjs' } from './libs/utils.mjs'
import Masto from "mastodon"; import Masto from "mastodon";
const __dirname = path.resolve(); const __dirname = path.resolve();

View File

@ -7,7 +7,7 @@ import rp from "request-promise";
import $ from "cheerio"; import $ from "cheerio";
import fs from "fs"; import fs from "fs";
import path from 'path'; import path from 'path';
import sendPostMastodon, {downloadImage, randomIntFromInterval, slugify} from './utils.mjs' import sendPostMastodon, {downloadImage, randomIntFromInterval, slugify} from './libs/utils.mjs'
import {getSourceOfDescriptorPageContent} from "./osm_get_description_picture.mjs"; import {getSourceOfDescriptorPageContent} from "./osm_get_description_picture.mjs";
const __dirname = path.resolve(); const __dirname = path.resolve();

View File

@ -1,13 +1,13 @@
import fs from "fs"; import fs from "fs";
import path from 'path'; import path from 'path';
import {getRandomElementOfArray, listFilesOfFolder} from '../utils.mjs' import {getRandomElementOfArray, listFilesOfFolder} from '../libs/utils.mjs'
import Masto from "mastodon"; import Masto from "mastodon";
const __dirname = path.resolve(); const __dirname = path.resolve();
// choisir un type de publication au hasard dans les dossiers "picture" // choisir un type de publication au hasard dans les dossiers "picture"
import pictureFolderDescriptions from "../describe_picture_folders.mjs" import pictureFolderDescriptions from "../libs/describe_picture_folders.mjs"
const typeOfFolder = getRandomElementOfArray(Object.keys(pictureFolderDescriptions)) const typeOfFolder = getRandomElementOfArray(Object.keys(pictureFolderDescriptions))
console.log(typeOfFolder) console.log(typeOfFolder)

View File

@ -1,6 +1,6 @@
// let firstimgsource = findFirstImageInContent(postObject.post_content) // let firstimgsource = findFirstImageInContent(postObject.post_content)
import {findFirstImageInContent} from "../utils.mjs"; import {findFirstImageInContent} from "../libs/utils.mjs";
let myContentHtml = let myContentHtml =
"Protonmail, la fameuse boite email zéro connaissance de la part de lhébergeur Suisse, développé à source ouverte depuis son origine et par des techos du CERN, permet de supprimer le contenu de pas mal de dossiers, mais pas de la boite de réception principale.\n" + "Protonmail, la fameuse boite email zéro connaissance de la part de lhébergeur Suisse, développé à source ouverte depuis son origine et par des techos du CERN, permet de supprimer le contenu de pas mal de dossiers, mais pas de la boite de réception principale.\n" +

View File

@ -0,0 +1,83 @@
import sendPostMastodon, {
getArticlesFromDaysInRssFeed,
makeTitleContentFromRss,
parseRssFeed,
} from "./libs/utils.mjs";
// const afis_rss_feed_url = 'https://afis.org/rss.xml';
// const days_back_in_rss_feed = 7;
async function postLink() {
// selectionner un compte parmi les gens suivis dans une liste
// export des abonnements utilisateur https://mastodon.cipherbliss.com/settings/exports/follows.csv/home/cipherbliss/Nextcloud/inbox/following_accounts.csv
// <https://mastodon.cipherbliss.com/api/v1/accounts/1/following?max_id=19658>; rel="next", <https://mastodon.cipherbliss.com/api/v1/accounts/1/following?since_id=20294>; rel="prev"
// https://mastodon.cipherbliss.com/api/v1/accounts/relationships?id[]=109484382634211819&id[]=109280269336881811&id[]=109280889310776283&id[]=109248504398682634&id[]=109310268216721885&id[]=109916178850231009&id[]=109541053771316772&id[]=151980&id[]=80685&id[]=106835252230809041&id[]=109167546376395742&id[]=109791782092094822&id[]=109770197347890560&id[]=109768052505508503&id[]=108202891614303950&id[]=109749045115736569&id[]=109739290485102920&id[]=109330282577556324&id[]=109280646010767217&id[]=109722335836985403&id[]=109542013632346871&id[]=109278739925137927&id[]=109659515794606337&id[]=109292570383233152&id[]=109660244192213202&id[]=109303521370666602&id[]=109523867267929049&id[]=109643780720044907&id[]=109386436202793665&id[]=109501996699619458&id[]=109632939061534470&id[]=109621031128728719&id[]=109620790782200477&id[]=139144&id[]=109592762903342668&id[]=109603261782470086&id[]=109591788539379136&id[]=109570049875530270&id[]=109381927764082328&id[]=109554213128019255
// exemple d'account:
/**
{
"id": "109484382634211819",
"username": "GalmeshRosewood",
"acct": "GalmeshRosewood@mastodon.art",
"display_name": "Galmesh Rosewood",
"locked": false,
"bot": false,
"discoverable": true,
"group": false,
"created_at": "2022-12-05T00:00:00.000Z",
"note": "<p>I'm Galmesh Rosewood and I want to be a lewd artist. (It's a Monkey Island ref). Male from France. Some content won't be family-friendly.</p>",
"url": "https://mastodon.art/@GalmeshRosewood",
"avatar": "https://mastodon.cipherbliss.com/system/cache/accounts/avatars/109/484/382/634/211/819/original/4d3c4ded9ae95202.jpg",
"avatar_static": "https://mastodon.cipherbliss.com/system/cache/accounts/avatars/109/484/382/634/211/819/original/4d3c4ded9ae95202.jpg",
"header": "https://mastodon.cipherbliss.com/system/cache/accounts/headers/109/484/382/634/211/819/original/5b1822520e4ae3b3.png",
"header_static": "https://mastodon.cipherbliss.com/system/cache/accounts/headers/109/484/382/634/211/819/original/5b1822520e4ae3b3.png",
"followers_count": 762,
"following_count": 195,
"statuses_count": 327,
"last_status_at": "2023-03-04",
"emojis": [],
"fields": [
{
"name": "Linktree",
"value": "<a href=\"https://linktr.ee/galmeshrosewood\" rel=\"nofollow noopener noreferrer\" target=\"_blank\"><span class=\"invisible\">https://</span><span class=\"\">linktr.ee/galmeshrosewood</span><span class=\"invisible\"></span></a>",
"verified_at": null
}
]
}
*/
// faire un repost de média
// POST /api/v1/statuses/109965725969165853/reblog
// autre action:
// faire une liste de recommandation de 3 gens à follow au hasard pris dans cette liste,
// avec en image attachée un des médias postés par les comptes en question
// console.log("envoi de post des articles afis sciences publiés depuis " + days_back_in_rss_feed + " jours par le compte afis91")
//
// let parsedFeed = await parseRssFeed(afis_rss_feed_url)
// let articles = await getArticlesFromDaysInRssFeed(parsedFeed, days_back_in_rss_feed)
// console.log('articles', articles)
// let contentOfPost = 'Articles des ' + days_back_in_rss_feed + ' derniers jours sur le site web de l\'AFIS: \n'
// + makeTitleContentFromRss(articles)
//
// let configPost = {
// author: 'afis91',
// website: 'afis.org',
// // disable_slugify: true,
// slug: "afis_picture",
// content_type: "text/markdown",
// folder_image: process.cwd() + '/assets/',
// image: "afis_picture_of_the_day.jpg",
// message: `# ${contentOfPost}
//
// #afis #science #pseudoScience #hebdo`,
// }
// sendPostMastodon(configPost)
}
postLink();

View File

@ -1,6 +1,6 @@
import fs from "fs"; import fs from "fs";
import path from 'path'; import path from 'path';
import {getRandomElementOfArray, listFilesOfFolder} from '../utils.mjs' import {getRandomElementOfArray, listFilesOfFolder} from '../libs/utils.mjs'
import Masto from "mastodon"; import Masto from "mastodon";
const __dirname = path.resolve(); const __dirname = path.resolve();

View File

@ -2,7 +2,7 @@ var express = require('express');
var router = express.Router(); var router = express.Router();
var sqlite3 = require('sqlite3'); var sqlite3 = require('sqlite3');
var Masto = require('mastodon'); var Masto = require('mastodon');
// import accounts_to_select from '../src/configs.js' // import accounts_to_select from '../src/configs.mjs'
const accounts_to_select = [ const accounts_to_select = [
{ {