From afef2388dc4cca877483f3facc36c1e302bd9e12 Mon Sep 17 00:00:00 2001 From: Tykayn Date: Fri, 22 Oct 2021 14:17:19 +0200 Subject: [PATCH] fix some stuff --- conversion.js | 458 ++++++++++++++++++------------------ main.js | 22 +- package.json | 2 +- public/stylesheets/main.css | 1 + 4 files changed, 243 insertions(+), 240 deletions(-) diff --git a/conversion.js b/conversion.js index 6cb42e2..c66ee88 100755 --- a/conversion.js +++ b/conversion.js @@ -4,248 +4,256 @@ const axios = require('axios'); class Conversion { - constructor( ) { - this.fetched_times = 0; - this.max_fetchUsers = 5; - this.usersMemo = {}; - } - - hello() { - console.log('hello from conversion'); - } - - likes() { - // read file likes - fs.readFile('source_data/likes.json', - // callback function that is called when reading file is done - function (err, data) { - // parse json - let jsonParsedLikes = JSON.parse(data); - // access elements - const lengthOfLikes = jsonParsedLikes.orderedItems.length; - console.log('likes length', lengthOfLikes); - return jsonParsedLikes; - }); - } - - filterToots(toots, options) { - let filteredToots = toots; - let filter = true; - console.log('before slicing filteredToots.length' , filteredToots.length) - // console.log('filteredToots[0]', filteredToots[0]) - - if(filter){ - - if (options.showMostRecentTootsOnTop) { - filteredToots = filteredToots.reverse(); - } - if (options.filterBiggerTootsBeforeSlicing) { - filteredToots = filteredToots.filter(item => { - return (item['object'].content && item['object'].content.length > options.min_length && item['object'].content.length < options.max_length); - }); - - filteredToots = filteredToots.slice(0, options.max_toots); - } else { - const slice = toots.slice(0, options.max_toots); - filteredToots = slice.filter(item => { - return (item['object'].content && item['object'].content.length > options.min_length && item['object'].content.length < options.max_length); - }); - } - // filteredToots.forEach(toot => { - // toot = this.findMediaUrl(toot); - // toot = this.removeLastChars(toot); - // return toot; - // }); - console.log('after slicing filteredToots.length' , filteredToots.length) - - + constructor() { + this.fetched_times = 0; + this.max_fetchUsers = 5; + this.usersMemo = {}; } - return filteredToots; - } - - // - /** - * - * @param toot - * @returns {*} - */ - findMediaUrl(toot) { - /** - * goal: - * https://mastodon.cipherbliss.com/system/media_attachments/files/000/858/113/original/74b370672892f884.jpg?1566230144 - * - * input data: - * "attributedTo":"https://mastodon.cipherbliss.com/users/tykayn", - * - * "attachment":[{"type":"Document", - * "mediaType":"image/png","url":"media_attachments/files/000/872/910/original/c82b422f302b8ec9.png", - * "name":null, - * "blurhash":"UnSOjgo~ysWAVYWBkWjXu5axVrjckqoze?Rk"}], - * - * we use the attribuedTo property to find instance, and map the url to the instance url, and add this property to the attachment - */ - if (toot['object'].attributedTo) { - - let splitted = toot['object'].attributedTo.split('/'); - let instanceUrl = splitted[2]; - - toot.instanceUrl = 'https://' + instanceUrl; - toot.attachment = toot['object'].attachment.map(att => { - att.href = toot.instanceUrl + '/system/' + att.url; - return att; - }); + hello() { + console.log('hello from conversion'); } - return toot; - } - findInstanceFromAttributedToUrl(attributedTo) { - let splitted = attributedTo.split('/'); - return splitted[2]; - } + likes() { + // read file likes + fs.readFile('source_data/likes.json', + // callback function that is called when reading file is done + function (err, data) { + // parse json + let jsonParsedLikes = JSON.parse(data); + // access elements + const lengthOfLikes = jsonParsedLikes.orderedItems.length; + console.log('likes length', lengthOfLikes); + return jsonParsedLikes; + }); + return {} + } - removeLastChars(toot) { - toot['object'].content = toot['object'].content.trim(); - return toot; - } + filterToots(toots, options) { + let filteredToots = toots; + let filter = true; + console.log('before slicing filteredToots.length', filteredToots.length) + // console.log('filteredToots[0]', filteredToots[0]) - makeStatsForToots(tootArray) { - let stats = { - recievers: {}, - hashtags : {}, - }; - // make statistics on who do we talk to, based on the cc field - tootArray.forEach(elem => { + if (filter) { - // stats on hashtags - if (elem['object'].tag) { - elem['object'].tag.forEach(tag => { - if (tag.type === 'Hashtag') { - if (!stats.hashtags[tag.name]) { - stats.hashtags[tag.name] = { - name : tag.name, - href : tag.href, - counter: 0, - }; + if (options.filterOnlyHavingMedias) { + toots = masto_converter.conversion.filterOnlyTootsWithMedias(toots); + console.log('toots.length only attachements', toots.length); + } + if (options.showMostRecentTootsOnTop) { + filteredToots = filteredToots.reverse(); + } + if (options.filterBiggerTootsBeforeSlicing) { + console.log('min length of content of toots ', options.min_length ); + filteredToots = filteredToots.filter(item => { + return (item['object'].content && item['object'].content.length >= options.min_length && item['object'].content.length <= options.max_length); + }); + + filteredToots = filteredToots.slice(0, options.max_toots); + } else { + const slice = toots.slice(0, options.max_toots); + filteredToots = slice.filter(item => { + return (item['object'].content && item['object'].content.length >= options.min_length && item['object'].content.length <= options.max_length); + }); + } + // filteredToots.forEach(toot => { + // toot = this.findMediaUrl(toot); + // toot = this.removeLastChars(toot); + // return toot; + // }); + console.log('after slicing filteredToots.length', filteredToots.length) + + + } + + return filteredToots; + } + + // + /** + * + * @param toot + * @returns {*} + */ + findMediaUrl(toot) { + /** + * goal: + * https://mastodon.cipherbliss.com/system/media_attachments/files/000/858/113/original/74b370672892f884.jpg?1566230144 + * + * input data: + * "attributedTo":"https://mastodon.cipherbliss.com/users/tykayn", + * + * "attachment":[{"type":"Document", + * "mediaType":"image/png","url":"media_attachments/files/000/872/910/original/c82b422f302b8ec9.png", + * "name":null, + * "blurhash":"UnSOjgo~ysWAVYWBkWjXu5axVrjckqoze?Rk"}], + * + * we use the attribuedTo property to find instance, and map the url to the instance url, and add this property to the attachment + */ + if (toot['object'].attributedTo) { + + let splitted = toot['object'].attributedTo.split('/'); + let instanceUrl = splitted[2]; + + toot.instanceUrl = 'https://' + instanceUrl; + toot.attachment = toot['object'].attachment.map(att => { + att.href = toot.instanceUrl + '/system/' + att.url; + return att; + }); + } + return toot; + } + + findInstanceFromAttributedToUrl(attributedTo) { + let splitted = attributedTo.split('/'); + return splitted[2]; + } + + removeLastChars(toot) { + toot['object'].content = toot['object'].content.trim(); + return toot; + } + + makeStatsForToots(tootArray) { + let stats = { + recievers: {}, + hashtags: {}, + }; + // make statistics on who do we talk to, based on the cc field + tootArray.forEach(elem => { + + // stats on hashtags + if (elem['object'].tag) { + elem['object'].tag.forEach(tag => { + if (tag.type === 'Hashtag') { + if (!stats.hashtags[tag.name]) { + stats.hashtags[tag.name] = { + name: tag.name, + href: tag.href, + counter: 0, + }; + } + stats.hashtags[tag.name].counter++; + } + }); + } + // stats on recievers of toots + if (elem['object'].cc) { + elem['object'].cc.forEach(urlOfUser => { + if (!stats.recievers[urlOfUser]) { + stats.recievers[urlOfUser] = { + user: this.urlToUser(urlOfUser), + name: urlOfUser, + infos: this.fetchUserInfo(urlOfUser), + counter: 0, + counterContentLength: 0, + }; + } + stats.recievers[urlOfUser].counter++; + stats.recievers[urlOfUser].counterContentLength += elem['object'].content.length; + }); } - stats.hashtags[tag.name].counter++; - } }); - } - // stats on recievers of toots - if (elem['object'].cc) { - elem['object'].cc.forEach(urlOfUser => { - if (!stats.recievers[urlOfUser]) { - stats.recievers[urlOfUser] = { - user : this.urlToUser(urlOfUser), - name : urlOfUser, - infos : this.fetchUserInfo(urlOfUser), - counter : 0, - counterContentLength: 0, - }; - } - stats.recievers[urlOfUser].counter++; - stats.recievers[urlOfUser].counterContentLength += elem['object'].content.length; + + stats = { + recievers: this.sortTootsByLength(stats.recievers), + hashtags: this.sortTootsByLength(stats.hashtags), + }; + return stats; + } + + sortTootsByLength(stats) { + const statKeys = Object.keys(stats); + const arrayToSort = []; + statKeys.forEach(elem => { + arrayToSort.push( + stats[elem], + ); }); - } - }); - - stats = { - recievers: this.sortTootsByLength(stats.recievers), - hashtags : this.sortTootsByLength(stats.hashtags), - }; - return stats; - } - - sortTootsByLength(stats) { - const statKeys = Object.keys(stats); - const arrayToSort = []; - statKeys.forEach(elem => { - arrayToSort.push( - stats[elem], - ); - }); - arrayToSort.sort((a, b) => { - return b.counter - a.counter; - }); - return arrayToSort; - } - - urlToUser(url) { - let sliceOfSlashes = url.split('/'); - let userObject = { - url : url, - username: sliceOfSlashes[sliceOfSlashes.length - 1], - }; - return userObject; - } - - filterOnlyTootsWithMedias(tootList) { - console.log('filterOnlyTootsWithMedias'); - return tootList.filter(toot => { - return toot['object'].attachment && toot['object'].attachment.length; - }); - } - - - /** - * find user avatar url from its user homepage url. - * API is: - * https://mastodon.social/.well-known/webfinger?resource=acct:gargron@mastodon.social - * @param attributeToUrl - */ - fetchUserInfo(attributeToUrl = 'https://mastodon.cipherbliss.com/users/tykayn') { - - // do not fetch "followers" and "activitystreams#Public" - if ( - attributeToUrl.indexOf('followers') !== -1 || - attributeToUrl.indexOf('activitystreams#Public') !== -1 - ) { - return {}; + arrayToSort.sort((a, b) => { + return b.counter - a.counter; + }); + return arrayToSort; } - // do not fetch too much - if (this.max_fetchUsers <= this.fetched_times) { - return {}; + + urlToUser(url) { + let sliceOfSlashes = url.split('/'); + let userObject = { + url: url, + username: sliceOfSlashes[sliceOfSlashes.length - 1], + }; + return userObject; } - this.fetched_times++; - let instanceHandle = this.findInstanceFromAttributedToUrl(attributeToUrl); - let splitted = attributeToUrl.split('/'); - let accountHandle = splitted[splitted.length - 1]; - // memo things - let memoEntry = this.usersMemo[accountHandle + '@' + instanceHandle]; - if (memoEntry) { - return memoEntry; + filterOnlyTootsWithMedias(tootList) { + console.log('filterOnlyTootsWithMedias from', tootList.length); + let filtered = tootList.filter(toot => { + return toot['object'].attachment && toot['object'].attachment.length; + }); + console.log('to ', filtered.length); + return filtered } - // webfinger version - // let fetchUri = `https://${instanceHandle}/.well-known/webfinger?resource=acct:${accountHandle}@${instanceHandle}`; - // fetchUri = `https://${instanceHandle}/api/v1/account/${accountHandle}@${instanceHandle}` - // fetchUri = 'https://mastodon.cipherbliss.com/users/tykayn'; - let headers = { - method : 'GET', - url : attributeToUrl, - headers: { - accept: 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"', - }, - }; - const self = this; - const ax = axios.create(headers); - ax.get().then(function (response) { - // handle success - // avatar is response.data.icon.url - // cover is response.data.image.url - // console.log(response.data.icon); - // console.log(response.data.image); - if (!self.usersMemo[accountHandle + '@' + instanceHandle]) { - self.usersMemo[accountHandle + '@' + instanceHandle] = {}; - } - self.usersMemo[accountHandle + '@' + instanceHandle] = response.data; - return response.data; - }); + /** + * find user avatar url from its user homepage url. + * API is: + * https://mastodon.social/.well-known/webfinger?resource=acct:gargron@mastodon.social + * @param attributeToUrl + */ + fetchUserInfo(attributeToUrl = 'https://mastodon.cipherbliss.com/users/tykayn') { - } + // do not fetch "followers" and "activitystreams#Public" + if ( + attributeToUrl.indexOf('followers') !== -1 || + attributeToUrl.indexOf('activitystreams#Public') !== -1 + ) { + return {}; + } + // do not fetch too much + if (this.max_fetchUsers <= this.fetched_times) { + return {}; + } + this.fetched_times++; + + let instanceHandle = this.findInstanceFromAttributedToUrl(attributeToUrl); + let splitted = attributeToUrl.split('/'); + let accountHandle = splitted[splitted.length - 1]; + // memo things + let memoEntry = this.usersMemo[accountHandle + '@' + instanceHandle]; + if (memoEntry) { + return memoEntry; + } + // webfinger version + // let fetchUri = `https://${instanceHandle}/.well-known/webfinger?resource=acct:${accountHandle}@${instanceHandle}`; + // fetchUri = `https://${instanceHandle}/api/v1/account/${accountHandle}@${instanceHandle}` + // fetchUri = 'https://mastodon.cipherbliss.com/users/tykayn'; + let headers = { + method: 'GET', + url: attributeToUrl, + headers: { + accept: 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"', + }, + }; + + const self = this; + const ax = axios.create(headers); + // ax.get().then(function (response) { + // // handle success + // // avatar is response.data.icon.url + // // cover is response.data.image.url + // // console.log(response.data.icon); + // // console.log(response.data.image); + // + // if (!self.usersMemo[accountHandle + '@' + instanceHandle]) { + // self.usersMemo[accountHandle + '@' + instanceHandle] = {}; + // } + // self.usersMemo[accountHandle + '@' + instanceHandle] = response.data; + // return response.data; + // }); + + } } diff --git a/main.js b/main.js index 10b839b..8abb5d1 100755 --- a/main.js +++ b/main.js @@ -13,16 +13,16 @@ var jsonParsedLikes, jsonParsedOutbox; * export configuration. * You can filter the export in the following vars */ -const min_length = 500; // minmum character length of toots to display -const max_length = 100000; // minmum character length of toots to display -const max_toots = 20// maximum length +const min_length = 300; // minmum character length of toots to display +const max_length = 100000; // max character length of toots to display +const max_toots = 200// maximum count of output const filterBiggerTootsBeforeSlicing = false; // filter only long toots -const filterOnlyHavingMedias = true; // filter only toots having medias +const filterOnlyHavingMedias = false; // filter only toots having medias const displayMedias = false; // display medias in toots const showMostRecentTootsOnTop = true; // sorting order // output write const writeStatsJson = false; // write json export file about statistics -const writeHtml = true; // write json export file about statistics +const writeHtml = false; // write json export file about statistics const TemplateVars = { pageTitle : 'Mastodon export converter to HTML', @@ -57,14 +57,10 @@ fs.readFile('source_data/outbox.json', jsonParsedOutbox = JSON.parse(data); toots = jsonParsedOutbox.orderedItems; // access elements - console.log('outbox toots length', toots.length); + console.log('-------- outbox toots length', toots.length); TemplateVars.outboxTotalLength = toots.length; - toots = jsonParsedOutbox.orderedItems; - if (filterOnlyHavingMedias) { - toots = masto_converter.conversion.filterOnlyTootsWithMedias(toots); - console.log('toots.length only attachements', toots.length); - } + toots = masto_converter.conversion.filterToots(toots, TemplateVars); console.log('min_chars', min_length); @@ -81,7 +77,7 @@ fs.readFile('source_data/outbox.json', }); const errfileHandler = (err) => { if (err) - console.log(err); + console.log('ERROR =======================' ,err); else { console.log('File statistics written successfully\n'); } @@ -96,12 +92,10 @@ app.get('/', (req, res) => { if(writeHtml){ fs.writeFile('output/my_toots.html', html, errfileHandler); - } res.render('index.pug', TemplateVars); - masto_converter.conversion.fetchUserInfo(); }); masto_converter.conversion.fetchUserInfo(); diff --git a/package.json b/package.json index fa5630a..d96b552 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "scripts": { - "start": "node main.js" + "start": "nodemon main.js" }, "dependencies": { "axios": "^0.19.2", diff --git a/public/stylesheets/main.css b/public/stylesheets/main.css index b8b7c5d..2593ee0 100644 --- a/public/stylesheets/main.css +++ b/public/stylesheets/main.css @@ -13,6 +13,7 @@ a { padding: 0.5em; border-radius: 0.25em; border-bottom: 1px solid #42495c; + color: white; } .published {