tykayn
/
mastodon-archive-stats
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

fix some stuff

This commit is contained in:
Tykayn 2021-10-22 14:17:19 +02:00 committed by tykayn
parent debae70091
commit afef2388dc
4 changed files with 243 additions and 240 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
conversion.js
View File

@ -4,7 +4,7 @@ const axios = require('axios');
class Conversion {
constructor( ) {
constructor() {
this.fetched_times = 0;
this.max_fetchUsers = 5;
this.usersMemo = {};
@ -26,29 +26,35 @@ class Conversion {
console.log('likes length', lengthOfLikes);
return jsonParsedLikes;
});
return {}
}
filterToots(toots, options) {
let filteredToots = toots;
let filter = true;
console.log('before slicing filteredToots.length' , filteredToots.length)
console.log('before slicing filteredToots.length', filteredToots.length)
// console.log('filteredToots[0]', filteredToots[0])
if(filter){
if (filter) {
if (options.filterOnlyHavingMedias) {
toots = masto_converter.conversion.filterOnlyTootsWithMedias(toots);
console.log('toots.length only attachements', toots.length);
}
if (options.showMostRecentTootsOnTop) {
filteredToots = filteredToots.reverse();
}
if (options.filterBiggerTootsBeforeSlicing) {
console.log('min length of content of toots ', options.min_length );
filteredToots = filteredToots.filter(item => {
return (item['object'].content && item['object'].content.length > options.min_length && item['object'].content.length < options.max_length);
return (item['object'].content && item['object'].content.length >= options.min_length && item['object'].content.length <= options.max_length);
});
filteredToots = filteredToots.slice(0, options.max_toots);
} else {
const slice = toots.slice(0, options.max_toots);
filteredToots = slice.filter(item => {
return (item['object'].content && item['object'].content.length > options.min_length && item['object'].content.length < options.max_length);
return (item['object'].content && item['object'].content.length >= options.min_length && item['object'].content.length <= options.max_length);
});
}
// filteredToots.forEach(toot => {
@ -56,7 +62,7 @@ class Conversion {
// toot = this.removeLastChars(toot);
// return toot;
// });
console.log('after slicing filteredToots.length' , filteredToots.length)
console.log('after slicing filteredToots.length', filteredToots.length)
}
@ -112,7 +118,7 @@ class Conversion {
makeStatsForToots(tootArray) {
let stats = {
recievers: {},
hashtags : {},
hashtags: {},
};
// make statistics on who do we talk to, based on the cc field
tootArray.forEach(elem => {
@ -123,8 +129,8 @@ class Conversion {
if (tag.type === 'Hashtag') {
if (!stats.hashtags[tag.name]) {
stats.hashtags[tag.name] = {
name : tag.name,
href : tag.href,
name: tag.name,
href: tag.href,
counter: 0,
};
}
@ -137,10 +143,10 @@ class Conversion {
elem['object'].cc.forEach(urlOfUser => {
if (!stats.recievers[urlOfUser]) {
stats.recievers[urlOfUser] = {
user : this.urlToUser(urlOfUser),
name : urlOfUser,
infos : this.fetchUserInfo(urlOfUser),
counter : 0,
user: this.urlToUser(urlOfUser),
name: urlOfUser,
infos: this.fetchUserInfo(urlOfUser),
counter: 0,
counterContentLength: 0,
};
}
@ -152,7 +158,7 @@ class Conversion {
stats = {
recievers: this.sortTootsByLength(stats.recievers),
hashtags : this.sortTootsByLength(stats.hashtags),
hashtags: this.sortTootsByLength(stats.hashtags),
};
return stats;
}
@ -174,17 +180,19 @@ class Conversion {
urlToUser(url) {
let sliceOfSlashes = url.split('/');
let userObject = {
url : url,
url: url,
username: sliceOfSlashes[sliceOfSlashes.length - 1],
};
return userObject;
}
filterOnlyTootsWithMedias(tootList) {
console.log('filterOnlyTootsWithMedias');
return tootList.filter(toot => {
console.log('filterOnlyTootsWithMedias from', tootList.length);
let filtered = tootList.filter(toot => {
return toot['object'].attachment && toot['object'].attachment.length;
});
console.log('to ', filtered.length);
return filtered
}
@ -222,8 +230,8 @@ class Conversion {
// fetchUri = `https://${instanceHandle}/api/v1/account/${accountHandle}@${instanceHandle}`
// fetchUri = 'https://mastodon.cipherbliss.com/users/tykayn';
let headers = {
method : 'GET',
url : attributeToUrl,
method: 'GET',
url: attributeToUrl,
headers: {
accept: 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
},
@ -231,19 +239,19 @@ class Conversion {
const self = this;
const ax = axios.create(headers);
ax.get().then(function (response) {
// handle success
// avatar is response.data.icon.url
// cover is response.data.image.url
// console.log(response.data.icon);
// console.log(response.data.image);
if (!self.usersMemo[accountHandle + '@' + instanceHandle]) {
self.usersMemo[accountHandle + '@' + instanceHandle] = {};
}
self.usersMemo[accountHandle + '@' + instanceHandle] = response.data;
return response.data;
});
// ax.get().then(function (response) {
// // handle success
// // avatar is response.data.icon.url
// // cover is response.data.image.url
// // console.log(response.data.icon);
// // console.log(response.data.image);
//
// if (!self.usersMemo[accountHandle + '@' + instanceHandle]) {
// self.usersMemo[accountHandle + '@' + instanceHandle] = {};
// }
// self.usersMemo[accountHandle + '@' + instanceHandle] = response.data;
// return response.data;
// });
}

22
main.js
View File

@ -13,16 +13,16 @@ var jsonParsedLikes, jsonParsedOutbox;
* export configuration.
* You can filter the export in the following vars
*/
const min_length = 500; // minmum character length of toots to display
const max_length = 100000; // minmum character length of toots to display
const max_toots = 20// maximum length
const min_length = 300; // minmum character length of toots to display
const max_length = 100000; // max character length of toots to display
const max_toots = 200// maximum count of output
const filterBiggerTootsBeforeSlicing = false; // filter only long toots
const filterOnlyHavingMedias = true; // filter only toots having medias
const filterOnlyHavingMedias = false; // filter only toots having medias
const displayMedias = false; // display medias in toots
const showMostRecentTootsOnTop = true; // sorting order
// output write
const writeStatsJson = false; // write json export file about statistics
const writeHtml = true; // write json export file about statistics
const writeHtml = false; // write json export file about statistics
const TemplateVars = {
pageTitle : 'Mastodon export converter to HTML',
@ -57,14 +57,10 @@ fs.readFile('source_data/outbox.json',
jsonParsedOutbox = JSON.parse(data);
toots = jsonParsedOutbox.orderedItems;
// access elements
console.log('outbox toots length', toots.length);
console.log('-------- outbox toots length', toots.length);
TemplateVars.outboxTotalLength = toots.length;
toots = jsonParsedOutbox.orderedItems;
if (filterOnlyHavingMedias) {
toots = masto_converter.conversion.filterOnlyTootsWithMedias(toots);
console.log('toots.length only attachements', toots.length);
}
toots = masto_converter.conversion.filterToots(toots, TemplateVars);
console.log('min_chars', min_length);
@ -81,7 +77,7 @@ fs.readFile('source_data/outbox.json',
});
const errfileHandler = (err) => {
if (err)
console.log(err);
console.log('ERROR =======================' ,err);
else {
console.log('File statistics written successfully\n');
}
@ -96,12 +92,10 @@ app.get('/', (req, res) => {
if(writeHtml){
fs.writeFile('output/my_toots.html', html, errfileHandler);
}
res.render('index.pug', TemplateVars);
masto_converter.conversion.fetchUserInfo();
});
masto_converter.conversion.fetchUserInfo();

2
package.json
View File

@ -1,6 +1,6 @@
{
"scripts": {
"start": "node main.js"
"start": "nodemon main.js"
},
"dependencies": {
"axios": "^0.19.2",

1
public/stylesheets/main.css
View File

@ -13,6 +13,7 @@ a {
padding: 0.5em;
border-radius: 0.25em;
border-bottom: 1px solid #42495c;
color: white;
}
.published {