tykayn
/
mastodon-archive-stats
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

📖 some more doc

This commit is contained in:
Baptiste Lemoine 2020-07-20 12:39:01 +02:00
parent 2d1c839891
commit 3bda33cc18
5 changed files with 209 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
README.md
View File

@ -20,6 +20,12 @@ and you will see the result in html.
## What statistics does this provide
For the moment:
* who you talked to the most, with counters. This uses the object.cc field of a toot.
* link to search on duckduckgo for hashtags and usernames
* what are your most used hashtags
* you can filter:
* only toots with medias
* change the order of filter
* min length of toot content
* what toots are containing at least a certain number of characters. You can use this to retrieve some long posts you made, to post them on a worthy place where you would develop your subject deeper. Somewhere like on your personal website, thats an easy way to find your longests posts. But be careful, it could also show how much you talk with trolls :D
## configuration

165
conversion.js
View File

@ -2,10 +2,11 @@ exports.name = 'masto_conversion';
const fs = require('fs');
class Conversion {
hello(){
console.log('hello from conversion')
hello() {
console.log('hello from conversion');
}
likes(){
likes() {
// read file likes
fs.readFile('source_data/likes.json',
// callback function that is called when reading file is done
@ -19,68 +20,118 @@ class Conversion {
});
}
filterToots(toots,options){
let minchartoots ;
filterToots(toots, options) {
let minchartoots;
if(options.filterBiggerTottsBeforeSlicing){
if (options.filterBiggerTottsBeforeSlicing) {
minchartoots = toots.filter(item => {
return item['object'].content && item['object'].content.length > options.min_length;
});
minchartoots = minchartoots.slice(0, options.max_toots);
}else{
} else {
const slice = toots.slice(0, options.max_toots);
minchartoots = slice.filter(item => {
return item['object'].content && item['object'].content.length > options.min_length;
});
}
return minchartoots
minchartoots.forEach(toot => {
toot = this.findMediaUrl(toot);
toot = this.removeLastChars(toot);
return toot;
});
return minchartoots;
}
makeStatsForToots(tootArray){
let stats = { recievers: {},
hashtags : {}};
// make statistics on who do we talk to, based on the cc field
tootArray.forEach(elem => {
//
/**
*
* @param toot
* @returns {*}
*/
findMediaUrl(toot) {
/**
* goal:
* https://mastodon.cipherbliss.com/system/media_attachments/files/000/858/113/original/74b370672892f884.jpg?1566230144
*
* input data:
* "attributedTo":"https://mastodon.cipherbliss.com/users/tykayn",
*
* "attachment":[{"type":"Document",
* "mediaType":"image/png","url":"media_attachments/files/000/872/910/original/c82b422f302b8ec9.png",
* "name":null,
* "blurhash":"UnSOjgo~ysWAVYWBkWjXu5axVrjckqoze?Rk"}],
*
* we use the attribuedTo property to find instance, and map the url to the instance url, and add this property to the attachment
*/
if (toot['object'].attributedTo) {
// stats on hashtags
if (elem['object'].tag) {
elem['object'].tag.forEach(tag => {
if(tag.type === 'Hashtag'){
if(!stats.hashtags[tag.name]){
stats.hashtags[tag.name] = {
name : tag.name,
counter : 0,
}
}
stats.hashtags[tag.name].counter++;
}
})
}
// stats on recievers of toots
if (elem['object'].cc) {
elem['object'].cc.forEach(copyFolk => {
if (!stats.recievers[copyFolk]) {
stats.recievers[copyFolk] = {
name : copyFolk,
let splitted = toot['object'].attributedTo.split('/');
let instanceUrl = splitted[2];
toot.instanceUrl = 'https://' + instanceUrl;
toot.attachment = toot['object'].attachment.map(att => {
att.href = toot.instanceUrl + '/system/' + att.url;
return att;
});
}
return toot;
}
removeLastChars(toot) {
toot['object'].content = toot['object'].content.trim();
return toot;
}
makeStatsForToots(tootArray) {
let stats = {
recievers: {},
hashtags : {},
};
// make statistics on who do we talk to, based on the cc field
tootArray.forEach(elem => {
// stats on hashtags
if (elem['object'].tag) {
elem['object'].tag.forEach(tag => {
if (tag.type === 'Hashtag') {
if (!stats.hashtags[tag.name]) {
stats.hashtags[tag.name] = {
name : tag.name,
href : tag.href,
counter: 0,
counterContentLength: 0,
};
}
stats.recievers[copyFolk].counter++;
stats.recievers[copyFolk].counterContentLength += elem['object'].content.length;
});
}
});
stats.hashtags[tag.name].counter++;
}
});
}
// stats on recievers of toots
if (elem['object'].cc) {
elem['object'].cc.forEach(copyFolk => {
if (!stats.recievers[copyFolk]) {
stats.recievers[copyFolk] = {
user : this.urlToUser(copyFolk),
name : copyFolk,
counter : 0,
counterContentLength: 0,
};
}
stats.recievers[copyFolk].counter++;
stats.recievers[copyFolk].counterContentLength += elem['object'].content.length;
});
}
});
console.log('stats.hashtags', stats.hashtags);
stats = {
recievers : this.sortTootsByLength(stats.recievers),
hashtags : this.sortTootsByLength(stats.hashtags),
};
return stats ;
console.log('stats.hashtags', stats.hashtags[0]);
stats = {
recievers: this.sortTootsByLength(stats.recievers),
hashtags : this.sortTootsByLength(stats.hashtags),
};
return stats;
}
sortTootsByLength(stats){
sortTootsByLength(stats) {
const statKeys = Object.keys(stats);
const arrayToSort = [];
statKeys.forEach(elem => {
@ -88,10 +139,28 @@ class Conversion {
stats[elem],
);
});
arrayToSort.sort( (a,b)=>{
return b.counter - a.counter
arrayToSort.sort((a, b) => {
return b.counter - a.counter;
});
return arrayToSort;
}
urlToUser(url) {
let sliceOfSlashes = url.split('/');
let userObject = {
url : url,
username: sliceOfSlashes[sliceOfSlashes.length - 1],
};
return userObject;
}
filterOnlyTootsWithMedias(tootList) {
console.log('filterOnlyTootsWithMedias')
return tootList.filter(toot => {
return toot['object'].attachment && toot['object'].attachment.length
});
}
}
exports.conversion = new Conversion();

48
main.js
View File

@ -10,58 +10,68 @@ var fs = require('fs');
var listenPort = 8088;
var jsonParsedLikes, jsonParsedOutbox;
// const min_length = 1050; // filter only long toots
const min_length = 300; // filter only long toots
const max_toots = 500; // filter only long toots
const min_length = 1; // filter only long toots
const max_toots = 10; // filter only long toots
const filterBiggerTottsBeforeSlicing = false; // filter only long toots
const filterOnlyHavingMedias = true; // filter only toots having medias
const writeStatsJson = false; // filter only toots having medias
const TemplateVars = {
pageTitle : 'Mastodon export converter to HTML',
likes : jsonParsedLikes,
outbox : jsonParsedOutbox,
outboxStatistics: {},
outbox_all : jsonParsedOutbox,
min_length ,
max_toots ,
min_length,
max_toots,
filterOnlyHavingMedias,
filterBiggerTottsBeforeSlicing,
writeStatsJson,
};
const masto_converter = require('./conversion');
console.log('masto_converter', masto_converter);
masto_converter.conversion.hello();
jsonParsedLikes = masto_converter.conversion.likes();
fs.readFile('source_data/outbox.json',
// callback function that is called when reading file is done
function (err, data) {
let minchartoots ;
let toots;
// parse json
jsonParsedOutbox = JSON.parse(data);
toots = jsonParsedOutbox.orderedItems;
// access elements
console.log('outbox toots length', jsonParsedOutbox.orderedItems.length);
console.log('outbox toots length', toots.length);
TemplateVars.outboxTotalLength = jsonParsedOutbox.orderedItems.length;
minchartoots = masto_converter.conversion.filterToots(jsonParsedOutbox.orderedItems, TemplateVars)
TemplateVars.outboxTotalLength = toots.length;
toots = jsonParsedOutbox.orderedItems;
if (filterOnlyHavingMedias) {
toots = masto_converter.conversion.filterOnlyTootsWithMedias(toots);
console.log('toots.length only attachements', toots.length );
}
toots = masto_converter.conversion.filterToots(toots, TemplateVars);
console.log('min_chars', min_length);
console.log('toots min char corresponding', minchartoots.length);
TemplateVars.outbox = minchartoots;
TemplateVars.outboxStatistics = masto_converter.conversion.makeStatsForToots(minchartoots);
console.log('toots min char corresponding', toots.length);
TemplateVars.outbox = toots;
TemplateVars.outboxStatistics = masto_converter.conversion.makeStatsForToots(toots);
const example = TemplateVars.outbox[1]['object'];
TemplateVars.example = example;
console.log('example', example)
fs.writeFile('output/statistics.json', JSON.stringify(TemplateVars.outboxStatistics), errfileHandler );
if (writeStatsJson) {
fs.writeFile('output/statistics.json', JSON.stringify(TemplateVars.outboxStatistics), errfileHandler);
}
});
const errfileHandler = (err) => {
if (err)
console.log(err);
else {
console.log("File statistics written successfully\n");
console.log('File statistics written successfully\n');
}
}
};
app.use(express.static('public'));
app.set('view engine', 'pug');
@ -70,7 +80,7 @@ app.get('/', (req, res) => {
const html = pug.render('index.pug', TemplateVars);
fs.writeFile('output/my_toots.html', html, errfileHandler );
fs.writeFile('output/my_toots.html', html, errfileHandler);
res.render('index.pug', TemplateVars);

40
public/stylesheets/main.css
View File

@ -1,24 +1,42 @@
body{
background: #222;
body {
background: #292a2a;
color: white;
padding: 1em 2em;
}
a{
color: #5561ff;
a {
color: #00a7d1;
}
.status{
background: #111;
.status {
background: #606984;
padding: 0.5em;
border-radius: 0.25em;
border-bottom: 1px solid #42495c;
}
.date-published{
.published {
width: 70ch;
}
.date-published {
font-size: 0.5em;
}
.status .published{
padding-left: 2em;
}
.stats{
.stats {
max-height: 15em;
overflow: auto;
}
.media-gallery {
}
.media-media-displayed {
min-height: 500px;
margin-right: 1em;
}
.hidden {
display: none;
}

31
views/index.pug
View File

@ -13,13 +13,17 @@ html(lang="en")
table
thead
tr
th= "search"
th= "hashtag"
th= "occurences"
tbody
each hashtag in outboxStatistics.hashtags
tr
td.search
a(href= 'https://duckduckgo.com/?q='+hashtag.name)
='search'
td.name
a(href=hashtag.name)=hashtag.name
a(href=hashtag.href)=hashtag.name
td.counter=hashtag.counter
td.counter=hashtag.counterContentLength
@ -28,14 +32,18 @@ html(lang="en")
table
thead
tr
th= "search"
th= "name"
th= "times"
th= "toots lengh sum"
tbody
each someone in outboxStatistics.recievers
tr
td.search
a(href= 'https://duckduckgo.com/?q='+someone.user.username)
='search'
td.name
a(href=someone.name)=someone.name
a(href=someone.name)=someone.user.username
td.counter=someone.counter
td.counter=someone.counterContentLength
h2 #{outbox.length} of Messages #{outboxTotalLength} in your outbox.First #{max_toots} toots, filtered by a minimal length of #{min_length} characters of content.
@ -47,4 +55,21 @@ html(lang="en")
div.status.status-public
a(href=oredredItem['object'].url)="see"
div.date-published=oredredItem['object'].published
blockquote.published(unescaped!=oredredItem['object'].content)
blockquote.published(escaped!=oredredItem['object'].content)
if oredredItem['object'].attachment && oredredItem['object'].attachment.length
each media in oredredItem['object'].attachment
div.media-gallery
div.counter.hidden #{oredredItem['object'].attachment.length}
if(media.mediaType.search('video'))
figure.media-media-displayed.media-image
a(href=media.href)
video(src=media.href)
span Media vidéo
elseif(media.mediaType.search('audio'))
span Media audio
elseif(media.mediaType.search('image'))
figure.media-media-displayed.media-image
a(href=media.href)
img(src=media.href,alt=media.href)