tykayn/mastodon-archive-stats
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

📖 some more doc

Browse Source
This commit is contained in:
Baptiste Lemoine 2020-07-20 12:39:01 +02:00
parent 2d1c839891
commit 3bda33cc18
5 changed files with 209 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
README.md
View File

@ -20,6 +20,12 @@ and you will see the result in html.
## What statistics does this provide
For the moment:
* who you talked to the most, with counters. This uses the object.cc field of a toot.
* link to search on duckduckgo for hashtags and usernames
* what are your most used hashtags
* you can filter:
* only toots with medias
* change the order of filter
* min length of toot content
* what toots are containing at least a certain number of characters. You can use this to retrieve some long posts you made, to post them on a worthy place where you would develop your subject deeper. Somewhere like on your personal website, thats an easy way to find your longests posts. But be careful, it could also show how much you talk with trolls :D
## configuration

85
conversion.js
View File

@ -3,8 +3,9 @@ const fs = require('fs');
class Conversion {
hello() {
console.log('hello from conversion')
console.log('hello from conversion');
}
likes() {
// read file likes
fs.readFile('source_data/likes.json',
@ -33,12 +34,60 @@ class Conversion {
return item['object'].content && item['object'].content.length > options.min_length;
});
}
return minchartoots
minchartoots.forEach(toot => {
toot = this.findMediaUrl(toot);
toot = this.removeLastChars(toot);
return toot;
});
return minchartoots;
}
//
/**
*
* @param toot
* @returns {*}
*/
findMediaUrl(toot) {
/**
* goal:
* https://mastodon.cipherbliss.com/system/media_attachments/files/000/858/113/original/74b370672892f884.jpg?1566230144
*
* input data:
* "attributedTo":"https://mastodon.cipherbliss.com/users/tykayn",
*
* "attachment":[{"type":"Document",
* "mediaType":"image/png","url":"media_attachments/files/000/872/910/original/c82b422f302b8ec9.png",
* "name":null,
* "blurhash":"UnSOjgo~ysWAVYWBkWjXu5axVrjckqoze?Rk"}],
*
* we use the attribuedTo property to find instance, and map the url to the instance url, and add this property to the attachment
*/
if (toot['object'].attributedTo) {
let splitted = toot['object'].attributedTo.split('/');
let instanceUrl = splitted[2];
toot.instanceUrl = 'https://' + instanceUrl;
toot.attachment = toot['object'].attachment.map(att => {
att.href = toot.instanceUrl + '/system/' + att.url;
return att;
});
}
return toot;
}
removeLastChars(toot) {
toot['object'].content = toot['object'].content.trim();
return toot;
}
makeStatsForToots(tootArray) {
let stats = { recievers: {},
hashtags : {}};
let stats = {
recievers: {},
hashtags : {},
};
// make statistics on who do we talk to, based on the cc field
tootArray.forEach(elem => {
@ -49,18 +98,20 @@ class Conversion {
if (!stats.hashtags[tag.name]) {
stats.hashtags[tag.name] = {
name : tag.name,
href : tag.href,
counter: 0,
}
};
}
stats.hashtags[tag.name].counter++;
}
})
});
}
// stats on recievers of toots
if (elem['object'].cc) {
elem['object'].cc.forEach(copyFolk => {
if (!stats.recievers[copyFolk]) {
stats.recievers[copyFolk] = {
user : this.urlToUser(copyFolk),
name : copyFolk,
counter : 0,
counterContentLength: 0,
@ -72,7 +123,7 @@ class Conversion {
}
});
console.log('stats.hashtags', stats.hashtags);
console.log('stats.hashtags', stats.hashtags[0]);
stats = {
recievers: this.sortTootsByLength(stats.recievers),
hashtags : this.sortTootsByLength(stats.hashtags),
@ -89,9 +140,27 @@ class Conversion {
);
});
arrayToSort.sort((a, b) => {
return b.counter - a.counter
return b.counter - a.counter;
});
return arrayToSort;
}
urlToUser(url) {
let sliceOfSlashes = url.split('/');
let userObject = {
url : url,
username: sliceOfSlashes[sliceOfSlashes.length - 1],
};
return userObject;
}
filterOnlyTootsWithMedias(tootList) {
console.log('filterOnlyTootsWithMedias')
return tootList.filter(toot => {
return toot['object'].attachment && toot['object'].attachment.length
});
}
}
exports.conversion = new Conversion();

40
main.js
View File

@ -10,9 +10,11 @@ var fs = require('fs');
var listenPort = 8088;
var jsonParsedLikes, jsonParsedOutbox;
// const min_length = 1050; // filter only long toots
const min_length = 300; // filter only long toots
const max_toots = 500; // filter only long toots
const min_length = 1; // filter only long toots
const max_toots = 10; // filter only long toots
const filterBiggerTottsBeforeSlicing = false; // filter only long toots
const filterOnlyHavingMedias = true; // filter only toots having medias
const writeStatsJson = false; // filter only toots having medias
const TemplateVars = {
pageTitle : 'Mastodon export converter to HTML',
likes : jsonParsedLikes,
@ -21,47 +23,55 @@ const TemplateVars = {
outbox_all : jsonParsedOutbox,
min_length,
max_toots,
filterOnlyHavingMedias,
filterBiggerTottsBeforeSlicing,
writeStatsJson,
};
const masto_converter = require('./conversion');
console.log('masto_converter', masto_converter);
masto_converter.conversion.hello();
jsonParsedLikes = masto_converter.conversion.likes();
fs.readFile('source_data/outbox.json',
// callback function that is called when reading file is done
function (err, data) {
let minchartoots ;
let toots;
// parse json
jsonParsedOutbox = JSON.parse(data);
toots = jsonParsedOutbox.orderedItems;
// access elements
console.log('outbox toots length', jsonParsedOutbox.orderedItems.length);
console.log('outbox toots length', toots.length);
TemplateVars.outboxTotalLength = jsonParsedOutbox.orderedItems.length;
minchartoots = masto_converter.conversion.filterToots(jsonParsedOutbox.orderedItems, TemplateVars)
TemplateVars.outboxTotalLength = toots.length;
toots = jsonParsedOutbox.orderedItems;
if (filterOnlyHavingMedias) {
toots = masto_converter.conversion.filterOnlyTootsWithMedias(toots);
console.log('toots.length only attachements', toots.length );
}
toots = masto_converter.conversion.filterToots(toots, TemplateVars);
console.log('min_chars', min_length);
console.log('toots min char corresponding', minchartoots.length);
TemplateVars.outbox = minchartoots;
TemplateVars.outboxStatistics = masto_converter.conversion.makeStatsForToots(minchartoots);
console.log('toots min char corresponding', toots.length);
TemplateVars.outbox = toots;
TemplateVars.outboxStatistics = masto_converter.conversion.makeStatsForToots(toots);
const example = TemplateVars.outbox[1]['object'];
TemplateVars.example = example;
console.log('example', example)
if (writeStatsJson) {
fs.writeFile('output/statistics.json', JSON.stringify(TemplateVars.outboxStatistics), errfileHandler);
}
});
const errfileHandler = (err) => {
if (err)
console.log(err);
else {
console.log("File statistics written successfully\n");
console.log('File statistics written successfully\n');
}
}
};
app.use(express.static('public'));
app.set('view engine', 'pug');

30
public/stylesheets/main.css
View File

@ -1,24 +1,42 @@
body {
background: #222;
background: #292a2a;
color: white;
padding: 1em 2em;
}
a {
color: #5561ff;
color: #00a7d1;
}
.status {
background: #111;
background: #606984;
padding: 0.5em;
border-radius: 0.25em;
border-bottom: 1px solid #42495c;
}
.published {
width: 70ch;
}
.date-published {
font-size: 0.5em;
}
.status .published{
padding-left: 2em;
}
.stats {
max-height: 15em;
overflow: auto;
}
.media-gallery {
}
.media-media-displayed {
min-height: 500px;
margin-right: 1em;
}
.hidden {
display: none;
}

31
views/index.pug
View File

@ -13,13 +13,17 @@ html(lang="en")
table
thead
tr
th= "search"
th= "hashtag"
th= "occurences"
tbody
each hashtag in outboxStatistics.hashtags
tr
td.search
a(href= 'https://duckduckgo.com/?q='+hashtag.name)
='search'
td.name
a(href=hashtag.name)=hashtag.name
a(href=hashtag.href)=hashtag.name
td.counter=hashtag.counter
td.counter=hashtag.counterContentLength
@ -28,14 +32,18 @@ html(lang="en")
table
thead
tr
th= "search"
th= "name"
th= "times"
th= "toots lengh sum"
tbody
each someone in outboxStatistics.recievers
tr
td.search
a(href= 'https://duckduckgo.com/?q='+someone.user.username)
='search'
td.name
a(href=someone.name)=someone.name
a(href=someone.name)=someone.user.username
td.counter=someone.counter
td.counter=someone.counterContentLength
h2 #{outbox.length} of Messages #{outboxTotalLength} in your outbox.First #{max_toots} toots, filtered by a minimal length of #{min_length} characters of content.
@ -47,4 +55,21 @@ html(lang="en")
div.status.status-public
a(href=oredredItem['object'].url)="see"
div.date-published=oredredItem['object'].published
blockquote.published(unescaped!=oredredItem['object'].content)
blockquote.published(escaped!=oredredItem['object'].content)
if oredredItem['object'].attachment && oredredItem['object'].attachment.length
each media in oredredItem['object'].attachment
div.media-gallery
div.counter.hidden #{oredredItem['object'].attachment.length}
if(media.mediaType.search('video'))
figure.media-media-displayed.media-image
a(href=media.href)
video(src=media.href)
span Media vidéo
elseif(media.mediaType.search('audio'))
span Media audio
elseif(media.mediaType.search('image'))
figure.media-media-displayed.media-image
a(href=media.href)
img(src=media.href,alt=media.href)