tykayn
/
mastodon-archive-stats
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

📖 some more doc

This commit is contained in:
Baptiste Lemoine 2020-07-20 12:39:01 +02:00
parent 2d1c839891
commit 3bda33cc18
5 changed files with 209 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
README.md
View File

@ -20,6 +20,12 @@ and you will see the result in html.
## What statistics does this provide ## What statistics does this provide
For the moment: For the moment:
* who you talked to the most, with counters. This uses the object.cc field of a toot. * who you talked to the most, with counters. This uses the object.cc field of a toot.
* link to search on duckduckgo for hashtags and usernames
* what are your most used hashtags
* you can filter:
* only toots with medias
* change the order of filter
* min length of toot content
* what toots are containing at least a certain number of characters. You can use this to retrieve some long posts you made, to post them on a worthy place where you would develop your subject deeper. Somewhere like on your personal website, thats an easy way to find your longests posts. But be careful, it could also show how much you talk with trolls :D * what toots are containing at least a certain number of characters. You can use this to retrieve some long posts you made, to post them on a worthy place where you would develop your subject deeper. Somewhere like on your personal website, thats an easy way to find your longests posts. But be careful, it could also show how much you talk with trolls :D
## configuration ## configuration

165
conversion.js
View File

@ -2,10 +2,11 @@ exports.name = 'masto_conversion';
const fs = require('fs'); const fs = require('fs');
class Conversion { class Conversion {
hello(){ hello() {
console.log('hello from conversion') console.log('hello from conversion');
} }
likes(){
likes() {
// read file likes // read file likes
fs.readFile('source_data/likes.json', fs.readFile('source_data/likes.json',
// callback function that is called when reading file is done // callback function that is called when reading file is done
@ -19,68 +20,118 @@ class Conversion {
}); });
} }
filterToots(toots,options){ filterToots(toots, options) {
let minchartoots ; let minchartoots;
if(options.filterBiggerTottsBeforeSlicing){ if (options.filterBiggerTottsBeforeSlicing) {
minchartoots = toots.filter(item => { minchartoots = toots.filter(item => {
return item['object'].content && item['object'].content.length > options.min_length; return item['object'].content && item['object'].content.length > options.min_length;
}); });
minchartoots = minchartoots.slice(0, options.max_toots); minchartoots = minchartoots.slice(0, options.max_toots);
}else{ } else {
const slice = toots.slice(0, options.max_toots); const slice = toots.slice(0, options.max_toots);
minchartoots = slice.filter(item => { minchartoots = slice.filter(item => {
return item['object'].content && item['object'].content.length > options.min_length; return item['object'].content && item['object'].content.length > options.min_length;
}); });
} }
return minchartoots minchartoots.forEach(toot => {
toot = this.findMediaUrl(toot);
toot = this.removeLastChars(toot);
return toot;
});
return minchartoots;
} }
makeStatsForToots(tootArray){ //
let stats = { recievers: {}, /**
hashtags : {}}; *
// make statistics on who do we talk to, based on the cc field * @param toot
tootArray.forEach(elem => { * @returns {*}
*/
findMediaUrl(toot) {
/**
* goal:
* https://mastodon.cipherbliss.com/system/media_attachments/files/000/858/113/original/74b370672892f884.jpg?1566230144
*
* input data:
* "attributedTo":"https://mastodon.cipherbliss.com/users/tykayn",
*
* "attachment":[{"type":"Document",
* "mediaType":"image/png","url":"media_attachments/files/000/872/910/original/c82b422f302b8ec9.png",
* "name":null,
* "blurhash":"UnSOjgo~ysWAVYWBkWjXu5axVrjckqoze?Rk"}],
*
* we use the attribuedTo property to find instance, and map the url to the instance url, and add this property to the attachment
*/
if (toot['object'].attributedTo) {
// stats on hashtags let splitted = toot['object'].attributedTo.split('/');
if (elem['object'].tag) { let instanceUrl = splitted[2];
elem['object'].tag.forEach(tag => {
if(tag.type === 'Hashtag'){ toot.instanceUrl = 'https://' + instanceUrl;
if(!stats.hashtags[tag.name]){ toot.attachment = toot['object'].attachment.map(att => {
stats.hashtags[tag.name] = { att.href = toot.instanceUrl + '/system/' + att.url;
name : tag.name, return att;
counter : 0, });
} }
} return toot;
stats.hashtags[tag.name].counter++; }
}
}) removeLastChars(toot) {
} toot['object'].content = toot['object'].content.trim();
// stats on recievers of toots return toot;
if (elem['object'].cc) { }
elem['object'].cc.forEach(copyFolk => {
if (!stats.recievers[copyFolk]) { makeStatsForToots(tootArray) {
stats.recievers[copyFolk] = { let stats = {
name : copyFolk, recievers: {},
hashtags : {},
};
// make statistics on who do we talk to, based on the cc field
tootArray.forEach(elem => {
// stats on hashtags
if (elem['object'].tag) {
elem['object'].tag.forEach(tag => {
if (tag.type === 'Hashtag') {
if (!stats.hashtags[tag.name]) {
stats.hashtags[tag.name] = {
name : tag.name,
href : tag.href,
counter: 0, counter: 0,
counterContentLength: 0,
}; };
} }
stats.recievers[copyFolk].counter++; stats.hashtags[tag.name].counter++;
stats.recievers[copyFolk].counterContentLength += elem['object'].content.length; }
}); });
} }
}); // stats on recievers of toots
if (elem['object'].cc) {
elem['object'].cc.forEach(copyFolk => {
if (!stats.recievers[copyFolk]) {
stats.recievers[copyFolk] = {
user : this.urlToUser(copyFolk),
name : copyFolk,
counter : 0,
counterContentLength: 0,
};
}
stats.recievers[copyFolk].counter++;
stats.recievers[copyFolk].counterContentLength += elem['object'].content.length;
});
}
});
console.log('stats.hashtags', stats.hashtags); console.log('stats.hashtags', stats.hashtags[0]);
stats = { stats = {
recievers : this.sortTootsByLength(stats.recievers), recievers: this.sortTootsByLength(stats.recievers),
hashtags : this.sortTootsByLength(stats.hashtags), hashtags : this.sortTootsByLength(stats.hashtags),
}; };
return stats ; return stats;
} }
sortTootsByLength(stats){ sortTootsByLength(stats) {
const statKeys = Object.keys(stats); const statKeys = Object.keys(stats);
const arrayToSort = []; const arrayToSort = [];
statKeys.forEach(elem => { statKeys.forEach(elem => {
@ -88,10 +139,28 @@ class Conversion {
stats[elem], stats[elem],
); );
}); });
arrayToSort.sort( (a,b)=>{ arrayToSort.sort((a, b) => {
return b.counter - a.counter return b.counter - a.counter;
}); });
return arrayToSort; return arrayToSort;
} }
urlToUser(url) {
let sliceOfSlashes = url.split('/');
let userObject = {
url : url,
username: sliceOfSlashes[sliceOfSlashes.length - 1],
};
return userObject;
}
filterOnlyTootsWithMedias(tootList) {
console.log('filterOnlyTootsWithMedias')
return tootList.filter(toot => {
return toot['object'].attachment && toot['object'].attachment.length
});
}
} }
exports.conversion = new Conversion(); exports.conversion = new Conversion();

48
main.js
View File

@ -10,58 +10,68 @@ var fs = require('fs');
var listenPort = 8088; var listenPort = 8088;
var jsonParsedLikes, jsonParsedOutbox; var jsonParsedLikes, jsonParsedOutbox;
// const min_length = 1050; // filter only long toots // const min_length = 1050; // filter only long toots
const min_length = 300; // filter only long toots const min_length = 1; // filter only long toots
const max_toots = 500; // filter only long toots const max_toots = 10; // filter only long toots
const filterBiggerTottsBeforeSlicing = false; // filter only long toots const filterBiggerTottsBeforeSlicing = false; // filter only long toots
const filterOnlyHavingMedias = true; // filter only toots having medias
const writeStatsJson = false; // filter only toots having medias
const TemplateVars = { const TemplateVars = {
pageTitle : 'Mastodon export converter to HTML', pageTitle : 'Mastodon export converter to HTML',
likes : jsonParsedLikes, likes : jsonParsedLikes,
outbox : jsonParsedOutbox, outbox : jsonParsedOutbox,
outboxStatistics: {}, outboxStatistics: {},
outbox_all : jsonParsedOutbox, outbox_all : jsonParsedOutbox,
min_length , min_length,
max_toots , max_toots,
filterOnlyHavingMedias,
filterBiggerTottsBeforeSlicing,
writeStatsJson,
}; };
const masto_converter = require('./conversion'); const masto_converter = require('./conversion');
console.log('masto_converter', masto_converter); console.log('masto_converter', masto_converter);
masto_converter.conversion.hello(); masto_converter.conversion.hello();
jsonParsedLikes = masto_converter.conversion.likes(); jsonParsedLikes = masto_converter.conversion.likes();
fs.readFile('source_data/outbox.json', fs.readFile('source_data/outbox.json',
// callback function that is called when reading file is done // callback function that is called when reading file is done
function (err, data) { function (err, data) {
let minchartoots ; let toots;
// parse json // parse json
jsonParsedOutbox = JSON.parse(data); jsonParsedOutbox = JSON.parse(data);
toots = jsonParsedOutbox.orderedItems;
// access elements // access elements
console.log('outbox toots length', jsonParsedOutbox.orderedItems.length); console.log('outbox toots length', toots.length);
TemplateVars.outboxTotalLength = jsonParsedOutbox.orderedItems.length; TemplateVars.outboxTotalLength = toots.length;
minchartoots = masto_converter.conversion.filterToots(jsonParsedOutbox.orderedItems, TemplateVars) toots = jsonParsedOutbox.orderedItems;
if (filterOnlyHavingMedias) {
toots = masto_converter.conversion.filterOnlyTootsWithMedias(toots);
console.log('toots.length only attachements', toots.length );
}
toots = masto_converter.conversion.filterToots(toots, TemplateVars);
console.log('min_chars', min_length); console.log('min_chars', min_length);
console.log('toots min char corresponding', minchartoots.length); console.log('toots min char corresponding', toots.length);
TemplateVars.outbox = minchartoots; TemplateVars.outbox = toots;
TemplateVars.outboxStatistics = masto_converter.conversion.makeStatsForToots(minchartoots); TemplateVars.outboxStatistics = masto_converter.conversion.makeStatsForToots(toots);
const example = TemplateVars.outbox[1]['object']; const example = TemplateVars.outbox[1]['object'];
TemplateVars.example = example; TemplateVars.example = example;
console.log('example', example)
fs.writeFile('output/statistics.json', JSON.stringify(TemplateVars.outboxStatistics), errfileHandler ); if (writeStatsJson) {
fs.writeFile('output/statistics.json', JSON.stringify(TemplateVars.outboxStatistics), errfileHandler);
}
}); });
const errfileHandler = (err) => { const errfileHandler = (err) => {
if (err) if (err)
console.log(err); console.log(err);
else { else {
console.log("File statistics written successfully\n"); console.log('File statistics written successfully\n');
} }
} };
app.use(express.static('public')); app.use(express.static('public'));
app.set('view engine', 'pug'); app.set('view engine', 'pug');
@ -70,7 +80,7 @@ app.get('/', (req, res) => {
const html = pug.render('index.pug', TemplateVars); const html = pug.render('index.pug', TemplateVars);
fs.writeFile('output/my_toots.html', html, errfileHandler ); fs.writeFile('output/my_toots.html', html, errfileHandler);
res.render('index.pug', TemplateVars); res.render('index.pug', TemplateVars);

40
public/stylesheets/main.css
View File

@ -1,24 +1,42 @@
body{ body {
background: #222; background: #292a2a;
color: white; color: white;
padding: 1em 2em; padding: 1em 2em;
} }
a{
color: #5561ff; a {
color: #00a7d1;
} }
.status{
background: #111; .status {
background: #606984;
padding: 0.5em; padding: 0.5em;
border-radius: 0.25em; border-radius: 0.25em;
border-bottom: 1px solid #42495c; border-bottom: 1px solid #42495c;
} }
.date-published{
.published {
width: 70ch;
}
.date-published {
font-size: 0.5em; font-size: 0.5em;
} }
.status .published{
padding-left: 2em; .stats {
}
.stats{
max-height: 15em; max-height: 15em;
overflow: auto; overflow: auto;
} }
.media-gallery {
}
.media-media-displayed {
min-height: 500px;
margin-right: 1em;
}
.hidden {
display: none;
}

31
views/index.pug
View File

@ -13,13 +13,17 @@ html(lang="en")
table table
thead thead
tr tr
th= "search"
th= "hashtag" th= "hashtag"
th= "occurences" th= "occurences"
tbody tbody
each hashtag in outboxStatistics.hashtags each hashtag in outboxStatistics.hashtags
tr tr
td.search
a(href= 'https://duckduckgo.com/?q='+hashtag.name)
='search'
td.name td.name
a(href=hashtag.name)=hashtag.name a(href=hashtag.href)=hashtag.name
td.counter=hashtag.counter td.counter=hashtag.counter
td.counter=hashtag.counterContentLength td.counter=hashtag.counterContentLength
@ -28,14 +32,18 @@ html(lang="en")
table table
thead thead
tr tr
th= "search"
th= "name" th= "name"
th= "times" th= "times"
th= "toots lengh sum" th= "toots lengh sum"
tbody tbody
each someone in outboxStatistics.recievers each someone in outboxStatistics.recievers
tr tr
td.search
a(href= 'https://duckduckgo.com/?q='+someone.user.username)
='search'
td.name td.name
a(href=someone.name)=someone.name a(href=someone.name)=someone.user.username
td.counter=someone.counter td.counter=someone.counter
td.counter=someone.counterContentLength td.counter=someone.counterContentLength
h2 #{outbox.length} of Messages #{outboxTotalLength} in your outbox.First #{max_toots} toots, filtered by a minimal length of #{min_length} characters of content. h2 #{outbox.length} of Messages #{outboxTotalLength} in your outbox.First #{max_toots} toots, filtered by a minimal length of #{min_length} characters of content.
@ -47,4 +55,21 @@ html(lang="en")
div.status.status-public div.status.status-public
a(href=oredredItem['object'].url)="see" a(href=oredredItem['object'].url)="see"
div.date-published=oredredItem['object'].published div.date-published=oredredItem['object'].published
blockquote.published(unescaped!=oredredItem['object'].content) blockquote.published(escaped!=oredredItem['object'].content)
if oredredItem['object'].attachment && oredredItem['object'].attachment.length
each media in oredredItem['object'].attachment
div.media-gallery
div.counter.hidden #{oredredItem['object'].attachment.length}
if(media.mediaType.search('video'))
figure.media-media-displayed.media-image
a(href=media.href)
video(src=media.href)
span Media vidéo
elseif(media.mediaType.search('audio'))
span Media audio
elseif(media.mediaType.search('image'))
figure.media-media-displayed.media-image
a(href=media.href)
img(src=media.href,alt=media.href)