📖 some more doc
This commit is contained in:
parent
2d1c839891
commit
3bda33cc18
|
@ -20,6 +20,12 @@ and you will see the result in html.
|
||||||
## What statistics does this provide
|
## What statistics does this provide
|
||||||
For the moment:
|
For the moment:
|
||||||
* who you talked to the most, with counters. This uses the object.cc field of a toot.
|
* who you talked to the most, with counters. This uses the object.cc field of a toot.
|
||||||
|
* link to search on duckduckgo for hashtags and usernames
|
||||||
|
* what are your most used hashtags
|
||||||
|
* you can filter:
|
||||||
|
* only toots with medias
|
||||||
|
* change the order of filter
|
||||||
|
* min length of toot content
|
||||||
* what toots are containing at least a certain number of characters. You can use this to retrieve some long posts you made, to post them on a worthy place where you would develop your subject deeper. Somewhere like on your personal website, thats an easy way to find your longests posts. But be careful, it could also show how much you talk with trolls :D
|
* what toots are containing at least a certain number of characters. You can use this to retrieve some long posts you made, to post them on a worthy place where you would develop your subject deeper. Somewhere like on your personal website, thats an easy way to find your longests posts. But be careful, it could also show how much you talk with trolls :D
|
||||||
|
|
||||||
## configuration
|
## configuration
|
||||||
|
|
115
conversion.js
115
conversion.js
|
@ -2,10 +2,11 @@ exports.name = 'masto_conversion';
|
||||||
const fs = require('fs');
|
const fs = require('fs');
|
||||||
|
|
||||||
class Conversion {
|
class Conversion {
|
||||||
hello(){
|
hello() {
|
||||||
console.log('hello from conversion')
|
console.log('hello from conversion');
|
||||||
}
|
}
|
||||||
likes(){
|
|
||||||
|
likes() {
|
||||||
// read file likes
|
// read file likes
|
||||||
fs.readFile('source_data/likes.json',
|
fs.readFile('source_data/likes.json',
|
||||||
// callback function that is called when reading file is done
|
// callback function that is called when reading file is done
|
||||||
|
@ -19,50 +20,100 @@ class Conversion {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
filterToots(toots,options){
|
filterToots(toots, options) {
|
||||||
let minchartoots ;
|
let minchartoots;
|
||||||
|
|
||||||
if(options.filterBiggerTottsBeforeSlicing){
|
if (options.filterBiggerTottsBeforeSlicing) {
|
||||||
minchartoots = toots.filter(item => {
|
minchartoots = toots.filter(item => {
|
||||||
return item['object'].content && item['object'].content.length > options.min_length;
|
return item['object'].content && item['object'].content.length > options.min_length;
|
||||||
});
|
});
|
||||||
minchartoots = minchartoots.slice(0, options.max_toots);
|
minchartoots = minchartoots.slice(0, options.max_toots);
|
||||||
}else{
|
} else {
|
||||||
const slice = toots.slice(0, options.max_toots);
|
const slice = toots.slice(0, options.max_toots);
|
||||||
minchartoots = slice.filter(item => {
|
minchartoots = slice.filter(item => {
|
||||||
return item['object'].content && item['object'].content.length > options.min_length;
|
return item['object'].content && item['object'].content.length > options.min_length;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
return minchartoots
|
minchartoots.forEach(toot => {
|
||||||
|
toot = this.findMediaUrl(toot);
|
||||||
|
toot = this.removeLastChars(toot);
|
||||||
|
return toot;
|
||||||
|
});
|
||||||
|
|
||||||
|
return minchartoots;
|
||||||
}
|
}
|
||||||
|
|
||||||
makeStatsForToots(tootArray){
|
//
|
||||||
let stats = { recievers: {},
|
/**
|
||||||
hashtags : {}};
|
*
|
||||||
|
* @param toot
|
||||||
|
* @returns {*}
|
||||||
|
*/
|
||||||
|
findMediaUrl(toot) {
|
||||||
|
/**
|
||||||
|
* goal:
|
||||||
|
* https://mastodon.cipherbliss.com/system/media_attachments/files/000/858/113/original/74b370672892f884.jpg?1566230144
|
||||||
|
*
|
||||||
|
* input data:
|
||||||
|
* "attributedTo":"https://mastodon.cipherbliss.com/users/tykayn",
|
||||||
|
*
|
||||||
|
* "attachment":[{"type":"Document",
|
||||||
|
* "mediaType":"image/png","url":"media_attachments/files/000/872/910/original/c82b422f302b8ec9.png",
|
||||||
|
* "name":null,
|
||||||
|
* "blurhash":"UnSOjgo~ysWAVYWBkWjXu5axVrjckqoze?Rk"}],
|
||||||
|
*
|
||||||
|
* we use the attribuedTo property to find instance, and map the url to the instance url, and add this property to the attachment
|
||||||
|
*/
|
||||||
|
if (toot['object'].attributedTo) {
|
||||||
|
|
||||||
|
let splitted = toot['object'].attributedTo.split('/');
|
||||||
|
let instanceUrl = splitted[2];
|
||||||
|
|
||||||
|
toot.instanceUrl = 'https://' + instanceUrl;
|
||||||
|
toot.attachment = toot['object'].attachment.map(att => {
|
||||||
|
att.href = toot.instanceUrl + '/system/' + att.url;
|
||||||
|
return att;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return toot;
|
||||||
|
}
|
||||||
|
|
||||||
|
removeLastChars(toot) {
|
||||||
|
toot['object'].content = toot['object'].content.trim();
|
||||||
|
return toot;
|
||||||
|
}
|
||||||
|
|
||||||
|
makeStatsForToots(tootArray) {
|
||||||
|
let stats = {
|
||||||
|
recievers: {},
|
||||||
|
hashtags : {},
|
||||||
|
};
|
||||||
// make statistics on who do we talk to, based on the cc field
|
// make statistics on who do we talk to, based on the cc field
|
||||||
tootArray.forEach(elem => {
|
tootArray.forEach(elem => {
|
||||||
|
|
||||||
// stats on hashtags
|
// stats on hashtags
|
||||||
if (elem['object'].tag) {
|
if (elem['object'].tag) {
|
||||||
elem['object'].tag.forEach(tag => {
|
elem['object'].tag.forEach(tag => {
|
||||||
if(tag.type === 'Hashtag'){
|
if (tag.type === 'Hashtag') {
|
||||||
if(!stats.hashtags[tag.name]){
|
if (!stats.hashtags[tag.name]) {
|
||||||
stats.hashtags[tag.name] = {
|
stats.hashtags[tag.name] = {
|
||||||
name : tag.name,
|
name : tag.name,
|
||||||
counter : 0,
|
href : tag.href,
|
||||||
}
|
counter: 0,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
stats.hashtags[tag.name].counter++;
|
stats.hashtags[tag.name].counter++;
|
||||||
}
|
}
|
||||||
})
|
});
|
||||||
}
|
}
|
||||||
// stats on recievers of toots
|
// stats on recievers of toots
|
||||||
if (elem['object'].cc) {
|
if (elem['object'].cc) {
|
||||||
elem['object'].cc.forEach(copyFolk => {
|
elem['object'].cc.forEach(copyFolk => {
|
||||||
if (!stats.recievers[copyFolk]) {
|
if (!stats.recievers[copyFolk]) {
|
||||||
stats.recievers[copyFolk] = {
|
stats.recievers[copyFolk] = {
|
||||||
|
user : this.urlToUser(copyFolk),
|
||||||
name : copyFolk,
|
name : copyFolk,
|
||||||
counter: 0,
|
counter : 0,
|
||||||
counterContentLength: 0,
|
counterContentLength: 0,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
@ -72,15 +123,15 @@ class Conversion {
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
console.log('stats.hashtags', stats.hashtags);
|
console.log('stats.hashtags', stats.hashtags[0]);
|
||||||
stats = {
|
stats = {
|
||||||
recievers : this.sortTootsByLength(stats.recievers),
|
recievers: this.sortTootsByLength(stats.recievers),
|
||||||
hashtags : this.sortTootsByLength(stats.hashtags),
|
hashtags : this.sortTootsByLength(stats.hashtags),
|
||||||
};
|
};
|
||||||
return stats ;
|
return stats;
|
||||||
}
|
}
|
||||||
|
|
||||||
sortTootsByLength(stats){
|
sortTootsByLength(stats) {
|
||||||
const statKeys = Object.keys(stats);
|
const statKeys = Object.keys(stats);
|
||||||
const arrayToSort = [];
|
const arrayToSort = [];
|
||||||
statKeys.forEach(elem => {
|
statKeys.forEach(elem => {
|
||||||
|
@ -88,10 +139,28 @@ class Conversion {
|
||||||
stats[elem],
|
stats[elem],
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
arrayToSort.sort( (a,b)=>{
|
arrayToSort.sort((a, b) => {
|
||||||
return b.counter - a.counter
|
return b.counter - a.counter;
|
||||||
});
|
});
|
||||||
return arrayToSort;
|
return arrayToSort;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
urlToUser(url) {
|
||||||
|
let sliceOfSlashes = url.split('/');
|
||||||
|
let userObject = {
|
||||||
|
url : url,
|
||||||
|
username: sliceOfSlashes[sliceOfSlashes.length - 1],
|
||||||
|
};
|
||||||
|
return userObject;
|
||||||
|
}
|
||||||
|
|
||||||
|
filterOnlyTootsWithMedias(tootList) {
|
||||||
|
console.log('filterOnlyTootsWithMedias')
|
||||||
|
return tootList.filter(toot => {
|
||||||
|
return toot['object'].attachment && toot['object'].attachment.length
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
exports.conversion = new Conversion();
|
exports.conversion = new Conversion();
|
||||||
|
|
48
main.js
48
main.js
|
@ -10,58 +10,68 @@ var fs = require('fs');
|
||||||
var listenPort = 8088;
|
var listenPort = 8088;
|
||||||
var jsonParsedLikes, jsonParsedOutbox;
|
var jsonParsedLikes, jsonParsedOutbox;
|
||||||
// const min_length = 1050; // filter only long toots
|
// const min_length = 1050; // filter only long toots
|
||||||
const min_length = 300; // filter only long toots
|
const min_length = 1; // filter only long toots
|
||||||
const max_toots = 500; // filter only long toots
|
const max_toots = 10; // filter only long toots
|
||||||
const filterBiggerTottsBeforeSlicing = false; // filter only long toots
|
const filterBiggerTottsBeforeSlicing = false; // filter only long toots
|
||||||
|
const filterOnlyHavingMedias = true; // filter only toots having medias
|
||||||
|
const writeStatsJson = false; // filter only toots having medias
|
||||||
const TemplateVars = {
|
const TemplateVars = {
|
||||||
pageTitle : 'Mastodon export converter to HTML',
|
pageTitle : 'Mastodon export converter to HTML',
|
||||||
likes : jsonParsedLikes,
|
likes : jsonParsedLikes,
|
||||||
outbox : jsonParsedOutbox,
|
outbox : jsonParsedOutbox,
|
||||||
outboxStatistics: {},
|
outboxStatistics: {},
|
||||||
outbox_all : jsonParsedOutbox,
|
outbox_all : jsonParsedOutbox,
|
||||||
min_length ,
|
min_length,
|
||||||
max_toots ,
|
max_toots,
|
||||||
|
filterOnlyHavingMedias,
|
||||||
|
filterBiggerTottsBeforeSlicing,
|
||||||
|
writeStatsJson,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
const masto_converter = require('./conversion');
|
const masto_converter = require('./conversion');
|
||||||
console.log('masto_converter', masto_converter);
|
console.log('masto_converter', masto_converter);
|
||||||
masto_converter.conversion.hello();
|
masto_converter.conversion.hello();
|
||||||
|
|
||||||
jsonParsedLikes = masto_converter.conversion.likes();
|
jsonParsedLikes = masto_converter.conversion.likes();
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
fs.readFile('source_data/outbox.json',
|
fs.readFile('source_data/outbox.json',
|
||||||
// callback function that is called when reading file is done
|
// callback function that is called when reading file is done
|
||||||
function (err, data) {
|
function (err, data) {
|
||||||
let minchartoots ;
|
let toots;
|
||||||
// parse json
|
// parse json
|
||||||
jsonParsedOutbox = JSON.parse(data);
|
jsonParsedOutbox = JSON.parse(data);
|
||||||
|
toots = jsonParsedOutbox.orderedItems;
|
||||||
// access elements
|
// access elements
|
||||||
console.log('outbox toots length', jsonParsedOutbox.orderedItems.length);
|
console.log('outbox toots length', toots.length);
|
||||||
|
|
||||||
TemplateVars.outboxTotalLength = jsonParsedOutbox.orderedItems.length;
|
TemplateVars.outboxTotalLength = toots.length;
|
||||||
minchartoots = masto_converter.conversion.filterToots(jsonParsedOutbox.orderedItems, TemplateVars)
|
toots = jsonParsedOutbox.orderedItems;
|
||||||
|
if (filterOnlyHavingMedias) {
|
||||||
|
toots = masto_converter.conversion.filterOnlyTootsWithMedias(toots);
|
||||||
|
console.log('toots.length only attachements', toots.length );
|
||||||
|
}
|
||||||
|
toots = masto_converter.conversion.filterToots(toots, TemplateVars);
|
||||||
|
|
||||||
console.log('min_chars', min_length);
|
console.log('min_chars', min_length);
|
||||||
console.log('toots min char corresponding', minchartoots.length);
|
console.log('toots min char corresponding', toots.length);
|
||||||
TemplateVars.outbox = minchartoots;
|
TemplateVars.outbox = toots;
|
||||||
TemplateVars.outboxStatistics = masto_converter.conversion.makeStatsForToots(minchartoots);
|
TemplateVars.outboxStatistics = masto_converter.conversion.makeStatsForToots(toots);
|
||||||
|
|
||||||
const example = TemplateVars.outbox[1]['object'];
|
const example = TemplateVars.outbox[1]['object'];
|
||||||
TemplateVars.example = example;
|
TemplateVars.example = example;
|
||||||
|
console.log('example', example)
|
||||||
|
|
||||||
fs.writeFile('output/statistics.json', JSON.stringify(TemplateVars.outboxStatistics), errfileHandler );
|
if (writeStatsJson) {
|
||||||
|
fs.writeFile('output/statistics.json', JSON.stringify(TemplateVars.outboxStatistics), errfileHandler);
|
||||||
|
}
|
||||||
});
|
});
|
||||||
const errfileHandler = (err) => {
|
const errfileHandler = (err) => {
|
||||||
if (err)
|
if (err)
|
||||||
console.log(err);
|
console.log(err);
|
||||||
else {
|
else {
|
||||||
console.log("File statistics written successfully\n");
|
console.log('File statistics written successfully\n');
|
||||||
}
|
}
|
||||||
}
|
};
|
||||||
|
|
||||||
|
|
||||||
app.use(express.static('public'));
|
app.use(express.static('public'));
|
||||||
app.set('view engine', 'pug');
|
app.set('view engine', 'pug');
|
||||||
|
@ -70,7 +80,7 @@ app.get('/', (req, res) => {
|
||||||
|
|
||||||
const html = pug.render('index.pug', TemplateVars);
|
const html = pug.render('index.pug', TemplateVars);
|
||||||
|
|
||||||
fs.writeFile('output/my_toots.html', html, errfileHandler );
|
fs.writeFile('output/my_toots.html', html, errfileHandler);
|
||||||
|
|
||||||
res.render('index.pug', TemplateVars);
|
res.render('index.pug', TemplateVars);
|
||||||
|
|
||||||
|
|
|
@ -1,24 +1,42 @@
|
||||||
body{
|
body {
|
||||||
background: #222;
|
background: #292a2a;
|
||||||
color: white;
|
color: white;
|
||||||
padding: 1em 2em;
|
padding: 1em 2em;
|
||||||
}
|
}
|
||||||
a{
|
|
||||||
color: #5561ff;
|
a {
|
||||||
|
color: #00a7d1;
|
||||||
}
|
}
|
||||||
.status{
|
|
||||||
background: #111;
|
.status {
|
||||||
|
background: #606984;
|
||||||
padding: 0.5em;
|
padding: 0.5em;
|
||||||
border-radius: 0.25em;
|
border-radius: 0.25em;
|
||||||
border-bottom: 1px solid #42495c;
|
border-bottom: 1px solid #42495c;
|
||||||
}
|
}
|
||||||
.date-published{
|
|
||||||
|
.published {
|
||||||
|
width: 70ch;
|
||||||
|
}
|
||||||
|
|
||||||
|
.date-published {
|
||||||
font-size: 0.5em;
|
font-size: 0.5em;
|
||||||
}
|
}
|
||||||
.status .published{
|
|
||||||
padding-left: 2em;
|
.stats {
|
||||||
}
|
|
||||||
.stats{
|
|
||||||
max-height: 15em;
|
max-height: 15em;
|
||||||
overflow: auto;
|
overflow: auto;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.media-gallery {
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
.media-media-displayed {
|
||||||
|
min-height: 500px;
|
||||||
|
margin-right: 1em;
|
||||||
|
}
|
||||||
|
|
||||||
|
.hidden {
|
||||||
|
display: none;
|
||||||
|
}
|
||||||
|
|
|
@ -13,13 +13,17 @@ html(lang="en")
|
||||||
table
|
table
|
||||||
thead
|
thead
|
||||||
tr
|
tr
|
||||||
|
th= "search"
|
||||||
th= "hashtag"
|
th= "hashtag"
|
||||||
th= "occurences"
|
th= "occurences"
|
||||||
tbody
|
tbody
|
||||||
each hashtag in outboxStatistics.hashtags
|
each hashtag in outboxStatistics.hashtags
|
||||||
tr
|
tr
|
||||||
|
td.search
|
||||||
|
a(href= 'https://duckduckgo.com/?q='+hashtag.name)
|
||||||
|
='search'
|
||||||
td.name
|
td.name
|
||||||
a(href=hashtag.name)=hashtag.name
|
a(href=hashtag.href)=hashtag.name
|
||||||
td.counter=hashtag.counter
|
td.counter=hashtag.counter
|
||||||
td.counter=hashtag.counterContentLength
|
td.counter=hashtag.counterContentLength
|
||||||
|
|
||||||
|
@ -28,14 +32,18 @@ html(lang="en")
|
||||||
table
|
table
|
||||||
thead
|
thead
|
||||||
tr
|
tr
|
||||||
|
th= "search"
|
||||||
th= "name"
|
th= "name"
|
||||||
th= "times"
|
th= "times"
|
||||||
th= "toots lengh sum"
|
th= "toots lengh sum"
|
||||||
tbody
|
tbody
|
||||||
each someone in outboxStatistics.recievers
|
each someone in outboxStatistics.recievers
|
||||||
tr
|
tr
|
||||||
|
td.search
|
||||||
|
a(href= 'https://duckduckgo.com/?q='+someone.user.username)
|
||||||
|
='search'
|
||||||
td.name
|
td.name
|
||||||
a(href=someone.name)=someone.name
|
a(href=someone.name)=someone.user.username
|
||||||
td.counter=someone.counter
|
td.counter=someone.counter
|
||||||
td.counter=someone.counterContentLength
|
td.counter=someone.counterContentLength
|
||||||
h2 #{outbox.length} of Messages #{outboxTotalLength} in your outbox.First #{max_toots} toots, filtered by a minimal length of #{min_length} characters of content.
|
h2 #{outbox.length} of Messages #{outboxTotalLength} in your outbox.First #{max_toots} toots, filtered by a minimal length of #{min_length} characters of content.
|
||||||
|
@ -47,4 +55,21 @@ html(lang="en")
|
||||||
div.status.status-public
|
div.status.status-public
|
||||||
a(href=oredredItem['object'].url)="see"
|
a(href=oredredItem['object'].url)="see"
|
||||||
div.date-published=oredredItem['object'].published
|
div.date-published=oredredItem['object'].published
|
||||||
blockquote.published(unescaped!=oredredItem['object'].content)
|
blockquote.published(escaped!=oredredItem['object'].content)
|
||||||
|
if oredredItem['object'].attachment && oredredItem['object'].attachment.length
|
||||||
|
each media in oredredItem['object'].attachment
|
||||||
|
div.media-gallery
|
||||||
|
div.counter.hidden #{oredredItem['object'].attachment.length}
|
||||||
|
if(media.mediaType.search('video'))
|
||||||
|
figure.media-media-displayed.media-image
|
||||||
|
a(href=media.href)
|
||||||
|
video(src=media.href)
|
||||||
|
span Media vidéo
|
||||||
|
elseif(media.mediaType.search('audio'))
|
||||||
|
span Media audio
|
||||||
|
elseif(media.mediaType.search('image'))
|
||||||
|
figure.media-media-displayed.media-image
|
||||||
|
a(href=media.href)
|
||||||
|
img(src=media.href,alt=media.href)
|
||||||
|
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue