diff --git a/README.md b/README.md index e5fa83f..494c2e0 100755 --- a/README.md +++ b/README.md @@ -20,6 +20,12 @@ and you will see the result in html. ## What statistics does this provide For the moment: * who you talked to the most, with counters. This uses the object.cc field of a toot. +* link to search on duckduckgo for hashtags and usernames +* what are your most used hashtags +* you can filter: + * only toots with medias + * change the order of filter + * min length of toot content * what toots are containing at least a certain number of characters. You can use this to retrieve some long posts you made, to post them on a worthy place where you would develop your subject deeper. Somewhere like on your personal website, thats an easy way to find your longests posts. But be careful, it could also show how much you talk with trolls :D ## configuration diff --git a/conversion.js b/conversion.js index 348d026..4f42bd7 100755 --- a/conversion.js +++ b/conversion.js @@ -2,10 +2,11 @@ exports.name = 'masto_conversion'; const fs = require('fs'); class Conversion { - hello(){ - console.log('hello from conversion') + hello() { + console.log('hello from conversion'); } - likes(){ + + likes() { // read file likes fs.readFile('source_data/likes.json', // callback function that is called when reading file is done @@ -19,68 +20,118 @@ class Conversion { }); } - filterToots(toots,options){ - let minchartoots ; + filterToots(toots, options) { + let minchartoots; - if(options.filterBiggerTottsBeforeSlicing){ + if (options.filterBiggerTottsBeforeSlicing) { minchartoots = toots.filter(item => { return item['object'].content && item['object'].content.length > options.min_length; }); minchartoots = minchartoots.slice(0, options.max_toots); - }else{ + } else { const slice = toots.slice(0, options.max_toots); minchartoots = slice.filter(item => { return item['object'].content && item['object'].content.length > options.min_length; }); } - return minchartoots + minchartoots.forEach(toot => { + toot = this.findMediaUrl(toot); + toot = this.removeLastChars(toot); + return toot; + }); + + return minchartoots; } - makeStatsForToots(tootArray){ - let stats = { recievers: {}, - hashtags : {}}; - // make statistics on who do we talk to, based on the cc field - tootArray.forEach(elem => { + // + /** + * + * @param toot + * @returns {*} + */ + findMediaUrl(toot) { + /** + * goal: + * https://mastodon.cipherbliss.com/system/media_attachments/files/000/858/113/original/74b370672892f884.jpg?1566230144 + * + * input data: + * "attributedTo":"https://mastodon.cipherbliss.com/users/tykayn", + * + * "attachment":[{"type":"Document", + * "mediaType":"image/png","url":"media_attachments/files/000/872/910/original/c82b422f302b8ec9.png", + * "name":null, + * "blurhash":"UnSOjgo~ysWAVYWBkWjXu5axVrjckqoze?Rk"}], + * + * we use the attribuedTo property to find instance, and map the url to the instance url, and add this property to the attachment + */ + if (toot['object'].attributedTo) { - // stats on hashtags - if (elem['object'].tag) { - elem['object'].tag.forEach(tag => { - if(tag.type === 'Hashtag'){ - if(!stats.hashtags[tag.name]){ - stats.hashtags[tag.name] = { - name : tag.name, - counter : 0, - } - } - stats.hashtags[tag.name].counter++; - } - }) - } - // stats on recievers of toots - if (elem['object'].cc) { - elem['object'].cc.forEach(copyFolk => { - if (!stats.recievers[copyFolk]) { - stats.recievers[copyFolk] = { - name : copyFolk, + let splitted = toot['object'].attributedTo.split('/'); + let instanceUrl = splitted[2]; + + toot.instanceUrl = 'https://' + instanceUrl; + toot.attachment = toot['object'].attachment.map(att => { + att.href = toot.instanceUrl + '/system/' + att.url; + return att; + }); + } + return toot; + } + + removeLastChars(toot) { + toot['object'].content = toot['object'].content.trim(); + return toot; + } + + makeStatsForToots(tootArray) { + let stats = { + recievers: {}, + hashtags : {}, + }; + // make statistics on who do we talk to, based on the cc field + tootArray.forEach(elem => { + + // stats on hashtags + if (elem['object'].tag) { + elem['object'].tag.forEach(tag => { + if (tag.type === 'Hashtag') { + if (!stats.hashtags[tag.name]) { + stats.hashtags[tag.name] = { + name : tag.name, + href : tag.href, counter: 0, - counterContentLength: 0, }; } - stats.recievers[copyFolk].counter++; - stats.recievers[copyFolk].counterContentLength += elem['object'].content.length; - }); - } - }); + stats.hashtags[tag.name].counter++; + } + }); + } + // stats on recievers of toots + if (elem['object'].cc) { + elem['object'].cc.forEach(copyFolk => { + if (!stats.recievers[copyFolk]) { + stats.recievers[copyFolk] = { + user : this.urlToUser(copyFolk), + name : copyFolk, + counter : 0, + counterContentLength: 0, + }; + } + stats.recievers[copyFolk].counter++; + stats.recievers[copyFolk].counterContentLength += elem['object'].content.length; + }); + } + }); - console.log('stats.hashtags', stats.hashtags); - stats = { - recievers : this.sortTootsByLength(stats.recievers), - hashtags : this.sortTootsByLength(stats.hashtags), - }; - return stats ; + console.log('stats.hashtags', stats.hashtags[0]); + stats = { + recievers: this.sortTootsByLength(stats.recievers), + hashtags : this.sortTootsByLength(stats.hashtags), + }; + return stats; } - sortTootsByLength(stats){ + sortTootsByLength(stats) { const statKeys = Object.keys(stats); const arrayToSort = []; statKeys.forEach(elem => { @@ -88,10 +139,28 @@ class Conversion { stats[elem], ); }); - arrayToSort.sort( (a,b)=>{ - return b.counter - a.counter + arrayToSort.sort((a, b) => { + return b.counter - a.counter; }); return arrayToSort; } + + urlToUser(url) { + let sliceOfSlashes = url.split('/'); + let userObject = { + url : url, + username: sliceOfSlashes[sliceOfSlashes.length - 1], + }; + return userObject; + } + + filterOnlyTootsWithMedias(tootList) { + console.log('filterOnlyTootsWithMedias') + return tootList.filter(toot => { + return toot['object'].attachment && toot['object'].attachment.length + }); + } + } + exports.conversion = new Conversion(); diff --git a/main.js b/main.js index b843ea6..20a5bcc 100755 --- a/main.js +++ b/main.js @@ -10,58 +10,68 @@ var fs = require('fs'); var listenPort = 8088; var jsonParsedLikes, jsonParsedOutbox; // const min_length = 1050; // filter only long toots -const min_length = 300; // filter only long toots -const max_toots = 500; // filter only long toots +const min_length = 1; // filter only long toots +const max_toots = 10; // filter only long toots const filterBiggerTottsBeforeSlicing = false; // filter only long toots +const filterOnlyHavingMedias = true; // filter only toots having medias +const writeStatsJson = false; // filter only toots having medias const TemplateVars = { pageTitle : 'Mastodon export converter to HTML', likes : jsonParsedLikes, outbox : jsonParsedOutbox, outboxStatistics: {}, outbox_all : jsonParsedOutbox, - min_length , - max_toots , + min_length, + max_toots, + filterOnlyHavingMedias, + filterBiggerTottsBeforeSlicing, + writeStatsJson, }; - const masto_converter = require('./conversion'); console.log('masto_converter', masto_converter); masto_converter.conversion.hello(); jsonParsedLikes = masto_converter.conversion.likes(); - - fs.readFile('source_data/outbox.json', // callback function that is called when reading file is done function (err, data) { - let minchartoots ; + let toots; // parse json jsonParsedOutbox = JSON.parse(data); + toots = jsonParsedOutbox.orderedItems; // access elements - console.log('outbox toots length', jsonParsedOutbox.orderedItems.length); + console.log('outbox toots length', toots.length); - TemplateVars.outboxTotalLength = jsonParsedOutbox.orderedItems.length; - minchartoots = masto_converter.conversion.filterToots(jsonParsedOutbox.orderedItems, TemplateVars) + TemplateVars.outboxTotalLength = toots.length; + toots = jsonParsedOutbox.orderedItems; + if (filterOnlyHavingMedias) { + toots = masto_converter.conversion.filterOnlyTootsWithMedias(toots); + console.log('toots.length only attachements', toots.length ); + } + toots = masto_converter.conversion.filterToots(toots, TemplateVars); console.log('min_chars', min_length); - console.log('toots min char corresponding', minchartoots.length); - TemplateVars.outbox = minchartoots; - TemplateVars.outboxStatistics = masto_converter.conversion.makeStatsForToots(minchartoots); + console.log('toots min char corresponding', toots.length); + TemplateVars.outbox = toots; + TemplateVars.outboxStatistics = masto_converter.conversion.makeStatsForToots(toots); const example = TemplateVars.outbox[1]['object']; TemplateVars.example = example; + console.log('example', example) - fs.writeFile('output/statistics.json', JSON.stringify(TemplateVars.outboxStatistics), errfileHandler ); + if (writeStatsJson) { + fs.writeFile('output/statistics.json', JSON.stringify(TemplateVars.outboxStatistics), errfileHandler); + } }); const errfileHandler = (err) => { if (err) console.log(err); else { - console.log("File statistics written successfully\n"); + console.log('File statistics written successfully\n'); } -} - +}; app.use(express.static('public')); app.set('view engine', 'pug'); @@ -70,7 +80,7 @@ app.get('/', (req, res) => { const html = pug.render('index.pug', TemplateVars); - fs.writeFile('output/my_toots.html', html, errfileHandler ); + fs.writeFile('output/my_toots.html', html, errfileHandler); res.render('index.pug', TemplateVars); diff --git a/public/stylesheets/main.css b/public/stylesheets/main.css index 1d6fc75..4255e11 100644 --- a/public/stylesheets/main.css +++ b/public/stylesheets/main.css @@ -1,24 +1,42 @@ -body{ - background: #222; +body { + background: #292a2a; color: white; padding: 1em 2em; } -a{ - color: #5561ff; + +a { + color: #00a7d1; } -.status{ - background: #111; + +.status { + background: #606984; padding: 0.5em; border-radius: 0.25em; border-bottom: 1px solid #42495c; } -.date-published{ + +.published { + width: 70ch; +} + +.date-published { font-size: 0.5em; } -.status .published{ - padding-left: 2em; -} -.stats{ + +.stats { max-height: 15em; overflow: auto; } + +.media-gallery { + +} + +.media-media-displayed { + min-height: 500px; + margin-right: 1em; +} + +.hidden { + display: none; +} diff --git a/views/index.pug b/views/index.pug index 43425c8..a4be68e 100644 --- a/views/index.pug +++ b/views/index.pug @@ -13,13 +13,17 @@ html(lang="en") table thead tr + th= "search" th= "hashtag" th= "occurences" tbody each hashtag in outboxStatistics.hashtags tr + td.search + a(href= 'https://duckduckgo.com/?q='+hashtag.name) + ='search' td.name - a(href=hashtag.name)=hashtag.name + a(href=hashtag.href)=hashtag.name td.counter=hashtag.counter td.counter=hashtag.counterContentLength @@ -28,14 +32,18 @@ html(lang="en") table thead tr + th= "search" th= "name" th= "times" th= "toots lengh sum" tbody each someone in outboxStatistics.recievers tr + td.search + a(href= 'https://duckduckgo.com/?q='+someone.user.username) + ='search' td.name - a(href=someone.name)=someone.name + a(href=someone.name)=someone.user.username td.counter=someone.counter td.counter=someone.counterContentLength h2 #{outbox.length} of Messages #{outboxTotalLength} in your outbox.First #{max_toots} toots, filtered by a minimal length of #{min_length} characters of content. @@ -47,4 +55,21 @@ html(lang="en") div.status.status-public a(href=oredredItem['object'].url)="see" div.date-published=oredredItem['object'].published - blockquote.published(unescaped!=oredredItem['object'].content) + blockquote.published(escaped!=oredredItem['object'].content) + if oredredItem['object'].attachment && oredredItem['object'].attachment.length + each media in oredredItem['object'].attachment + div.media-gallery + div.counter.hidden #{oredredItem['object'].attachment.length} + if(media.mediaType.search('video')) + figure.media-media-displayed.media-image + a(href=media.href) + video(src=media.href) + span Media vidéo + elseif(media.mediaType.search('audio')) + span Media audio + elseif(media.mediaType.search('image')) + figure.media-media-displayed.media-image + a(href=media.href) + img(src=media.href,alt=media.href) + +