diff --git a/conversion.js b/conversion.js index 48bde4d..0a226e4 100755 --- a/conversion.js +++ b/conversion.js @@ -1,5 +1,6 @@ exports.name = 'masto_conversion'; const fs = require('fs'); +const axios = require('axios'); class Conversion { hello() { @@ -23,7 +24,7 @@ class Conversion { filterToots(toots, options) { let filteredToots = toots; - if(options.showMostRecentTootsOnTop){ + if (options.showMostRecentTootsOnTop) { filteredToots = filteredToots.reverse(); } if (options.filterBiggerTottsBeforeSlicing) { @@ -81,6 +82,11 @@ class Conversion { return toot; } + findInstanceFromAttributedToUrl(attributedTo) { + let splitted = attributedTo.split('/'); + return splitted[2]; + } + removeLastChars(toot) { toot['object'].content = toot['object'].content.trim(); return toot; @@ -111,17 +117,19 @@ class Conversion { } // stats on recievers of toots if (elem['object'].cc) { - elem['object'].cc.forEach(copyFolk => { - if (!stats.recievers[copyFolk]) { - stats.recievers[copyFolk] = { - user : this.urlToUser(copyFolk), - name : copyFolk, + elem['object'].cc.forEach(urlOfUser => { + console.log('copyFolk', urlOfUser); + if (!stats.recievers[urlOfUser]) { + stats.recievers[urlOfUser] = { + user : this.urlToUser(urlOfUser), + name : urlOfUser, + infos : this.fetchUserInfo(urlOfUser), counter : 0, counterContentLength: 0, }; } - stats.recievers[copyFolk].counter++; - stats.recievers[copyFolk].counterContentLength += elem['object'].content.length; + stats.recievers[urlOfUser].counter++; + stats.recievers[urlOfUser].counterContentLength += elem['object'].content.length; }); } }); @@ -158,14 +166,75 @@ class Conversion { } filterOnlyTootsWithMedias(tootList) { - console.log('filterOnlyTootsWithMedias') + console.log('filterOnlyTootsWithMedias'); return tootList.filter(toot => { - return toot['object'].attachment && toot['object'].attachment.length + return toot['object'].attachment && toot['object'].attachment.length; }); } - userAvatars= {}; - fetchUserAvatar(user){ + fetched_times = 0; + max_fetchUsers = 5; + usersMemo = {}; + + /** + * find user avatar url from its user homepage url. + * API is: + * https://mastodon.social/.well-known/webfinger?resource=acct:gargron@mastodon.social + * @param attributeToUrl + */ + fetchUserInfo(attributeToUrl = 'https://mastodon.cipherbliss.com/users/tykayn') { + + // do not fetch "followers" and "activitystreams#Public" + if ( + attributeToUrl.indexOf('followers') !== -1 || + attributeToUrl.indexOf('activitystreams#Public') !== -1 + ) { + return {}; + } + // do not fetch too much + if (this.max_fetchUsers <= this.fetched_times) { + return {}; + } + this.fetched_times++; + console.log('attributeToUrl', attributeToUrl); + // return {}; + + let instanceHandle = this.findInstanceFromAttributedToUrl(attributeToUrl); + let splitted = attributeToUrl.split('/'); + let accountHandle = splitted[splitted.length - 1]; + console.log('splitted', splitted); + // memo things + let memoEntry = this.usersMemo[accountHandle + '@' + instanceHandle]; + if (memoEntry) { + return memoEntry; + } + // webfinger version + // let fetchUri = `https://${instanceHandle}/.well-known/webfinger?resource=acct:${accountHandle}@${instanceHandle}`; + // fetchUri = `https://${instanceHandle}/api/v1/account/${accountHandle}@${instanceHandle}` + // fetchUri = 'https://mastodon.cipherbliss.com/users/tykayn'; + let headers = { + method : 'GET', + url : attributeToUrl, + headers: { + accept: 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"', + }, + }; + + const self = this; + const ax = axios.create(headers); + ax.get().then(function (response) { + // handle success + // avatar is response.data.icon.url + // cover is response.data.image.url + console.log(response.data.icon); + console.log(response.data.image); + + if (!self.usersMemo[accountHandle + '@' + instanceHandle]) { + self.usersMemo[accountHandle + '@' + instanceHandle] = {}; + } + self.usersMemo[accountHandle + '@' + instanceHandle] = response.data; + return response.data; + }); } diff --git a/main.js b/main.js index d0b0b4f..2768c61 100755 --- a/main.js +++ b/main.js @@ -11,9 +11,10 @@ var listenPort = 8088; var jsonParsedLikes, jsonParsedOutbox; // const min_length = 1050; // filter only long toots const min_length = 1; // filter only long toots -const max_toots = 10; // filter only long toots +const max_toots = 50; // filter only long toots const filterBiggerTottsBeforeSlicing = false; // filter only long toots const filterOnlyHavingMedias = true; // filter only toots having medias +const displayMedias = false; // filter only toots having medias const writeStatsJson = false; // filter only toots having medias const showMostRecentTootsOnTop = true; // filter only toots having medias const TemplateVars = { @@ -27,7 +28,8 @@ const TemplateVars = { filterOnlyHavingMedias, filterBiggerTottsBeforeSlicing, writeStatsJson, - showMostRecentTootsOnTop + showMostRecentTootsOnTop, + displayMedias, }; const masto_converter = require('./conversion'); @@ -50,7 +52,7 @@ fs.readFile('source_data/outbox.json', toots = jsonParsedOutbox.orderedItems; if (filterOnlyHavingMedias) { toots = masto_converter.conversion.filterOnlyTootsWithMedias(toots); - console.log('toots.length only attachements', toots.length ); + console.log('toots.length only attachements', toots.length); } toots = masto_converter.conversion.filterToots(toots, TemplateVars); @@ -61,7 +63,7 @@ fs.readFile('source_data/outbox.json', const example = TemplateVars.outbox[1]['object']; TemplateVars.example = example; - console.log('example', example) + // console.log('example', example) if (writeStatsJson) { fs.writeFile('output/statistics.json', JSON.stringify(TemplateVars.outboxStatistics), errfileHandler); @@ -82,11 +84,13 @@ app.get('/', (req, res) => { const html = pug.render('index.pug', TemplateVars); - fs.writeFile('output/my_toots.html', html, errfileHandler); + // fs.writeFile('output/my_toots.html', html, errfileHandler); res.render('index.pug', TemplateVars); + masto_converter.conversion.fetchUserInfo(); }); + masto_converter.conversion.fetchUserInfo(); app.listen(listenPort, () => console.log(`Server is live at http://localhost:${listenPort}`)); diff --git a/package.json b/package.json index 2825245..69c909b 100644 --- a/package.json +++ b/package.json @@ -3,9 +3,11 @@ "start": "nodemon main.js" }, "dependencies": { + "axios": "^0.19.2", "bulma": "^0.9.0", "express": "^4.17.1", "flaticon": "^0.1.0", + "node-fetch": "^2.6.0", "nodemon": "^2.0.4", "pug": "^3.0.0" } diff --git a/views/partials/config.pug b/views/partials/config.pug index e370062..b0e7f87 100644 --- a/views/partials/config.pug +++ b/views/partials/config.pug @@ -5,3 +5,5 @@ div.config p write statistics in output folder #{writeStatsJson} p showing only toots with medias attached to them #{filterOnlyHavingMedias} p show most recent toot first #{showMostRecentTootsOnTop} + if(!displayMedias) + p.no-media Medias are not displayed diff --git a/views/partials/contacts.pug b/views/partials/contacts.pug index dd4cdc3..d917f00 100644 --- a/views/partials/contacts.pug +++ b/views/partials/contacts.pug @@ -1,9 +1,11 @@ fieldset.stats.stats-recievers - h3 You sent messages to these people + h3 You sent messages to these #{outboxStatistics.recievers.length} people table thead tr th= "search" + th= "cover" + th= "avatar" th= "name" th= "times" th= "toots lengh sum" @@ -13,6 +15,14 @@ fieldset.stats.stats-recievers td.search a(href= 'https://duckduckgo.com/?q='+someone.user.username) ='search' + td.cover + if(someone.infos && someone.infos.image) + a(href=someone.infos.image.url) + img(src=someone.infos.image.url) + td.avatar + if(someone.infos && someone.infos.icon) + a(href=someone.infos.icon.url) + img(src=someone.infos.icon.url) td.name a(href=someone.name)=someone.user.username td.counter=someone.counter diff --git a/views/partials/toots.pug b/views/partials/toots.pug index a2461c3..95b3b0a 100644 --- a/views/partials/toots.pug +++ b/views/partials/toots.pug @@ -1,6 +1,8 @@ .columns-area__panels__main img.header(src='/header.jpg') + + div.column div.item-list each oredredItem in outbox @@ -12,5 +14,8 @@ div.date-published=oredredItem['object'].published blockquote.published(escaped!=oredredItem['object'].content) if oredredItem['object'].attachment && oredredItem['object'].attachment.length - each media in oredredItem['object'].attachment - include medias.pug + if(!displayMedias) + sub.no-media Medias are not displayed + if(displayMedias) + each media in oredredItem['object'].attachment + include medias.pug diff --git a/yarn.lock b/yarn.lock index 0f53c8f..1e29adf 100644 --- a/yarn.lock +++ b/yarn.lock @@ -141,6 +141,13 @@ aws4@^1.8.0: resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.10.0.tgz#a17b3a8ea811060e74d47d306122400ad4497ae2" integrity sha512-3YDiu347mtVtjpyV3u5kVqQLP242c06zwDOgpeRnybmXlYYsLbtTrUBUm8i8srONt+FWobl5aibnU1030PeeuA== +axios@^0.19.2: + version "0.19.2" + resolved "https://registry.yarnpkg.com/axios/-/axios-0.19.2.tgz#3ea36c5d8818d0d5f8a8a97a6d36b86cdc00cb27" + integrity sha512-fjgm5MvRHLhx+osE2xoekY70AhARk3a6hkN+3Io1jc00jtquGvxYlKlsFUhmUET0V5te6CcZI7lcv2Ym61mjHA== + dependencies: + follow-redirects "1.5.10" + babel-walk@3.0.0-canary-5: version "3.0.0-canary-5" resolved "https://registry.yarnpkg.com/babel-walk/-/babel-walk-3.0.0-canary-5.tgz#f66ecd7298357aee44955f235a6ef54219104b11" @@ -385,6 +392,13 @@ debug@2.6.9, debug@^2.2.0: dependencies: ms "2.0.0" +debug@=3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261" + integrity sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g== + dependencies: + ms "2.0.0" + debug@^3.2.6: version "3.2.6" resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.6.tgz#e83d17de16d8a7efb7717edbe5fb10135eee629b" @@ -581,6 +595,13 @@ flaticon@^0.1.0: request "^2.88.2" request-promise "^4.2.5" +follow-redirects@1.5.10: + version "1.5.10" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.5.10.tgz#7b7a9f9aea2fdff36786a94ff643ed07f4ff5e2a" + integrity sha512-0V5l4Cizzvqt5D44aTXbFZz+FtyXV1vrDN6qrelxtfYQKW0KO0W2T/hkE8xvGa/540LkZlkaUjO4ailYTFtHVQ== + dependencies: + debug "=3.1.0" + forever-agent@~0.6.1: version "0.6.1" resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" @@ -1035,6 +1056,11 @@ negotiator@0.6.2: resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.2.tgz#feacf7ccf525a77ae9634436a64883ffeca346fb" integrity sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw== +node-fetch@^2.6.0: + version "2.6.0" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd" + integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA== + node-html-parser@^1.2.16: version "1.2.20" resolved "https://registry.yarnpkg.com/node-html-parser/-/node-html-parser-1.2.20.tgz#37e9ebc627dbe3ff446eea4ac93e3d254b7c6ee4"