exports.name = 'masto_conversion'; const fs = require('fs'); const axios = require('axios'); class Conversion { constructor( ) { this.fetched_times = 0; this.max_fetchUsers = 5; this.usersMemo = {}; } hello() { console.log('hello from conversion'); } likes() { // read file likes fs.readFile('source_data/likes.json', // callback function that is called when reading file is done function (err, data) { // parse json let jsonParsedLikes = JSON.parse(data); // access elements const lengthOfLikes = jsonParsedLikes.orderedItems.length; console.log('likes length', lengthOfLikes); return jsonParsedLikes; }); } filterToots(toots, options) { let filteredToots = toots; if (options.showMostRecentTootsOnTop) { filteredToots = filteredToots.reverse(); } if (options.filterBiggerTottsBeforeSlicing) { filteredToots = filteredToots.filter(item => { return item['object'].content && item['object'].content.length > options.min_length; }); filteredToots = filteredToots.slice(0, options.max_toots); } else { const slice = toots.slice(0, options.max_toots); filteredToots = slice.filter(item => { return item['object'].content && item['object'].content.length > options.min_length; }); } filteredToots.forEach(toot => { toot = this.findMediaUrl(toot); toot = this.removeLastChars(toot); return toot; }); return filteredToots; } // /** * * @param toot * @returns {*} */ findMediaUrl(toot) { /** * goal: * https://mastodon.cipherbliss.com/system/media_attachments/files/000/858/113/original/74b370672892f884.jpg?1566230144 * * input data: * "attributedTo":"https://mastodon.cipherbliss.com/users/tykayn", * * "attachment":[{"type":"Document", * "mediaType":"image/png","url":"media_attachments/files/000/872/910/original/c82b422f302b8ec9.png", * "name":null, * "blurhash":"UnSOjgo~ysWAVYWBkWjXu5axVrjckqoze?Rk"}], * * we use the attribuedTo property to find instance, and map the url to the instance url, and add this property to the attachment */ if (toot['object'].attributedTo) { let splitted = toot['object'].attributedTo.split('/'); let instanceUrl = splitted[2]; toot.instanceUrl = 'https://' + instanceUrl; toot.attachment = toot['object'].attachment.map(att => { att.href = toot.instanceUrl + '/system/' + att.url; return att; }); } return toot; } findInstanceFromAttributedToUrl(attributedTo) { let splitted = attributedTo.split('/'); return splitted[2]; } removeLastChars(toot) { toot['object'].content = toot['object'].content.trim(); return toot; } makeStatsForToots(tootArray) { let stats = { recievers: {}, hashtags : {}, }; // make statistics on who do we talk to, based on the cc field tootArray.forEach(elem => { // stats on hashtags if (elem['object'].tag) { elem['object'].tag.forEach(tag => { if (tag.type === 'Hashtag') { if (!stats.hashtags[tag.name]) { stats.hashtags[tag.name] = { name : tag.name, href : tag.href, counter: 0, }; } stats.hashtags[tag.name].counter++; } }); } // stats on recievers of toots if (elem['object'].cc) { elem['object'].cc.forEach(urlOfUser => { console.log('copyFolk', urlOfUser); if (!stats.recievers[urlOfUser]) { stats.recievers[urlOfUser] = { user : this.urlToUser(urlOfUser), name : urlOfUser, infos : this.fetchUserInfo(urlOfUser), counter : 0, counterContentLength: 0, }; } stats.recievers[urlOfUser].counter++; stats.recievers[urlOfUser].counterContentLength += elem['object'].content.length; }); } }); console.log('stats.hashtags', stats.hashtags[0]); stats = { recievers: this.sortTootsByLength(stats.recievers), hashtags : this.sortTootsByLength(stats.hashtags), }; return stats; } sortTootsByLength(stats) { const statKeys = Object.keys(stats); const arrayToSort = []; statKeys.forEach(elem => { arrayToSort.push( stats[elem], ); }); arrayToSort.sort((a, b) => { return b.counter - a.counter; }); return arrayToSort; } urlToUser(url) { let sliceOfSlashes = url.split('/'); let userObject = { url : url, username: sliceOfSlashes[sliceOfSlashes.length - 1], }; return userObject; } filterOnlyTootsWithMedias(tootList) { console.log('filterOnlyTootsWithMedias'); return tootList.filter(toot => { return toot['object'].attachment && toot['object'].attachment.length; }); } /** * find user avatar url from its user homepage url. * API is: * https://mastodon.social/.well-known/webfinger?resource=acct:gargron@mastodon.social * @param attributeToUrl */ fetchUserInfo(attributeToUrl = 'https://mastodon.cipherbliss.com/users/tykayn') { // do not fetch "followers" and "activitystreams#Public" if ( attributeToUrl.indexOf('followers') !== -1 || attributeToUrl.indexOf('activitystreams#Public') !== -1 ) { return {}; } // do not fetch too much if (this.max_fetchUsers <= this.fetched_times) { return {}; } this.fetched_times++; console.log('attributeToUrl', attributeToUrl); // return {}; let instanceHandle = this.findInstanceFromAttributedToUrl(attributeToUrl); let splitted = attributeToUrl.split('/'); let accountHandle = splitted[splitted.length - 1]; console.log('splitted', splitted); // memo things let memoEntry = this.usersMemo[accountHandle + '@' + instanceHandle]; if (memoEntry) { return memoEntry; } // webfinger version // let fetchUri = `https://${instanceHandle}/.well-known/webfinger?resource=acct:${accountHandle}@${instanceHandle}`; // fetchUri = `https://${instanceHandle}/api/v1/account/${accountHandle}@${instanceHandle}` // fetchUri = 'https://mastodon.cipherbliss.com/users/tykayn'; let headers = { method : 'GET', url : attributeToUrl, headers: { accept: 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"', }, }; const self = this; const ax = axios.create(headers); ax.get().then(function (response) { // handle success // avatar is response.data.icon.url // cover is response.data.image.url console.log(response.data.icon); console.log(response.data.image); if (!self.usersMemo[accountHandle + '@' + instanceHandle]) { self.usersMemo[accountHandle + '@' + instanceHandle] = {}; } self.usersMemo[accountHandle + '@' + instanceHandle] = response.data; return response.data; }); } } exports.conversion = new Conversion();