exports.name = 'masto_conversion'; const fs = require('fs'); const axios = require('axios'); class Conversion { constructor() { this.fetched_times = 0; this.max_fetchUsers = 5; this.usersMemo = {}; } hello() { console.log('hello from conversion'); } likes() { // read file likes fs.readFile('source_data/likes.json', // callback function that is called when reading file is done function (err, data) { // parse json let jsonParsedLikes = JSON.parse(data); // access elements const lengthOfLikes = jsonParsedLikes.orderedItems.length; console.log('likes length', lengthOfLikes); return jsonParsedLikes; }); return {} } filterToots(toots, options) { let filteredToots = toots; let filter = true; console.log('before slicing filteredToots.length', filteredToots.length) filteredToots = filteredToots.filter(item => { if(item['object'].content){ item['object'].content = item['object'].content.trim() } return item; }); if (filter) { if (options.filterOnlyHavingMedias) { toots = this.filterOnlyTootsWithMedias(toots); console.log('toots.length only attachements', toots.length); } if (options.showMostRecentTootsOnTop) { filteredToots = filteredToots.reverse(); } if (options.searchTerm) { console.log('filtrer les toots ayant pour contenu: ', options.searchTerm); filteredToots = filteredToots.filter(item => { return (item['object'].content && item['object'].content.length && item['object'].content.indexOf(options.searchTerm) !== -1); }); } if (options.filterBiggerTootsBeforeSlicing) { console.log('min length of content of toots ', options.min_length ); filteredToots = filteredToots.filter(item => { return (item['object'].content && item['object'].content.length >= options.min_length && item['object'].content.length <= options.max_length); }); filteredToots = filteredToots.slice(0, options.max_toots); } else { const slice = toots.slice(0, options.max_toots); filteredToots = slice.filter(item => { return (item['object'].content && item['object'].content.length >= options.min_length && item['object'].content.length <= options.max_length); }); } console.log('after slicing filteredToots.length', filteredToots.length) } return filteredToots; } // /** * * @param toot * @returns {*} */ findMediaUrl(toot) { /** * goal: * https://mastodon.cipherbliss.com/system/media_attachments/files/000/858/113/original/74b370672892f884.jpg?1566230144 * * input data: * "attributedTo":"https://mastodon.cipherbliss.com/users/tykayn", * * "attachment":[{"type":"Document", * "mediaType":"image/png","url":"media_attachments/files/000/872/910/original/c82b422f302b8ec9.png", * "name":null, * "blurhash":"UnSOjgo~ysWAVYWBkWjXu5axVrjckqoze?Rk"}], * * we use the attribuedTo property to find instance, and map the url to the instance url, and add this property to the attachment */ if (toot['object'].attributedTo) { let splitted = toot['object'].attributedTo.split('/'); let instanceUrl = splitted[2]; toot.instanceUrl = 'https://' + instanceUrl; toot.attachment = toot['object'].attachment.map(att => { att.href = toot.instanceUrl + '/system/' + att.url; return att; }); } return toot; } findInstanceFromAttributedToUrl(attributedTo) { let splitted = attributedTo.split('/'); return splitted[2]; } removeLastChars(toot) { toot['object'].content = toot['object'].content.trim(); return toot; } makeStatsForToots(tootArray) { let stats = { recievers: {}, hashtags: {}, }; // make statistics on who do we talk to, based on the cc field tootArray.forEach(elem => { // stats on hashtags if (elem['object'].tag) { elem['object'].tag.forEach(tag => { if (tag.type === 'Hashtag') { if (!stats.hashtags[tag.name]) { stats.hashtags[tag.name] = { name: tag.name, href: tag.href, counter: 0, }; } stats.hashtags[tag.name].counter++; } }); } // stats on recievers of toots if (elem['object'].cc) { elem['object'].cc.forEach(urlOfUser => { if (!stats.recievers[urlOfUser]) { stats.recievers[urlOfUser] = { user: this.urlToUser(urlOfUser), name: urlOfUser, infos: this.fetchUserInfo(urlOfUser), counter: 0, counterContentLength: 0, }; } stats.recievers[urlOfUser].counter++; stats.recievers[urlOfUser].counterContentLength += elem['object'].content.length; }); } }); stats = { recievers: this.sortTootsByLength(stats.recievers), hashtags: this.sortTootsByLength(stats.hashtags), }; return stats; } sortTootsByLength(stats) { const statKeys = Object.keys(stats); const arrayToSort = []; statKeys.forEach(elem => { arrayToSort.push( stats[elem], ); }); arrayToSort.sort((a, b) => { return b.counter - a.counter; }); return arrayToSort; } urlToUser(url) { let sliceOfSlashes = url.split('/'); let userObject = { url: url, username: sliceOfSlashes[sliceOfSlashes.length - 1], }; return userObject; } filterOnlyTootsWithMedias(tootList) { console.log('filterOnlyTootsWithMedias from', tootList.length); let filtered = tootList.filter(toot => { return toot['object'].attachment && toot['object'].attachment.length; }); console.log('to ', filtered.length); return filtered } /** * find user avatar url from its user homepage url. * API is: * https://mastodon.social/.well-known/webfinger?resource=acct:gargron@mastodon.social * @param attributeToUrl */ fetchUserInfo(attributeToUrl = 'https://mastodon.cipherbliss.com/users/tykayn') { // do not fetch "followers" and "activitystreams#Public" if ( attributeToUrl.indexOf('followers') !== -1 || attributeToUrl.indexOf('activitystreams#Public') !== -1 ) { return {}; } // do not fetch too much if (this.max_fetchUsers <= this.fetched_times) { return {}; } this.fetched_times++; let instanceHandle = this.findInstanceFromAttributedToUrl(attributeToUrl); let splitted = attributeToUrl.split('/'); let accountHandle = splitted[splitted.length - 1]; // memo things let memoEntry = this.usersMemo[accountHandle + '@' + instanceHandle]; if (memoEntry) { return memoEntry; } // webfinger version // let fetchUri = `https://${instanceHandle}/.well-known/webfinger?resource=acct:${accountHandle}@${instanceHandle}`; // fetchUri = `https://${instanceHandle}/api/v1/account/${accountHandle}@${instanceHandle}` // fetchUri = 'https://mastodon.cipherbliss.com/users/tykayn'; let headers = { method: 'GET', url: attributeToUrl, headers: { accept: 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"', }, }; const self = this; const ax = axios.create(headers); // ax.get().then(function (response) { // // handle success // // avatar is response.data.icon.url // // cover is response.data.image.url // // console.log(response.data.icon); // // console.log(response.data.image); // // if (!self.usersMemo[accountHandle + '@' + instanceHandle]) { // self.usersMemo[accountHandle + '@' + instanceHandle] = {}; // } // self.usersMemo[accountHandle + '@' + instanceHandle] = response.data; // return response.data; // }); } } exports.conversion = new Conversion();