mastodon-archive-stats/conversion.js

exports.name = 'masto_conversion';
const fs = require('fs');
const axios = require('axios');


class Conversion {
    constructor() {
        this.fetched_times = 0;
        this.max_fetchUsers = 5;
        this.usersMemo = {};
    }

    hello() {
        console.log('hello from conversion');
    }

    likes() {
        // read file likes
        fs.readFile('source_data/likes.json',
          // callback function that is called when reading file is done
          function (err, data) {
            // parse json
            let jsonParsedLikes = JSON.parse(data);
            // access elements
            const lengthOfLikes = jsonParsedLikes.orderedItems.length;
            console.log('likes length', lengthOfLikes);
            return jsonParsedLikes;
          });
        return {}
    }

    filterToots(toots, options) {
        let filteredToots = toots;
        let filter = true;
        console.log('before slicing filteredToots.length', filteredToots.length)

        filteredToots = filteredToots.filter(item => {
            if(item['object'].content){

            item['object'].content = item['object'].content.trim()
            }
            return item;
        });

        if (filter) {

            if (options.filterOnlyHavingMedias) {
                toots = this.filterOnlyTootsWithMedias(toots);
                console.log('toots.length only attachements', toots.length);
            }
            if (options.showMostRecentTootsOnTop) {
                filteredToots = filteredToots.reverse();
            }
            if (options.searchTerm) {
                console.log('filtrer les toots ayant pour contenu: ', options.searchTerm);
                filteredToots = filteredToots.filter(item => {
                    return (item['object'].content && item['object'].content.length  && item['object'].content.indexOf(options.searchTerm) !== -1);
                });
            }
            if (options.filterBiggerTootsBeforeSlicing) {
                console.log('min length of content of toots ', options.min_length );
                filteredToots = filteredToots.filter(item => {
                    return (item['object'].content && item['object'].content.length >= options.min_length && item['object'].content.length <= options.max_length);
                });

                filteredToots = filteredToots.slice(0, options.max_toots);
            } else {
                const slice = toots.slice(0, options.max_toots);
                filteredToots = slice.filter(item => {
                    return (item['object'].content && item['object'].content.length >= options.min_length && item['object'].content.length <= options.max_length);
                });
            }
            console.log('after slicing filteredToots.length', filteredToots.length)


        }

        return filteredToots;
    }

    //
    /**
     *
     * @param toot
     * @returns {*}
     */
    findMediaUrl(toot) {
        /**
         * goal:
         * https://mastodon.cipherbliss.com/system/media_attachments/files/000/858/113/original/74b370672892f884.jpg?1566230144
         *
         * input data:
         * "attributedTo":"https://mastodon.cipherbliss.com/users/tykayn",
         *
         * "attachment":[{"type":"Document",
         * "mediaType":"image/png","url":"media_attachments/files/000/872/910/original/c82b422f302b8ec9.png",
         * "name":null,
         * "blurhash":"UnSOjgo~ysWAVYWBkWjXu5axVrjckqoze?Rk"}],
         *
         * we use the attribuedTo property to find instance, and map the url to the instance url, and add this property to the attachment
         */
        if (toot['object'].attributedTo) {

            let splitted = toot['object'].attributedTo.split('/');
            let instanceUrl = splitted[2];

            toot.instanceUrl = 'https://' + instanceUrl;
            toot.attachment = toot['object'].attachment.map(att => {
                att.href = toot.instanceUrl + '/system/' + att.url;
                return att;
            });
        }
        return toot;
    }

    findInstanceFromAttributedToUrl(attributedTo) {
        let splitted = attributedTo.split('/');
        return splitted[2];
    }

    removeLastChars(toot) {
        toot['object'].content = toot['object'].content.trim();
        return toot;
    }

    makeStatsForToots(tootArray) {
        let stats = {
            recievers: {},
            hashtags: {},
        };
        // make statistics on who do we talk to, based on the cc field
        tootArray.forEach(elem => {

            // stats on hashtags
            if (elem['object'].tag) {
                elem['object'].tag.forEach(tag => {
                    if (tag.type === 'Hashtag') {
                        if (!stats.hashtags[tag.name]) {
                            stats.hashtags[tag.name] = {
                                name: tag.name,
                                href: tag.href,
                                counter: 0,
                            };
                        }
                        stats.hashtags[tag.name].counter++;
                    }
                });
            }
            // stats on recievers of toots
            if (elem['object'].cc) {
                elem['object'].cc.forEach(urlOfUser => {
                    if (!stats.recievers[urlOfUser]) {
                        stats.recievers[urlOfUser] = {
                            user: this.urlToUser(urlOfUser),
                            name: urlOfUser,
                            infos: this.fetchUserInfo(urlOfUser),
                            counter: 0,
                            counterContentLength: 0,
                        };
                    }
                    stats.recievers[urlOfUser].counter++;
                    stats.recievers[urlOfUser].counterContentLength += elem['object'].content.length;
                });
            }
        });

        stats = {
            recievers: this.sortTootsByLength(stats.recievers),
            hashtags: this.sortTootsByLength(stats.hashtags),
        };
        return stats;
    }

    sortTootsByLength(stats) {
        const statKeys = Object.keys(stats);
        const arrayToSort = [];
        statKeys.forEach(elem => {
            arrayToSort.push(
                stats[elem],
            );
        });
        arrayToSort.sort((a, b) => {
            return b.counter - a.counter;
        });
        return arrayToSort;
    }

    urlToUser(url) {
        let sliceOfSlashes = url.split('/');
        let userObject = {
            url: url,
            username: sliceOfSlashes[sliceOfSlashes.length - 1],
        };
        return userObject;
    }

    filterOnlyTootsWithMedias(tootList) {
        console.log('filterOnlyTootsWithMedias from', tootList.length);
        let filtered = tootList.filter(toot => {
            return toot['object'].attachment && toot['object'].attachment.length;
        });
        console.log('to ', filtered.length);
        return filtered
    }


    /**
     * find user avatar url from its user homepage url.
     * API is:
     * https://mastodon.social/.well-known/webfinger?resource=acct:gargron@mastodon.social
     * @param attributeToUrl
     */
    fetchUserInfo(attributeToUrl = 'https://mastodon.cipherbliss.com/users/tykayn') {

        // do not fetch "followers" and "activitystreams#Public"
        if (
            attributeToUrl.indexOf('followers') !== -1 ||
            attributeToUrl.indexOf('activitystreams#Public') !== -1
        ) {
            return {};
        }
        // do not fetch too much
        if (this.max_fetchUsers <= this.fetched_times) {
            return {};
        }
        this.fetched_times++;

        let instanceHandle = this.findInstanceFromAttributedToUrl(attributeToUrl);
        let splitted = attributeToUrl.split('/');
        let accountHandle = splitted[splitted.length - 1];
        // memo things
        let memoEntry = this.usersMemo[accountHandle + '@' + instanceHandle];
        if (memoEntry) {
            return memoEntry;
        }
        // webfinger version
        // let fetchUri = `https://${instanceHandle}/.well-known/webfinger?resource=acct:${accountHandle}@${instanceHandle}`;
        // fetchUri = `https://${instanceHandle}/api/v1/account/${accountHandle}@${instanceHandle}`
        // fetchUri = 'https://mastodon.cipherbliss.com/users/tykayn';
        let headers = {
            method: 'GET',
            url: attributeToUrl,
            headers: {
                accept: 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
            },
        };

        const self = this;
        const ax = axios.create(headers);
        // ax.get().then(function (response) {
        //     // handle success
        //     // avatar is response.data.icon.url
        //     // cover is response.data.image.url
        //     // console.log(response.data.icon);
        //     // console.log(response.data.image);
        //
        //     if (!self.usersMemo[accountHandle + '@' + instanceHandle]) {
        //         self.usersMemo[accountHandle + '@' + instanceHandle] = {};
        //     }
        //     self.usersMemo[accountHandle + '@' + instanceHandle] = response.data;
        //     return response.data;
        // });

    }

}

exports.conversion = new Conversion();