mastodon-archive-stats/conversion.js

exports.name = 'masto_conversion';
const fs = require('fs');
const axios = require('axios');


class Conversion {
  constructor( ) {
    this.fetched_times = 0;
    this.max_fetchUsers = 5;
    this.usersMemo = {};
  }

  hello() {
    console.log('hello from conversion');
  }

  likes() {
    // read file likes
    fs.readFile('source_data/likes.json',
      // callback function that is called when reading file is done
      function (err, data) {
        // parse json
        let jsonParsedLikes = JSON.parse(data);
        // access elements
        const lengthOfLikes = jsonParsedLikes.orderedItems.length;
        console.log('likes length', lengthOfLikes);
        return jsonParsedLikes;
      });
  }

  filterToots(toots, options) {
    let filteredToots = toots;

    if (options.showMostRecentTootsOnTop) {
      filteredToots = filteredToots.reverse();
    }
    if (options.filterBiggerTottsBeforeSlicing) {
      filteredToots = filteredToots.filter(item => {
        return item['object'].content && item['object'].content.length > options.min_length;
      });
      filteredToots = filteredToots.slice(0, options.max_toots);
    } else {
      const slice = toots.slice(0, options.max_toots);
      filteredToots = slice.filter(item => {
        return item['object'].content && item['object'].content.length > options.min_length;
      });
    }
    filteredToots.forEach(toot => {
      toot = this.findMediaUrl(toot);
      toot = this.removeLastChars(toot);
      return toot;
    });

    return filteredToots;
  }

  //
  /**
   *
   * @param toot
   * @returns {*}
   */
  findMediaUrl(toot) {
    /**
     * goal:
     * https://mastodon.cipherbliss.com/system/media_attachments/files/000/858/113/original/74b370672892f884.jpg?1566230144
     *
     * input data:
     * "attributedTo":"https://mastodon.cipherbliss.com/users/tykayn",
     *
     * "attachment":[{"type":"Document",
     * "mediaType":"image/png","url":"media_attachments/files/000/872/910/original/c82b422f302b8ec9.png",
     * "name":null,
     * "blurhash":"UnSOjgo~ysWAVYWBkWjXu5axVrjckqoze?Rk"}],
     *
     * we use the attribuedTo property to find instance, and map the url to the instance url, and add this property to the attachment
     */
    if (toot['object'].attributedTo) {

      let splitted = toot['object'].attributedTo.split('/');
      let instanceUrl = splitted[2];

      toot.instanceUrl = 'https://' + instanceUrl;
      toot.attachment = toot['object'].attachment.map(att => {
        att.href = toot.instanceUrl + '/system/' + att.url;
        return att;
      });
    }
    return toot;
  }

  findInstanceFromAttributedToUrl(attributedTo) {
    let splitted = attributedTo.split('/');
    return splitted[2];
  }

  removeLastChars(toot) {
    toot['object'].content = toot['object'].content.trim();
    return toot;
  }

  makeStatsForToots(tootArray) {
    let stats = {
      recievers: {},
      hashtags : {},
    };
    // make statistics on who do we talk to, based on the cc field
    tootArray.forEach(elem => {

      // stats on hashtags
      if (elem['object'].tag) {
        elem['object'].tag.forEach(tag => {
          if (tag.type === 'Hashtag') {
            if (!stats.hashtags[tag.name]) {
              stats.hashtags[tag.name] = {
                name   : tag.name,
                href   : tag.href,
                counter: 0,
              };
            }
            stats.hashtags[tag.name].counter++;
          }
        });
      }
      // stats on recievers of toots
      if (elem['object'].cc) {
        elem['object'].cc.forEach(urlOfUser => {
          console.log('copyFolk', urlOfUser);
          if (!stats.recievers[urlOfUser]) {
            stats.recievers[urlOfUser] = {
              user                : this.urlToUser(urlOfUser),
              name                : urlOfUser,
              infos               : this.fetchUserInfo(urlOfUser),
              counter             : 0,
              counterContentLength: 0,
            };
          }
          stats.recievers[urlOfUser].counter++;
          stats.recievers[urlOfUser].counterContentLength += elem['object'].content.length;
        });
      }
    });

    console.log('stats.hashtags', stats.hashtags[0]);
    stats = {
      recievers: this.sortTootsByLength(stats.recievers),
      hashtags : this.sortTootsByLength(stats.hashtags),
    };
    return stats;
  }

  sortTootsByLength(stats) {
    const statKeys = Object.keys(stats);
    const arrayToSort = [];
    statKeys.forEach(elem => {
      arrayToSort.push(
        stats[elem],
      );
    });
    arrayToSort.sort((a, b) => {
      return b.counter - a.counter;
    });
    return arrayToSort;
  }

  urlToUser(url) {
    let sliceOfSlashes = url.split('/');
    let userObject = {
      url     : url,
      username: sliceOfSlashes[sliceOfSlashes.length - 1],
    };
    return userObject;
  }

  filterOnlyTootsWithMedias(tootList) {
    console.log('filterOnlyTootsWithMedias');
    return tootList.filter(toot => {
      return toot['object'].attachment && toot['object'].attachment.length;
    });
  }


  /**
   * find user avatar url from its user homepage url.
   * API is:
   * https://mastodon.social/.well-known/webfinger?resource=acct:gargron@mastodon.social
   * @param attributeToUrl
   */
  fetchUserInfo(attributeToUrl = 'https://mastodon.cipherbliss.com/users/tykayn') {

    // do not fetch "followers" and "activitystreams#Public"
    if (
      attributeToUrl.indexOf('followers') !== -1 ||
      attributeToUrl.indexOf('activitystreams#Public') !== -1
    ) {
      return {};
    }
    // do not fetch too much
    if (this.max_fetchUsers <= this.fetched_times) {
      return {};
    }
    this.fetched_times++;
    console.log('attributeToUrl', attributeToUrl);
    // return {};

    let instanceHandle = this.findInstanceFromAttributedToUrl(attributeToUrl);
    let splitted = attributeToUrl.split('/');
    let accountHandle = splitted[splitted.length - 1];
    console.log('splitted', splitted);
    // memo things
    let memoEntry = this.usersMemo[accountHandle + '@' + instanceHandle];
    if (memoEntry) {
      return memoEntry;
    }
    // webfinger version
    // let fetchUri = `https://${instanceHandle}/.well-known/webfinger?resource=acct:${accountHandle}@${instanceHandle}`;
    // fetchUri = `https://${instanceHandle}/api/v1/account/${accountHandle}@${instanceHandle}`
    // fetchUri = 'https://mastodon.cipherbliss.com/users/tykayn';
    let headers = {
      method : 'GET',
      url    : attributeToUrl,
      headers: {
        accept: 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
      },
    };

    const self = this;
    const ax = axios.create(headers);
    ax.get().then(function (response) {
      // handle success
      // avatar is response.data.icon.url
      // cover is response.data.image.url
      console.log(response.data.icon);
      console.log(response.data.image);

      if (!self.usersMemo[accountHandle + '@' + instanceHandle]) {
        self.usersMemo[accountHandle + '@' + instanceHandle] = {};
      }
      self.usersMemo[accountHandle + '@' + instanceHandle] = response.data;
      return response.data;
    });

  }

}

exports.conversion = new Conversion();