fix some stuff

This commit is contained in:
Tykayn 2021-10-22 14:17:19 +02:00 committed by tykayn
parent debae70091
commit afef2388dc
4 changed files with 243 additions and 240 deletions

View File

@ -4,248 +4,256 @@ const axios = require('axios');
class Conversion {
constructor( ) {
this.fetched_times = 0;
this.max_fetchUsers = 5;
this.usersMemo = {};
}
hello() {
console.log('hello from conversion');
}
likes() {
// read file likes
fs.readFile('source_data/likes.json',
// callback function that is called when reading file is done
function (err, data) {
// parse json
let jsonParsedLikes = JSON.parse(data);
// access elements
const lengthOfLikes = jsonParsedLikes.orderedItems.length;
console.log('likes length', lengthOfLikes);
return jsonParsedLikes;
});
}
filterToots(toots, options) {
let filteredToots = toots;
let filter = true;
console.log('before slicing filteredToots.length' , filteredToots.length)
// console.log('filteredToots[0]', filteredToots[0])
if(filter){
if (options.showMostRecentTootsOnTop) {
filteredToots = filteredToots.reverse();
}
if (options.filterBiggerTootsBeforeSlicing) {
filteredToots = filteredToots.filter(item => {
return (item['object'].content && item['object'].content.length > options.min_length && item['object'].content.length < options.max_length);
});
filteredToots = filteredToots.slice(0, options.max_toots);
} else {
const slice = toots.slice(0, options.max_toots);
filteredToots = slice.filter(item => {
return (item['object'].content && item['object'].content.length > options.min_length && item['object'].content.length < options.max_length);
});
}
// filteredToots.forEach(toot => {
// toot = this.findMediaUrl(toot);
// toot = this.removeLastChars(toot);
// return toot;
// });
console.log('after slicing filteredToots.length' , filteredToots.length)
constructor() {
this.fetched_times = 0;
this.max_fetchUsers = 5;
this.usersMemo = {};
}
return filteredToots;
}
//
/**
*
* @param toot
* @returns {*}
*/
findMediaUrl(toot) {
/**
* goal:
* https://mastodon.cipherbliss.com/system/media_attachments/files/000/858/113/original/74b370672892f884.jpg?1566230144
*
* input data:
* "attributedTo":"https://mastodon.cipherbliss.com/users/tykayn",
*
* "attachment":[{"type":"Document",
* "mediaType":"image/png","url":"media_attachments/files/000/872/910/original/c82b422f302b8ec9.png",
* "name":null,
* "blurhash":"UnSOjgo~ysWAVYWBkWjXu5axVrjckqoze?Rk"}],
*
* we use the attribuedTo property to find instance, and map the url to the instance url, and add this property to the attachment
*/
if (toot['object'].attributedTo) {
let splitted = toot['object'].attributedTo.split('/');
let instanceUrl = splitted[2];
toot.instanceUrl = 'https://' + instanceUrl;
toot.attachment = toot['object'].attachment.map(att => {
att.href = toot.instanceUrl + '/system/' + att.url;
return att;
});
hello() {
console.log('hello from conversion');
}
return toot;
}
findInstanceFromAttributedToUrl(attributedTo) {
let splitted = attributedTo.split('/');
return splitted[2];
}
likes() {
// read file likes
fs.readFile('source_data/likes.json',
// callback function that is called when reading file is done
function (err, data) {
// parse json
let jsonParsedLikes = JSON.parse(data);
// access elements
const lengthOfLikes = jsonParsedLikes.orderedItems.length;
console.log('likes length', lengthOfLikes);
return jsonParsedLikes;
});
return {}
}
removeLastChars(toot) {
toot['object'].content = toot['object'].content.trim();
return toot;
}
filterToots(toots, options) {
let filteredToots = toots;
let filter = true;
console.log('before slicing filteredToots.length', filteredToots.length)
// console.log('filteredToots[0]', filteredToots[0])
makeStatsForToots(tootArray) {
let stats = {
recievers: {},
hashtags : {},
};
// make statistics on who do we talk to, based on the cc field
tootArray.forEach(elem => {
if (filter) {
// stats on hashtags
if (elem['object'].tag) {
elem['object'].tag.forEach(tag => {
if (tag.type === 'Hashtag') {
if (!stats.hashtags[tag.name]) {
stats.hashtags[tag.name] = {
name : tag.name,
href : tag.href,
counter: 0,
};
if (options.filterOnlyHavingMedias) {
toots = masto_converter.conversion.filterOnlyTootsWithMedias(toots);
console.log('toots.length only attachements', toots.length);
}
if (options.showMostRecentTootsOnTop) {
filteredToots = filteredToots.reverse();
}
if (options.filterBiggerTootsBeforeSlicing) {
console.log('min length of content of toots ', options.min_length );
filteredToots = filteredToots.filter(item => {
return (item['object'].content && item['object'].content.length >= options.min_length && item['object'].content.length <= options.max_length);
});
filteredToots = filteredToots.slice(0, options.max_toots);
} else {
const slice = toots.slice(0, options.max_toots);
filteredToots = slice.filter(item => {
return (item['object'].content && item['object'].content.length >= options.min_length && item['object'].content.length <= options.max_length);
});
}
// filteredToots.forEach(toot => {
// toot = this.findMediaUrl(toot);
// toot = this.removeLastChars(toot);
// return toot;
// });
console.log('after slicing filteredToots.length', filteredToots.length)
}
return filteredToots;
}
//
/**
*
* @param toot
* @returns {*}
*/
findMediaUrl(toot) {
/**
* goal:
* https://mastodon.cipherbliss.com/system/media_attachments/files/000/858/113/original/74b370672892f884.jpg?1566230144
*
* input data:
* "attributedTo":"https://mastodon.cipherbliss.com/users/tykayn",
*
* "attachment":[{"type":"Document",
* "mediaType":"image/png","url":"media_attachments/files/000/872/910/original/c82b422f302b8ec9.png",
* "name":null,
* "blurhash":"UnSOjgo~ysWAVYWBkWjXu5axVrjckqoze?Rk"}],
*
* we use the attribuedTo property to find instance, and map the url to the instance url, and add this property to the attachment
*/
if (toot['object'].attributedTo) {
let splitted = toot['object'].attributedTo.split('/');
let instanceUrl = splitted[2];
toot.instanceUrl = 'https://' + instanceUrl;
toot.attachment = toot['object'].attachment.map(att => {
att.href = toot.instanceUrl + '/system/' + att.url;
return att;
});
}
return toot;
}
findInstanceFromAttributedToUrl(attributedTo) {
let splitted = attributedTo.split('/');
return splitted[2];
}
removeLastChars(toot) {
toot['object'].content = toot['object'].content.trim();
return toot;
}
makeStatsForToots(tootArray) {
let stats = {
recievers: {},
hashtags: {},
};
// make statistics on who do we talk to, based on the cc field
tootArray.forEach(elem => {
// stats on hashtags
if (elem['object'].tag) {
elem['object'].tag.forEach(tag => {
if (tag.type === 'Hashtag') {
if (!stats.hashtags[tag.name]) {
stats.hashtags[tag.name] = {
name: tag.name,
href: tag.href,
counter: 0,
};
}
stats.hashtags[tag.name].counter++;
}
});
}
// stats on recievers of toots
if (elem['object'].cc) {
elem['object'].cc.forEach(urlOfUser => {
if (!stats.recievers[urlOfUser]) {
stats.recievers[urlOfUser] = {
user: this.urlToUser(urlOfUser),
name: urlOfUser,
infos: this.fetchUserInfo(urlOfUser),
counter: 0,
counterContentLength: 0,
};
}
stats.recievers[urlOfUser].counter++;
stats.recievers[urlOfUser].counterContentLength += elem['object'].content.length;
});
}
stats.hashtags[tag.name].counter++;
}
});
}
// stats on recievers of toots
if (elem['object'].cc) {
elem['object'].cc.forEach(urlOfUser => {
if (!stats.recievers[urlOfUser]) {
stats.recievers[urlOfUser] = {
user : this.urlToUser(urlOfUser),
name : urlOfUser,
infos : this.fetchUserInfo(urlOfUser),
counter : 0,
counterContentLength: 0,
};
}
stats.recievers[urlOfUser].counter++;
stats.recievers[urlOfUser].counterContentLength += elem['object'].content.length;
stats = {
recievers: this.sortTootsByLength(stats.recievers),
hashtags: this.sortTootsByLength(stats.hashtags),
};
return stats;
}
sortTootsByLength(stats) {
const statKeys = Object.keys(stats);
const arrayToSort = [];
statKeys.forEach(elem => {
arrayToSort.push(
stats[elem],
);
});
}
});
stats = {
recievers: this.sortTootsByLength(stats.recievers),
hashtags : this.sortTootsByLength(stats.hashtags),
};
return stats;
}
sortTootsByLength(stats) {
const statKeys = Object.keys(stats);
const arrayToSort = [];
statKeys.forEach(elem => {
arrayToSort.push(
stats[elem],
);
});
arrayToSort.sort((a, b) => {
return b.counter - a.counter;
});
return arrayToSort;
}
urlToUser(url) {
let sliceOfSlashes = url.split('/');
let userObject = {
url : url,
username: sliceOfSlashes[sliceOfSlashes.length - 1],
};
return userObject;
}
filterOnlyTootsWithMedias(tootList) {
console.log('filterOnlyTootsWithMedias');
return tootList.filter(toot => {
return toot['object'].attachment && toot['object'].attachment.length;
});
}
/**
* find user avatar url from its user homepage url.
* API is:
* https://mastodon.social/.well-known/webfinger?resource=acct:gargron@mastodon.social
* @param attributeToUrl
*/
fetchUserInfo(attributeToUrl = 'https://mastodon.cipherbliss.com/users/tykayn') {
// do not fetch "followers" and "activitystreams#Public"
if (
attributeToUrl.indexOf('followers') !== -1 ||
attributeToUrl.indexOf('activitystreams#Public') !== -1
) {
return {};
arrayToSort.sort((a, b) => {
return b.counter - a.counter;
});
return arrayToSort;
}
// do not fetch too much
if (this.max_fetchUsers <= this.fetched_times) {
return {};
urlToUser(url) {
let sliceOfSlashes = url.split('/');
let userObject = {
url: url,
username: sliceOfSlashes[sliceOfSlashes.length - 1],
};
return userObject;
}
this.fetched_times++;
let instanceHandle = this.findInstanceFromAttributedToUrl(attributeToUrl);
let splitted = attributeToUrl.split('/');
let accountHandle = splitted[splitted.length - 1];
// memo things
let memoEntry = this.usersMemo[accountHandle + '@' + instanceHandle];
if (memoEntry) {
return memoEntry;
filterOnlyTootsWithMedias(tootList) {
console.log('filterOnlyTootsWithMedias from', tootList.length);
let filtered = tootList.filter(toot => {
return toot['object'].attachment && toot['object'].attachment.length;
});
console.log('to ', filtered.length);
return filtered
}
// webfinger version
// let fetchUri = `https://${instanceHandle}/.well-known/webfinger?resource=acct:${accountHandle}@${instanceHandle}`;
// fetchUri = `https://${instanceHandle}/api/v1/account/${accountHandle}@${instanceHandle}`
// fetchUri = 'https://mastodon.cipherbliss.com/users/tykayn';
let headers = {
method : 'GET',
url : attributeToUrl,
headers: {
accept: 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
},
};
const self = this;
const ax = axios.create(headers);
ax.get().then(function (response) {
// handle success
// avatar is response.data.icon.url
// cover is response.data.image.url
// console.log(response.data.icon);
// console.log(response.data.image);
if (!self.usersMemo[accountHandle + '@' + instanceHandle]) {
self.usersMemo[accountHandle + '@' + instanceHandle] = {};
}
self.usersMemo[accountHandle + '@' + instanceHandle] = response.data;
return response.data;
});
/**
* find user avatar url from its user homepage url.
* API is:
* https://mastodon.social/.well-known/webfinger?resource=acct:gargron@mastodon.social
* @param attributeToUrl
*/
fetchUserInfo(attributeToUrl = 'https://mastodon.cipherbliss.com/users/tykayn') {
}
// do not fetch "followers" and "activitystreams#Public"
if (
attributeToUrl.indexOf('followers') !== -1 ||
attributeToUrl.indexOf('activitystreams#Public') !== -1
) {
return {};
}
// do not fetch too much
if (this.max_fetchUsers <= this.fetched_times) {
return {};
}
this.fetched_times++;
let instanceHandle = this.findInstanceFromAttributedToUrl(attributeToUrl);
let splitted = attributeToUrl.split('/');
let accountHandle = splitted[splitted.length - 1];
// memo things
let memoEntry = this.usersMemo[accountHandle + '@' + instanceHandle];
if (memoEntry) {
return memoEntry;
}
// webfinger version
// let fetchUri = `https://${instanceHandle}/.well-known/webfinger?resource=acct:${accountHandle}@${instanceHandle}`;
// fetchUri = `https://${instanceHandle}/api/v1/account/${accountHandle}@${instanceHandle}`
// fetchUri = 'https://mastodon.cipherbliss.com/users/tykayn';
let headers = {
method: 'GET',
url: attributeToUrl,
headers: {
accept: 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
},
};
const self = this;
const ax = axios.create(headers);
// ax.get().then(function (response) {
// // handle success
// // avatar is response.data.icon.url
// // cover is response.data.image.url
// // console.log(response.data.icon);
// // console.log(response.data.image);
//
// if (!self.usersMemo[accountHandle + '@' + instanceHandle]) {
// self.usersMemo[accountHandle + '@' + instanceHandle] = {};
// }
// self.usersMemo[accountHandle + '@' + instanceHandle] = response.data;
// return response.data;
// });
}
}

22
main.js
View File

@ -13,16 +13,16 @@ var jsonParsedLikes, jsonParsedOutbox;
* export configuration.
* You can filter the export in the following vars
*/
const min_length = 500; // minmum character length of toots to display
const max_length = 100000; // minmum character length of toots to display
const max_toots = 20// maximum length
const min_length = 300; // minmum character length of toots to display
const max_length = 100000; // max character length of toots to display
const max_toots = 200// maximum count of output
const filterBiggerTootsBeforeSlicing = false; // filter only long toots
const filterOnlyHavingMedias = true; // filter only toots having medias
const filterOnlyHavingMedias = false; // filter only toots having medias
const displayMedias = false; // display medias in toots
const showMostRecentTootsOnTop = true; // sorting order
// output write
const writeStatsJson = false; // write json export file about statistics
const writeHtml = true; // write json export file about statistics
const writeHtml = false; // write json export file about statistics
const TemplateVars = {
pageTitle : 'Mastodon export converter to HTML',
@ -57,14 +57,10 @@ fs.readFile('source_data/outbox.json',
jsonParsedOutbox = JSON.parse(data);
toots = jsonParsedOutbox.orderedItems;
// access elements
console.log('outbox toots length', toots.length);
console.log('-------- outbox toots length', toots.length);
TemplateVars.outboxTotalLength = toots.length;
toots = jsonParsedOutbox.orderedItems;
if (filterOnlyHavingMedias) {
toots = masto_converter.conversion.filterOnlyTootsWithMedias(toots);
console.log('toots.length only attachements', toots.length);
}
toots = masto_converter.conversion.filterToots(toots, TemplateVars);
console.log('min_chars', min_length);
@ -81,7 +77,7 @@ fs.readFile('source_data/outbox.json',
});
const errfileHandler = (err) => {
if (err)
console.log(err);
console.log('ERROR =======================' ,err);
else {
console.log('File statistics written successfully\n');
}
@ -96,12 +92,10 @@ app.get('/', (req, res) => {
if(writeHtml){
fs.writeFile('output/my_toots.html', html, errfileHandler);
}
res.render('index.pug', TemplateVars);
masto_converter.conversion.fetchUserInfo();
});
masto_converter.conversion.fetchUserInfo();

View File

@ -1,6 +1,6 @@
{
"scripts": {
"start": "node main.js"
"start": "nodemon main.js"
},
"dependencies": {
"axios": "^0.19.2",

View File

@ -13,6 +13,7 @@ a {
padding: 0.5em;
border-radius: 0.25em;
border-bottom: 1px solid #42495c;
color: white;
}
.published {