fix some stuff
This commit is contained in:
tykayn
parent
debae70091
commit
afef2388dc
458
conversion.js
458
conversion.js
@ -4,248 +4,256 @@ const axios = require('axios');
|
||||
|
||||
|
||||
class Conversion {
|
||||
constructor( ) {
|
||||
this.fetched_times = 0;
|
||||
this.max_fetchUsers = 5;
|
||||
this.usersMemo = {};
|
||||
}
|
||||
|
||||
hello() {
|
||||
console.log('hello from conversion');
|
||||
}
|
||||
|
||||
likes() {
|
||||
// read file likes
|
||||
fs.readFile('source_data/likes.json',
|
||||
// callback function that is called when reading file is done
|
||||
function (err, data) {
|
||||
// parse json
|
||||
let jsonParsedLikes = JSON.parse(data);
|
||||
// access elements
|
||||
const lengthOfLikes = jsonParsedLikes.orderedItems.length;
|
||||
console.log('likes length', lengthOfLikes);
|
||||
return jsonParsedLikes;
|
||||
});
|
||||
}
|
||||
|
||||
filterToots(toots, options) {
|
||||
let filteredToots = toots;
|
||||
let filter = true;
|
||||
console.log('before slicing filteredToots.length' , filteredToots.length)
|
||||
// console.log('filteredToots[0]', filteredToots[0])
|
||||
|
||||
if(filter){
|
||||
|
||||
if (options.showMostRecentTootsOnTop) {
|
||||
filteredToots = filteredToots.reverse();
|
||||
}
|
||||
if (options.filterBiggerTootsBeforeSlicing) {
|
||||
filteredToots = filteredToots.filter(item => {
|
||||
return (item['object'].content && item['object'].content.length > options.min_length && item['object'].content.length < options.max_length);
|
||||
});
|
||||
|
||||
filteredToots = filteredToots.slice(0, options.max_toots);
|
||||
} else {
|
||||
const slice = toots.slice(0, options.max_toots);
|
||||
filteredToots = slice.filter(item => {
|
||||
return (item['object'].content && item['object'].content.length > options.min_length && item['object'].content.length < options.max_length);
|
||||
});
|
||||
}
|
||||
// filteredToots.forEach(toot => {
|
||||
// toot = this.findMediaUrl(toot);
|
||||
// toot = this.removeLastChars(toot);
|
||||
// return toot;
|
||||
// });
|
||||
console.log('after slicing filteredToots.length' , filteredToots.length)
|
||||
|
||||
|
||||
constructor() {
|
||||
this.fetched_times = 0;
|
||||
this.max_fetchUsers = 5;
|
||||
this.usersMemo = {};
|
||||
}
|
||||
|
||||
return filteredToots;
|
||||
}
|
||||
|
||||
//
|
||||
/**
|
||||
*
|
||||
* @param toot
|
||||
* @returns {*}
|
||||
*/
|
||||
findMediaUrl(toot) {
|
||||
/**
|
||||
* goal:
|
||||
* https://mastodon.cipherbliss.com/system/media_attachments/files/000/858/113/original/74b370672892f884.jpg?1566230144
|
||||
*
|
||||
* input data:
|
||||
* "attributedTo":"https://mastodon.cipherbliss.com/users/tykayn",
|
||||
*
|
||||
* "attachment":[{"type":"Document",
|
||||
* "mediaType":"image/png","url":"media_attachments/files/000/872/910/original/c82b422f302b8ec9.png",
|
||||
* "name":null,
|
||||
* "blurhash":"UnSOjgo~ysWAVYWBkWjXu5axVrjckqoze?Rk"}],
|
||||
*
|
||||
* we use the attribuedTo property to find instance, and map the url to the instance url, and add this property to the attachment
|
||||
*/
|
||||
if (toot['object'].attributedTo) {
|
||||
|
||||
let splitted = toot['object'].attributedTo.split('/');
|
||||
let instanceUrl = splitted[2];
|
||||
|
||||
toot.instanceUrl = 'https://' + instanceUrl;
|
||||
toot.attachment = toot['object'].attachment.map(att => {
|
||||
att.href = toot.instanceUrl + '/system/' + att.url;
|
||||
return att;
|
||||
});
|
||||
hello() {
|
||||
console.log('hello from conversion');
|
||||
}
|
||||
return toot;
|
||||
}
|
||||
|
||||
findInstanceFromAttributedToUrl(attributedTo) {
|
||||
let splitted = attributedTo.split('/');
|
||||
return splitted[2];
|
||||
}
|
||||
likes() {
|
||||
// read file likes
|
||||
fs.readFile('source_data/likes.json',
|
||||
// callback function that is called when reading file is done
|
||||
function (err, data) {
|
||||
// parse json
|
||||
let jsonParsedLikes = JSON.parse(data);
|
||||
// access elements
|
||||
const lengthOfLikes = jsonParsedLikes.orderedItems.length;
|
||||
console.log('likes length', lengthOfLikes);
|
||||
return jsonParsedLikes;
|
||||
});
|
||||
return {}
|
||||
}
|
||||
|
||||
removeLastChars(toot) {
|
||||
toot['object'].content = toot['object'].content.trim();
|
||||
return toot;
|
||||
}
|
||||
filterToots(toots, options) {
|
||||
let filteredToots = toots;
|
||||
let filter = true;
|
||||
console.log('before slicing filteredToots.length', filteredToots.length)
|
||||
// console.log('filteredToots[0]', filteredToots[0])
|
||||
|
||||
makeStatsForToots(tootArray) {
|
||||
let stats = {
|
||||
recievers: {},
|
||||
hashtags : {},
|
||||
};
|
||||
// make statistics on who do we talk to, based on the cc field
|
||||
tootArray.forEach(elem => {
|
||||
if (filter) {
|
||||
|
||||
// stats on hashtags
|
||||
if (elem['object'].tag) {
|
||||
elem['object'].tag.forEach(tag => {
|
||||
if (tag.type === 'Hashtag') {
|
||||
if (!stats.hashtags[tag.name]) {
|
||||
stats.hashtags[tag.name] = {
|
||||
name : tag.name,
|
||||
href : tag.href,
|
||||
counter: 0,
|
||||
};
|
||||
if (options.filterOnlyHavingMedias) {
|
||||
toots = masto_converter.conversion.filterOnlyTootsWithMedias(toots);
|
||||
console.log('toots.length only attachements', toots.length);
|
||||
}
|
||||
if (options.showMostRecentTootsOnTop) {
|
||||
filteredToots = filteredToots.reverse();
|
||||
}
|
||||
if (options.filterBiggerTootsBeforeSlicing) {
|
||||
console.log('min length of content of toots ', options.min_length );
|
||||
filteredToots = filteredToots.filter(item => {
|
||||
return (item['object'].content && item['object'].content.length >= options.min_length && item['object'].content.length <= options.max_length);
|
||||
});
|
||||
|
||||
filteredToots = filteredToots.slice(0, options.max_toots);
|
||||
} else {
|
||||
const slice = toots.slice(0, options.max_toots);
|
||||
filteredToots = slice.filter(item => {
|
||||
return (item['object'].content && item['object'].content.length >= options.min_length && item['object'].content.length <= options.max_length);
|
||||
});
|
||||
}
|
||||
// filteredToots.forEach(toot => {
|
||||
// toot = this.findMediaUrl(toot);
|
||||
// toot = this.removeLastChars(toot);
|
||||
// return toot;
|
||||
// });
|
||||
console.log('after slicing filteredToots.length', filteredToots.length)
|
||||
|
||||
|
||||
}
|
||||
|
||||
return filteredToots;
|
||||
}
|
||||
|
||||
//
|
||||
/**
|
||||
*
|
||||
* @param toot
|
||||
* @returns {*}
|
||||
*/
|
||||
findMediaUrl(toot) {
|
||||
/**
|
||||
* goal:
|
||||
* https://mastodon.cipherbliss.com/system/media_attachments/files/000/858/113/original/74b370672892f884.jpg?1566230144
|
||||
*
|
||||
* input data:
|
||||
* "attributedTo":"https://mastodon.cipherbliss.com/users/tykayn",
|
||||
*
|
||||
* "attachment":[{"type":"Document",
|
||||
* "mediaType":"image/png","url":"media_attachments/files/000/872/910/original/c82b422f302b8ec9.png",
|
||||
* "name":null,
|
||||
* "blurhash":"UnSOjgo~ysWAVYWBkWjXu5axVrjckqoze?Rk"}],
|
||||
*
|
||||
* we use the attribuedTo property to find instance, and map the url to the instance url, and add this property to the attachment
|
||||
*/
|
||||
if (toot['object'].attributedTo) {
|
||||
|
||||
let splitted = toot['object'].attributedTo.split('/');
|
||||
let instanceUrl = splitted[2];
|
||||
|
||||
toot.instanceUrl = 'https://' + instanceUrl;
|
||||
toot.attachment = toot['object'].attachment.map(att => {
|
||||
att.href = toot.instanceUrl + '/system/' + att.url;
|
||||
return att;
|
||||
});
|
||||
}
|
||||
return toot;
|
||||
}
|
||||
|
||||
findInstanceFromAttributedToUrl(attributedTo) {
|
||||
let splitted = attributedTo.split('/');
|
||||
return splitted[2];
|
||||
}
|
||||
|
||||
removeLastChars(toot) {
|
||||
toot['object'].content = toot['object'].content.trim();
|
||||
return toot;
|
||||
}
|
||||
|
||||
makeStatsForToots(tootArray) {
|
||||
let stats = {
|
||||
recievers: {},
|
||||
hashtags: {},
|
||||
};
|
||||
// make statistics on who do we talk to, based on the cc field
|
||||
tootArray.forEach(elem => {
|
||||
|
||||
// stats on hashtags
|
||||
if (elem['object'].tag) {
|
||||
elem['object'].tag.forEach(tag => {
|
||||
if (tag.type === 'Hashtag') {
|
||||
if (!stats.hashtags[tag.name]) {
|
||||
stats.hashtags[tag.name] = {
|
||||
name: tag.name,
|
||||
href: tag.href,
|
||||
counter: 0,
|
||||
};
|
||||
}
|
||||
stats.hashtags[tag.name].counter++;
|
||||
}
|
||||
});
|
||||
}
|
||||
// stats on recievers of toots
|
||||
if (elem['object'].cc) {
|
||||
elem['object'].cc.forEach(urlOfUser => {
|
||||
if (!stats.recievers[urlOfUser]) {
|
||||
stats.recievers[urlOfUser] = {
|
||||
user: this.urlToUser(urlOfUser),
|
||||
name: urlOfUser,
|
||||
infos: this.fetchUserInfo(urlOfUser),
|
||||
counter: 0,
|
||||
counterContentLength: 0,
|
||||
};
|
||||
}
|
||||
stats.recievers[urlOfUser].counter++;
|
||||
stats.recievers[urlOfUser].counterContentLength += elem['object'].content.length;
|
||||
});
|
||||
}
|
||||
stats.hashtags[tag.name].counter++;
|
||||
}
|
||||
});
|
||||
}
|
||||
// stats on recievers of toots
|
||||
if (elem['object'].cc) {
|
||||
elem['object'].cc.forEach(urlOfUser => {
|
||||
if (!stats.recievers[urlOfUser]) {
|
||||
stats.recievers[urlOfUser] = {
|
||||
user : this.urlToUser(urlOfUser),
|
||||
name : urlOfUser,
|
||||
infos : this.fetchUserInfo(urlOfUser),
|
||||
counter : 0,
|
||||
counterContentLength: 0,
|
||||
};
|
||||
}
|
||||
stats.recievers[urlOfUser].counter++;
|
||||
stats.recievers[urlOfUser].counterContentLength += elem['object'].content.length;
|
||||
|
||||
stats = {
|
||||
recievers: this.sortTootsByLength(stats.recievers),
|
||||
hashtags: this.sortTootsByLength(stats.hashtags),
|
||||
};
|
||||
return stats;
|
||||
}
|
||||
|
||||
sortTootsByLength(stats) {
|
||||
const statKeys = Object.keys(stats);
|
||||
const arrayToSort = [];
|
||||
statKeys.forEach(elem => {
|
||||
arrayToSort.push(
|
||||
stats[elem],
|
||||
);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
stats = {
|
||||
recievers: this.sortTootsByLength(stats.recievers),
|
||||
hashtags : this.sortTootsByLength(stats.hashtags),
|
||||
};
|
||||
return stats;
|
||||
}
|
||||
|
||||
sortTootsByLength(stats) {
|
||||
const statKeys = Object.keys(stats);
|
||||
const arrayToSort = [];
|
||||
statKeys.forEach(elem => {
|
||||
arrayToSort.push(
|
||||
stats[elem],
|
||||
);
|
||||
});
|
||||
arrayToSort.sort((a, b) => {
|
||||
return b.counter - a.counter;
|
||||
});
|
||||
return arrayToSort;
|
||||
}
|
||||
|
||||
urlToUser(url) {
|
||||
let sliceOfSlashes = url.split('/');
|
||||
let userObject = {
|
||||
url : url,
|
||||
username: sliceOfSlashes[sliceOfSlashes.length - 1],
|
||||
};
|
||||
return userObject;
|
||||
}
|
||||
|
||||
filterOnlyTootsWithMedias(tootList) {
|
||||
console.log('filterOnlyTootsWithMedias');
|
||||
return tootList.filter(toot => {
|
||||
return toot['object'].attachment && toot['object'].attachment.length;
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* find user avatar url from its user homepage url.
|
||||
* API is:
|
||||
* https://mastodon.social/.well-known/webfinger?resource=acct:gargron@mastodon.social
|
||||
* @param attributeToUrl
|
||||
*/
|
||||
fetchUserInfo(attributeToUrl = 'https://mastodon.cipherbliss.com/users/tykayn') {
|
||||
|
||||
// do not fetch "followers" and "activitystreams#Public"
|
||||
if (
|
||||
attributeToUrl.indexOf('followers') !== -1 ||
|
||||
attributeToUrl.indexOf('activitystreams#Public') !== -1
|
||||
) {
|
||||
return {};
|
||||
arrayToSort.sort((a, b) => {
|
||||
return b.counter - a.counter;
|
||||
});
|
||||
return arrayToSort;
|
||||
}
|
||||
// do not fetch too much
|
||||
if (this.max_fetchUsers <= this.fetched_times) {
|
||||
return {};
|
||||
|
||||
urlToUser(url) {
|
||||
let sliceOfSlashes = url.split('/');
|
||||
let userObject = {
|
||||
url: url,
|
||||
username: sliceOfSlashes[sliceOfSlashes.length - 1],
|
||||
};
|
||||
return userObject;
|
||||
}
|
||||
this.fetched_times++;
|
||||
|
||||
let instanceHandle = this.findInstanceFromAttributedToUrl(attributeToUrl);
|
||||
let splitted = attributeToUrl.split('/');
|
||||
let accountHandle = splitted[splitted.length - 1];
|
||||
// memo things
|
||||
let memoEntry = this.usersMemo[accountHandle + '@' + instanceHandle];
|
||||
if (memoEntry) {
|
||||
return memoEntry;
|
||||
filterOnlyTootsWithMedias(tootList) {
|
||||
console.log('filterOnlyTootsWithMedias from', tootList.length);
|
||||
let filtered = tootList.filter(toot => {
|
||||
return toot['object'].attachment && toot['object'].attachment.length;
|
||||
});
|
||||
console.log('to ', filtered.length);
|
||||
return filtered
|
||||
}
|
||||
// webfinger version
|
||||
// let fetchUri = `https://${instanceHandle}/.well-known/webfinger?resource=acct:${accountHandle}@${instanceHandle}`;
|
||||
// fetchUri = `https://${instanceHandle}/api/v1/account/${accountHandle}@${instanceHandle}`
|
||||
// fetchUri = 'https://mastodon.cipherbliss.com/users/tykayn';
|
||||
let headers = {
|
||||
method : 'GET',
|
||||
url : attributeToUrl,
|
||||
headers: {
|
||||
accept: 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
|
||||
},
|
||||
};
|
||||
|
||||
const self = this;
|
||||
const ax = axios.create(headers);
|
||||
ax.get().then(function (response) {
|
||||
// handle success
|
||||
// avatar is response.data.icon.url
|
||||
// cover is response.data.image.url
|
||||
// console.log(response.data.icon);
|
||||
// console.log(response.data.image);
|
||||
|
||||
if (!self.usersMemo[accountHandle + '@' + instanceHandle]) {
|
||||
self.usersMemo[accountHandle + '@' + instanceHandle] = {};
|
||||
}
|
||||
self.usersMemo[accountHandle + '@' + instanceHandle] = response.data;
|
||||
return response.data;
|
||||
});
|
||||
/**
|
||||
* find user avatar url from its user homepage url.
|
||||
* API is:
|
||||
* https://mastodon.social/.well-known/webfinger?resource=acct:gargron@mastodon.social
|
||||
* @param attributeToUrl
|
||||
*/
|
||||
fetchUserInfo(attributeToUrl = 'https://mastodon.cipherbliss.com/users/tykayn') {
|
||||
|
||||
}
|
||||
// do not fetch "followers" and "activitystreams#Public"
|
||||
if (
|
||||
attributeToUrl.indexOf('followers') !== -1 ||
|
||||
attributeToUrl.indexOf('activitystreams#Public') !== -1
|
||||
) {
|
||||
return {};
|
||||
}
|
||||
// do not fetch too much
|
||||
if (this.max_fetchUsers <= this.fetched_times) {
|
||||
return {};
|
||||
}
|
||||
this.fetched_times++;
|
||||
|
||||
let instanceHandle = this.findInstanceFromAttributedToUrl(attributeToUrl);
|
||||
let splitted = attributeToUrl.split('/');
|
||||
let accountHandle = splitted[splitted.length - 1];
|
||||
// memo things
|
||||
let memoEntry = this.usersMemo[accountHandle + '@' + instanceHandle];
|
||||
if (memoEntry) {
|
||||
return memoEntry;
|
||||
}
|
||||
// webfinger version
|
||||
// let fetchUri = `https://${instanceHandle}/.well-known/webfinger?resource=acct:${accountHandle}@${instanceHandle}`;
|
||||
// fetchUri = `https://${instanceHandle}/api/v1/account/${accountHandle}@${instanceHandle}`
|
||||
// fetchUri = 'https://mastodon.cipherbliss.com/users/tykayn';
|
||||
let headers = {
|
||||
method: 'GET',
|
||||
url: attributeToUrl,
|
||||
headers: {
|
||||
accept: 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
|
||||
},
|
||||
};
|
||||
|
||||
const self = this;
|
||||
const ax = axios.create(headers);
|
||||
// ax.get().then(function (response) {
|
||||
// // handle success
|
||||
// // avatar is response.data.icon.url
|
||||
// // cover is response.data.image.url
|
||||
// // console.log(response.data.icon);
|
||||
// // console.log(response.data.image);
|
||||
//
|
||||
// if (!self.usersMemo[accountHandle + '@' + instanceHandle]) {
|
||||
// self.usersMemo[accountHandle + '@' + instanceHandle] = {};
|
||||
// }
|
||||
// self.usersMemo[accountHandle + '@' + instanceHandle] = response.data;
|
||||
// return response.data;
|
||||
// });
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
22
main.js
22
main.js
@ -13,16 +13,16 @@ var jsonParsedLikes, jsonParsedOutbox;
|
||||
* export configuration.
|
||||
* You can filter the export in the following vars
|
||||
*/
|
||||
const min_length = 500; // minmum character length of toots to display
|
||||
const max_length = 100000; // minmum character length of toots to display
|
||||
const max_toots = 20// maximum length
|
||||
const min_length = 300; // minmum character length of toots to display
|
||||
const max_length = 100000; // max character length of toots to display
|
||||
const max_toots = 200// maximum count of output
|
||||
const filterBiggerTootsBeforeSlicing = false; // filter only long toots
|
||||
const filterOnlyHavingMedias = true; // filter only toots having medias
|
||||
const filterOnlyHavingMedias = false; // filter only toots having medias
|
||||
const displayMedias = false; // display medias in toots
|
||||
const showMostRecentTootsOnTop = true; // sorting order
|
||||
// output write
|
||||
const writeStatsJson = false; // write json export file about statistics
|
||||
const writeHtml = true; // write json export file about statistics
|
||||
const writeHtml = false; // write json export file about statistics
|
||||
|
||||
const TemplateVars = {
|
||||
pageTitle : 'Mastodon export converter to HTML',
|
||||
@ -57,14 +57,10 @@ fs.readFile('source_data/outbox.json',
|
||||
jsonParsedOutbox = JSON.parse(data);
|
||||
toots = jsonParsedOutbox.orderedItems;
|
||||
// access elements
|
||||
console.log('outbox toots length', toots.length);
|
||||
console.log('-------- outbox toots length', toots.length);
|
||||
|
||||
TemplateVars.outboxTotalLength = toots.length;
|
||||
toots = jsonParsedOutbox.orderedItems;
|
||||
if (filterOnlyHavingMedias) {
|
||||
toots = masto_converter.conversion.filterOnlyTootsWithMedias(toots);
|
||||
console.log('toots.length only attachements', toots.length);
|
||||
}
|
||||
|
||||
toots = masto_converter.conversion.filterToots(toots, TemplateVars);
|
||||
|
||||
console.log('min_chars', min_length);
|
||||
@ -81,7 +77,7 @@ fs.readFile('source_data/outbox.json',
|
||||
});
|
||||
const errfileHandler = (err) => {
|
||||
if (err)
|
||||
console.log(err);
|
||||
console.log('ERROR =======================' ,err);
|
||||
else {
|
||||
console.log('File statistics written successfully\n');
|
||||
}
|
||||
@ -96,12 +92,10 @@ app.get('/', (req, res) => {
|
||||
|
||||
if(writeHtml){
|
||||
fs.writeFile('output/my_toots.html', html, errfileHandler);
|
||||
|
||||
}
|
||||
|
||||
res.render('index.pug', TemplateVars);
|
||||
|
||||
masto_converter.conversion.fetchUserInfo();
|
||||
});
|
||||
masto_converter.conversion.fetchUserInfo();
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
{
|
||||
"scripts": {
|
||||
"start": "node main.js"
|
||||
"start": "nodemon main.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"axios": "^0.19.2",
|
||||
|
||||
@ -13,6 +13,7 @@ a {
|
||||
padding: 0.5em;
|
||||
border-radius: 0.25em;
|
||||
border-bottom: 1px solid #42495c;
|
||||
color: white;
|
||||
}
|
||||
|
||||
.published {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user