framalibre-scraping/main.js

67 lines
1.6 KiB
JavaScript
Raw Normal View History

// parse framalibre catalog and merge json in output
// API URL example 'https://framalibre.org/content/1075/json'
const fs = require('fs');
const axios = require('axios');
const min_id = 0;
// const max_id = 1417; // 1417 = fiche de "i hate money"
const max_id = 1417; // 1417 = fiche de "i hate money"
const mergedCatalog = [];
class Methodos {
fetchCatalog() {
const self = this;
for (var ii = min_id; ii <= max_id; ii++) {
console.log('ID', ii, ')')
console.log('call api with id ', ii)
axios.get(`https://framalibre.org/content/${ii}/json`)
.then((resp) => {
for (let index in resp.data.nodes) {
let toAdd = resp.data.nodes[index].node
console.log('fiche', toAdd.Titre)
this.addtocatalog(toAdd)
}
})
.then(resp => {
this.writeOutPut(mergedCatalog);
})
.catch(err => console.error(err))
}
}
writeOutPut(mergedCatalog) {
console.log('mergedCatalog.length', mergedCatalog.length)
let m = mergedCatalog.map(elem => JSON.stringify(elem));
let outJson = {}
for (let index in mergedCatalog) {
outJson[index * 1 + 1] = mergedCatalog[index];
}
let catalogPath = `./output/catalog_from_${min_id}_to_${max_id}.json`;
let stringifiedCatalog = JSON.stringify(outJson)
console.log('stringifiedCatalog', stringifiedCatalog)
fs.writeFile(catalogPath, stringifiedCatalog, (err) => {
if (err) throw err;
console.log('catalog.js Replaced!');
})
return mergedCatalog;
}
addtocatalog(index) {
mergedCatalog.push(index)
}
}
let m = new Methodos();
m.fetchCatalog();