framalibre-scraping/main.js

// parse framalibre catalog and merge json in output
// API URL example 'https://framalibre.org/content/1075/json'

const fs = require('fs');
const axios = require('axios');

const min_id = 0;
// const max_id = 1417; // 1417 = fiche de "i hate money"
const max_id = 1417; // 1417 = fiche de "i hate money"

const mergedCatalog = [];

class Methodos {
  fetchCatalog() {

    const self = this;
    for (var ii = min_id; ii <= max_id; ii++) {
      console.log('ID', ii, ')')

      console.log('call api with id ', ii)
      axios.get(`https://framalibre.org/content/${ii}/json`)
        .then((resp) => {
          for (let index in resp.data.nodes) {
            let toAdd = resp.data.nodes[index].node
            console.log('fiche', toAdd.Titre)
            this.addtocatalog(toAdd)
          }

        })
        .then(resp => {
          this.writeOutPut(mergedCatalog);
        })
        .catch(err => console.error(err))

    }

  }

  writeOutPut(mergedCatalog) {
    console.log('mergedCatalog.length', mergedCatalog.length)

    let m = mergedCatalog.map(elem => JSON.stringify(elem));
    let outJson = {}
    for (let index in mergedCatalog) {
      outJson[index * 1 + 1] = mergedCatalog[index];
    }
    let catalogPath = `./output/catalog_from_${min_id}_to_${max_id}.json`;
    let stringifiedCatalog = JSON.stringify(outJson)
    console.log('stringifiedCatalog', stringifiedCatalog)
    fs.writeFile(catalogPath, stringifiedCatalog, (err) => {
      if (err) throw err;
      console.log('catalog.js Replaced!');
    })
    return mergedCatalog;
  }

  addtocatalog(index) {
    mergedCatalog.push(index)
  }

}

let m = new Methodos();
m.fetchCatalog();