tykayn/framalibre-scraping
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

unique soft chaton ref

Browse Source
This commit is contained in:
Tykayn 2022-07-11 13:22:28 +02:00 committed by caligulanorris
parent 7de5c71ab7
commit 9127b4c7bf
2 changed files with 2032 additions and 5000 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
chatons.js
View File

@ -100,18 +100,17 @@ async function scrapeDataCatalogcontent(url_page, shouldWeSave) {
let chaton_name = $('h2.chatons-public-subtitle').eq(0).text().trim(); let chaton_name = $('h2.chatons-public-subtitle').eq(0).text().trim();
// stats départements // stats départements
let dep = $('.field--name-field-zip-code .field__item')?.text().substring(0, 2); let dep = $('.field--name-field-zip-code .field__item')?.text().substring(0, 2);
if (dep != null && chaton_name && chaton_name.length) { if (dep != null && chaton_name && chaton_name.length) {
if (!departements[dep]) { if (!departements[dep]) {
console.log('ajout département ',dep) console.log('ajout département ', dep);
departements[dep] = []; departements[dep] = [];
} }
console.log('zip', dep, chaton_name); console.log('zip', dep, chaton_name);
departements[dep].push(chaton_name); departements[dep].push(chaton_name);
console.log('departements', departements) console.log('departements', departements);
} }
// propriétés de la page chaton // propriétés de la page chaton
@ -127,15 +126,19 @@ async function scrapeDataCatalogcontent(url_page, shouldWeSave) {
softwares.push(soft); softwares.push(soft);
if (soft_name) { if (soft_name) {
console.log('soft_name', soft_name) console.log('soft_name', soft_name);
if (!softwaresGeneral[soft_name]) { if (!softwaresGeneral[soft_name]) {
softwaresGeneral[soft_name] = []; softwaresGeneral[soft_name] = [];
} }
softwaresGeneral[soft_name].push({ let softChaton = {
name: chaton_name, name: chaton_name,
url : $('.field--name-field-website-url .field__item')?.text().trim(), url : $('.field--name-field-website-url .field__item')?.text().trim(),
}); };
if (softwaresGeneral[soft_name].indexOf(softChaton) !== -1) {
softwaresGeneral[soft_name].push(softChaton);
}
} }
}); });
pagesChatons.push({ pagesChatons.push({
@ -153,7 +156,6 @@ async function scrapeDataCatalogcontent(url_page, shouldWeSave) {
softwares, softwares,
}); });
if (shouldWeSave) { if (shouldWeSave) {
setTimeout(persistCatalog, 2000); setTimeout(persistCatalog, 2000);

7016
output/chatons_links.json
View File

File diff suppressed because it is too large Load Diff