tykayn/framalibre-scraping
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

unique soft chaton ref

Browse Source
This commit is contained in:
Tykayn 2022-07-11 13:22:28 +02:00 committed by caligulanorris
parent 7de5c71ab7
commit 9127b4c7bf
2 changed files with 2032 additions and 5000 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
chatons.js
View File

@ -100,18 +100,17 @@ async function scrapeDataCatalogcontent(url_page, shouldWeSave) {
let chaton_name = $('h2.chatons-public-subtitle').eq(0).text().trim();
// stats départements
let dep = $('.field--name-field-zip-code .field__item')?.text().substring(0, 2);
if (dep != null && chaton_name && chaton_name.length) {
if (!departements[dep]) {
console.log('ajout département ',dep)
console.log('ajout département ', dep);
departements[dep] = [];
}
console.log('zip', dep, chaton_name);
departements[dep].push(chaton_name);
console.log('departements', departements)
console.log('departements', departements);
}
// propriétés de la page chaton
@ -127,15 +126,19 @@ async function scrapeDataCatalogcontent(url_page, shouldWeSave) {
softwares.push(soft);
if (soft_name) {
console.log('soft_name', soft_name)
console.log('soft_name', soft_name);
if (!softwaresGeneral[soft_name]) {
softwaresGeneral[soft_name] = [];
}
softwaresGeneral[soft_name].push({
let softChaton = {
name: chaton_name,
url : $('.field--name-field-website-url .field__item')?.text().trim(),
});
};
if (softwaresGeneral[soft_name].indexOf(softChaton) !== -1) {
softwaresGeneral[soft_name].push(softChaton);
}
}
});
pagesChatons.push({
@ -153,7 +156,6 @@ async function scrapeDataCatalogcontent(url_page, shouldWeSave) {
softwares,
});
if (shouldWeSave) {
setTimeout(persistCatalog, 2000);

7016
output/chatons_links.json
View File

File diff suppressed because it is too large Load Diff