scrap one list of books
This commit is contained in:
parent
757528b952
commit
31d8fd53c8
101
scraping/main.ts
Normal file
101
scraping/main.ts
Normal file
@ -0,0 +1,101 @@
|
||||
/**
|
||||
scrapping des livres de la médiathèque de briis
|
||||
**/
|
||||
// @ts-ignore
|
||||
import https from 'https';
|
||||
import WriteFile from "./utils";
|
||||
|
||||
const axios = require('axios');
|
||||
const cheerio = require('cheerio');
|
||||
const url: string = "www.mediatheque-de-briis-sous-forges.net";
|
||||
const fetching_path: string = "/mediatheque-de-briis-sous-forges.net/opac/recherche/catalogue?node=0&value=0&page=2";
|
||||
|
||||
interface Book {
|
||||
author: string
|
||||
title: string
|
||||
description: string
|
||||
img: string
|
||||
}
|
||||
const books: Book[] = [];
|
||||
// autres pages:
|
||||
// http://www.mediatheque-de-briis-sous-forges.net/mediatheque-de-briis-sous-forges.net/opac/recherche/catalogue?node=0&value=0&page=2
|
||||
const page_max = 1927
|
||||
|
||||
const getTables = (html: string): any => {
|
||||
const $ = cheerio.load(html);
|
||||
const tableElements = $(
|
||||
"table.notice"
|
||||
);
|
||||
return tableElements;
|
||||
};
|
||||
|
||||
const getHtml = async (hostname: string, path: string): Promise<string> =>
|
||||
new Promise((resolve, reject) => {
|
||||
https
|
||||
.get(
|
||||
{
|
||||
hostname,
|
||||
path,
|
||||
// port:80,
|
||||
method: "GET",
|
||||
},
|
||||
(res) => {
|
||||
let html = "";
|
||||
res.on("data", function (chunk) {
|
||||
html += chunk;
|
||||
});
|
||||
res.on("end", function () {
|
||||
resolve(html);
|
||||
});
|
||||
}
|
||||
)
|
||||
.on("error", (error) => {
|
||||
console.error(error);
|
||||
reject(error);
|
||||
});
|
||||
});
|
||||
|
||||
// fetchData(url).then((res: any) => {
|
||||
// const html = res.data;
|
||||
// const $ = cheerio.load(html);
|
||||
// const statsTable :any = $('table.notice');
|
||||
// console.log('statsTable', statsTable)
|
||||
// statsTable.each(function(){
|
||||
// let elem:any = this;
|
||||
// let author = $(elem).find('td').eq(2).text();
|
||||
// // let img = $(this).find('img').attr('src');
|
||||
// // console.log(elem);
|
||||
// console.log(author);
|
||||
// });
|
||||
// })
|
||||
|
||||
|
||||
function writeBookScrapping() {
|
||||
WriteFile('books.json', JSON.stringify(books, null , 2))
|
||||
}
|
||||
|
||||
getHtml(url, fetching_path)
|
||||
.then(getTables)
|
||||
.then(
|
||||
(tables: any) => tables.each(
|
||||
(_: any, table: any) => {
|
||||
const $ = cheerio.load(table);
|
||||
// console.log('une table')
|
||||
// let author = $().find('a.notice').text();
|
||||
let text_description = $(table).find('td').eq(1).text();
|
||||
let boom = text_description.split('\n');
|
||||
let splitting = boom[1].split('/')
|
||||
let img_src = $(table).find('td img').attr('src');
|
||||
console.log(img_src);
|
||||
books.push({
|
||||
author: boom[0],
|
||||
title: splitting[0],
|
||||
description: splitting[1],
|
||||
img: img_src
|
||||
})
|
||||
// console.log(cheerio.load(table).html())
|
||||
}
|
||||
)
|
||||
)
|
||||
.then(writeBookScrapping)
|
||||
.catch((error) => console.log(error));
|
||||
62
scraping/output/books.json
Normal file
62
scraping/output/books.json
Normal file
@ -0,0 +1,62 @@
|
||||
[
|
||||
{
|
||||
"author": "ALLENDE, Isabel",
|
||||
"title": "Portrait sépia ",
|
||||
"description": " Isabel Allende ; Trad. de l'espagnol par Claude de Frayssinet. - Paris : Grasset et Fasquelle, 2001. - 1 vol. , 391 p. : couv. ill. ; 24 x 15 cm.",
|
||||
"img": "http://images-eu.amazon.com/images/P/2246617715.08.MZZZZZZZ.jpg"
|
||||
},
|
||||
{
|
||||
"author": "AMETTE, Jacques-Pierre",
|
||||
"title": "La Maîtresse de Brecht : roman ",
|
||||
"description": " Jacques-Pierre Amette. - Paris : Albin Michel, 2003. - 300 p. : jaquette ill. ; 20 cm.",
|
||||
"img": "http://images-eu.amazon.com/images/P/2226141634.08.MZZZZZZZ.jpg"
|
||||
},
|
||||
{
|
||||
"author": "ANDRIÂC, Ivo",
|
||||
"title": "Mara la courtisane : et autres nouvelles ",
|
||||
"description": " Ivo Andriâc ; Trad. du serbo-croate par Pascale Delpech. - Paris : Belfond, 1999. - 234 p. : couv. ill. en coul. ; 23 cm. - (Littérature étrangère).",
|
||||
"img": "http://images-eu.amazon.com/images/P/2714435572.08.MZZZZZZZ.jpg"
|
||||
},
|
||||
{
|
||||
"author": "ANGLADE, Jean",
|
||||
"title": "Un Lit d'aubépine : roman ",
|
||||
"description": " Jean Anglade. - Paris : Presses de la Cité, 1995. - 325 p. : couv. ill. en coul. ; 23 cm. - (Production Jeannine Balland).",
|
||||
"img": "http://images-eu.amazon.com/images/P/2258039568.08.MZZZZZZZ.jpg"
|
||||
},
|
||||
{
|
||||
"author": "ARNOTHY, Christine",
|
||||
"title": "J'ai quinze ans et je ne veux pas mourir ; (suivi de) Il n'est pas si facile de vivre ",
|
||||
"description": " Christine Arnothy. - Paris : France loisirs, 1981. - 330 p ; 23 cm.",
|
||||
"img": "http://images-eu.amazon.com/images/P/2724211065.08.MZZZZZZZ.jpg"
|
||||
},
|
||||
{
|
||||
"author": "CHAUVIN, Rémy",
|
||||
"title": "Le Monde animal et ses comportements complexes ",
|
||||
"description": " Rémy Chauvin, Bernadette Chauvin. - Paris : Plon, 1977. - 282 p : ill ; 21 cm.",
|
||||
"img": "http://images-eu.amazon.com/images/P/2259002331.08.MZZZZZZZ.jpg"
|
||||
},
|
||||
{
|
||||
"author": "D'ARZO, Silvio",
|
||||
"title": "Maison des autres , (Contient) Un moment comme ça ",
|
||||
"description": " texte de Silvio D'Arzo ; Trad. de l'italien par Bernard Simeone, Philippe Renard ; Préf. Attilio Bertolucci. - Lagrasse : Verdier, 1997. - 1 vol. , 86 p. : - ; 22 x 14 cm. - (Terra d'altri, ISSN 0989-4160).",
|
||||
"img": "http://images-eu.amazon.com/images/P/2864322838.08.MZZZZZZZ.jpg"
|
||||
},
|
||||
{
|
||||
"author": "ASSOULINE, Pierre",
|
||||
"title": "La Cliente : roman ",
|
||||
"description": " Pierre Assouline. - Paris : Gallimard, 1998. - 191 p. ; 21 cm.",
|
||||
"img": "http://images-eu.amazon.com/images/P/207075278X.08.MZZZZZZZ.jpg"
|
||||
},
|
||||
{
|
||||
"author": "ATKINSON, Kate",
|
||||
"title": "Dans les replis du temps ",
|
||||
"description": " texte de Kate Atkinson ; Trad. de l'anglais par Jean Bourdier. - Paris : Librairie générale française, 1999. - 1 vol. , 403 p. : ill., couv. ill. en coul. ; 18 x 11 cm. - (Le livre de poche ; 14687).",
|
||||
"img": "http://images-eu.amazon.com/images/P/2253146870.08.MZZZZZZZ.jpg"
|
||||
},
|
||||
{
|
||||
"author": "ATKINSON, Kate",
|
||||
"title": "Dans les coulisses du musée : roman ",
|
||||
"description": " Kate Atkinson ; Trad. de l'anglais par Jean Bourdier. - Paris : Bernard de Fallois, 1996. - 348 p. : couv. ill. en coul. ; 23 cm.",
|
||||
"img": "http://images-eu.amazon.com/images/P/2877062775.08.MZZZZZZZ.jpg"
|
||||
}
|
||||
]
|
||||
15
scraping/utils.ts
Normal file
15
scraping/utils.ts
Normal file
@ -0,0 +1,15 @@
|
||||
import * as fs from "node:fs";
|
||||
|
||||
export default function WriteFile(fileName: string, fileContent: any) {
|
||||
console.log('write file', fileName)
|
||||
return fs.writeFile(
|
||||
`./output/${fileName}`,
|
||||
fileContent,
|
||||
'utf8',
|
||||
(err) => {
|
||||
if (err) {
|
||||
console.log(`Error writing file: ${err}`)
|
||||
}
|
||||
}
|
||||
)
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user