From 055a8114416283957fda875d337de27e58b97fe7 Mon Sep 17 00:00:00 2001 From: Tykayn Date: Tue, 11 Jan 2022 23:09:17 +0100 Subject: [PATCH] test mongoose --- models/Event.js | 14 ++ output/.gitkeep | 0 package-lock.json | 418 +++++++++++++++++++++++++++++++++++-- package.json | 9 +- scrapers/ccpl.js | 31 --- scrapers/ccpl.ts | 82 ++++++++ sources_examples/input.csv | 1 + tsconfig.json | 17 ++ utils.ts | 3 +- yarn.lock | 167 ++++++++++++++- 10 files changed, 680 insertions(+), 62 deletions(-) create mode 100644 models/Event.js create mode 100644 output/.gitkeep delete mode 100644 scrapers/ccpl.js create mode 100644 scrapers/ccpl.ts create mode 100644 sources_examples/input.csv create mode 100644 tsconfig.json diff --git a/models/Event.js b/models/Event.js new file mode 100644 index 0000000..aa747ef --- /dev/null +++ b/models/Event.js @@ -0,0 +1,14 @@ +const mongoose = require('mongoose'); + +let EventMobilizonSchema = new mongoose.Schema({ + title: String, + date_start: Date, + date_end: Date, + description: Date, + url: Date, + dateCrawled: Date +}); + +let EventMobilizon = mongoose.model('EventMobilizon', userSchema); + +module.exports = EventMobilizon; \ No newline at end of file diff --git a/output/.gitkeep b/output/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/package-lock.json b/package-lock.json index 7fc79fc..8a6738d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,11 +9,11 @@ "version": "1.0.0", "license": "ISC", "dependencies": { - "@types/node": "^14.18.5", "colors": "^1.4.0", "escape-goat": "^3.0.0", "esm": "^3.2.25", "moment": "^2.29.1", + "mongoose": "^6.1.6", "node-fetch": "^2.6.1", "rss-parser": "^3.10.0", "ts-node": "^10.4.0", @@ -21,8 +21,11 @@ "uuid": "^8.3.2" }, "devDependencies": { + "@types/node": "^17.0.8", "@types/pg": "^8.6.3", + "@types/puppeteer": "^5.4.4", "@types/uuid": "^8.3.4", + "fs": "^0.0.1-security", "pg": "^8.7.1", "puppeteer": "^13.0.1" } @@ -67,9 +70,9 @@ "integrity": "sha512-eZxlbI8GZscaGS7kkc/trHTT5xgrjH3/1n2JDwusC9iahPKWMRvRjJSAN5mCXviuTGQ/lHnhvv8Q1YTpnfz9gA==" }, "node_modules/@types/node": { - "version": "14.18.5", - "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.5.tgz", - "integrity": "sha512-LMy+vDDcQR48EZdEx5wRX1q/sEl6NdGuHXPnfeL8ixkwCOSZ2qnIyIZmcCbdX0MeRqHhAcHmX+haCbrS8Run+A==" + "version": "17.0.8", + "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.8.tgz", + "integrity": "sha512-YofkM6fGv4gDJq78g4j0mMuGMkZVxZDgtU0JRdx6FgiJDG+0fY0GKVolOV8WqVmEhLCXkQRjwDdKyPxJp/uucg==" }, "node_modules/@types/pg": { "version": "8.6.3", @@ -82,12 +85,35 @@ "pg-types": "^2.2.0" } }, + "node_modules/@types/puppeteer": { + "version": "5.4.4", + "resolved": "https://registry.npmjs.org/@types/puppeteer/-/puppeteer-5.4.4.tgz", + "integrity": "sha512-3Nau+qi69CN55VwZb0ATtdUAlYlqOOQ3OfQfq0Hqgc4JMFXiQT/XInlwQ9g6LbicDslE6loIFsXFklGh5XmI6Q==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/uuid": { "version": "8.3.4", "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-8.3.4.tgz", "integrity": "sha512-c/I8ZRb51j+pYGAu5CrFMRxqZ2ke4y2grEBO5AUjgSkSk+qT2Ea+OdWElz/OiMf5MNpn2b17kuVBwZLQJXzihw==", "dev": true }, + "node_modules/@types/webidl-conversions": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-6.1.1.tgz", + "integrity": "sha512-XAahCdThVuCFDQLT7R7Pk/vqeObFNL3YqRyFZg+AqAP/W1/w3xHaIxuW7WszQqTbIBOPRcItYJIou3i/mppu3Q==" + }, + "node_modules/@types/whatwg-url": { + "version": "8.2.1", + "resolved": "https://registry.npmjs.org/@types/whatwg-url/-/whatwg-url-8.2.1.tgz", + "integrity": "sha512-2YubE1sjj5ifxievI5Ge1sckb9k/Er66HyR2c+3+I6VDUUg1TLPdYYTEbQ+DjRkS4nTxMJhgWfSfMRD2sl2EYQ==", + "dependencies": { + "@types/node": "*", + "@types/webidl-conversions": "*" + } + }, "node_modules/@types/yauzl": { "version": "2.9.2", "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.9.2.tgz", @@ -144,7 +170,6 @@ "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "dev": true, "funding": [ { "type": "github", @@ -181,11 +206,21 @@ "concat-map": "0.0.1" } }, + "node_modules/bson": { + "version": "4.6.1", + "resolved": "https://registry.npmjs.org/bson/-/bson-4.6.1.tgz", + "integrity": "sha512-I1LQ7Hz5zgwR4QquilLNZwbhPw0Apx7i7X9kGMBTsqPdml/03Q9NBtD9nt/19ahjlphktQImrnderxqpzeVDjw==", + "dependencies": { + "buffer": "^5.6.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, "node_modules/buffer": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", - "dev": true, "funding": [ { "type": "github", @@ -252,7 +287,6 @@ "version": "4.3.2", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==", - "dev": true, "dependencies": { "ms": "2.1.2" }, @@ -265,6 +299,14 @@ } } }, + "node_modules/denque": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.0.1.tgz", + "integrity": "sha512-tfiWc6BQLXNLpNiR5iGd0Ocu3P3VpxfzFiqubLgMfhfOw9WyvgJBd46CClNn9k3qfbjvT//0cf7AlYRX/OslMQ==", + "engines": { + "node": ">=0.10" + } + }, "node_modules/devtools-protocol": { "version": "0.0.937139", "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.937139.tgz", @@ -357,6 +399,12 @@ "node": ">=8" } }, + "node_modules/fs": { + "version": "0.0.1-security", + "resolved": "https://registry.npmjs.org/fs/-/fs-0.0.1-security.tgz", + "integrity": "sha1-invTcYa23d84E/I4WLV+yq9eQdQ=", + "dev": true + }, "node_modules/fs-constants": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", @@ -421,7 +469,6 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", - "dev": true, "funding": [ { "type": "github", @@ -453,6 +500,11 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", "dev": true }, + "node_modules/kareem": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/kareem/-/kareem-2.3.3.tgz", + "integrity": "sha512-uESCXM2KdtOQ8LOvKyTUXEeg0MkYp4wGglTIpGcYHvjJcS5sn2Wkfrfit8m4xSbaNDAw2KdI9elgkOxZbrFYbg==" + }, "node_modules/locate-path": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", @@ -470,6 +522,12 @@ "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==" }, + "node_modules/memory-pager": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz", + "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==", + "optional": true + }, "node_modules/minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", @@ -496,11 +554,116 @@ "node": "*" } }, + "node_modules/mongodb": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.2.2.tgz", + "integrity": "sha512-zt8rCTnTKyMQppyt63qMnrLM5dbADgUk18ORPF1XbtHLIYCyc9hattaYHi0pqMvNxDpgGgUofSVzS+UQErgTug==", + "dependencies": { + "bson": "^4.6.0", + "denque": "^2.0.1", + "mongodb-connection-string-url": "^2.3.2" + }, + "engines": { + "node": ">=12.9.0" + }, + "optionalDependencies": { + "saslprep": "^1.0.3" + } + }, + "node_modules/mongodb-connection-string-url": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/mongodb-connection-string-url/-/mongodb-connection-string-url-2.4.1.tgz", + "integrity": "sha512-d5Kd2bVsKcSA7YI/yo57fSTtMwRQdFkvc5IZwod1RRxJtECeWPPSo7zqcUGJELifRA//Igs4spVtYAmvFCatug==", + "dependencies": { + "@types/whatwg-url": "^8.2.1", + "whatwg-url": "^11.0.0" + } + }, + "node_modules/mongodb-connection-string-url/node_modules/tr46": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz", + "integrity": "sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==", + "dependencies": { + "punycode": "^2.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/mongodb-connection-string-url/node_modules/webidl-conversions": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", + "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", + "engines": { + "node": ">=12" + } + }, + "node_modules/mongodb-connection-string-url/node_modules/whatwg-url": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-11.0.0.tgz", + "integrity": "sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ==", + "dependencies": { + "tr46": "^3.0.0", + "webidl-conversions": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/mongoose": { + "version": "6.1.6", + "resolved": "https://registry.npmjs.org/mongoose/-/mongoose-6.1.6.tgz", + "integrity": "sha512-wvNRJ547x/Bn3EMhwbVInNsEp2OdlMxk4Q/vfgAkw8OI+giZQ72S90ZH0H6VzzIhs4lCU9SwXHYa2O0sPicnEQ==", + "dependencies": { + "@types/node": "< 17.0.6", + "bson": "^4.2.2", + "kareem": "2.3.3", + "mongodb": "4.2.2", + "mpath": "0.8.4", + "mquery": "4.0.0", + "ms": "2.1.2", + "regexp-clone": "1.0.0", + "sift": "13.5.2", + "sliced": "1.0.1" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mongoose" + } + }, + "node_modules/mongoose/node_modules/@types/node": { + "version": "17.0.5", + "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.5.tgz", + "integrity": "sha512-w3mrvNXLeDYV1GKTZorGJQivK6XLCoGwpnyJFbJVK/aTBQUxOCaa/GlFAAN3OTDFcb7h5tiFG+YXCO2By+riZw==" + }, + "node_modules/mpath": { + "version": "0.8.4", + "resolved": "https://registry.npmjs.org/mpath/-/mpath-0.8.4.tgz", + "integrity": "sha512-DTxNZomBcTWlrMW76jy1wvV37X/cNNxPW1y2Jzd4DZkAaC5ZGsm8bfGfNOthcDuRJujXLqiuS6o3Tpy0JEoh7g==", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/mquery": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/mquery/-/mquery-4.0.0.tgz", + "integrity": "sha512-nGjm89lHja+T/b8cybAby6H0YgA4qYC/lx6UlwvHGqvTq8bDaNeCwl1sY8uRELrNbVWJzIihxVd+vphGGn1vBw==", + "dependencies": { + "debug": "4.x", + "regexp-clone": "^1.0.0", + "sliced": "1.0.1" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "node_modules/node-fetch": { "version": "2.6.5", @@ -745,6 +908,14 @@ "once": "^1.3.1" } }, + "node_modules/punycode": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", + "engines": { + "node": ">=6" + } + }, "node_modules/puppeteer": { "version": "13.0.1", "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-13.0.1.tgz", @@ -783,6 +954,11 @@ "node": ">= 6" } }, + "node_modules/regexp-clone": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/regexp-clone/-/regexp-clone-1.0.0.tgz", + "integrity": "sha512-TuAasHQNamyyJ2hb97IuBEif4qBHGjPHBS64sZwytpLEqtBQ1gPJTnOaQ6qmpET16cK14kkjbazl6+p0RRv0yw==" + }, "node_modules/rimraf": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", @@ -827,11 +1003,42 @@ } ] }, + "node_modules/saslprep": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/saslprep/-/saslprep-1.0.3.tgz", + "integrity": "sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag==", + "optional": true, + "dependencies": { + "sparse-bitfield": "^3.0.3" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/sax": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" }, + "node_modules/sift": { + "version": "13.5.2", + "resolved": "https://registry.npmjs.org/sift/-/sift-13.5.2.tgz", + "integrity": "sha512-+gxdEOMA2J+AI+fVsCqeNn7Tgx3M9ZN9jdi95939l1IJ8cZsqS8sqpJyOkic2SJk+1+98Uwryt/gL6XDaV+UZA==" + }, + "node_modules/sliced": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/sliced/-/sliced-1.0.1.tgz", + "integrity": "sha1-CzpmK10Ewxd7GSa+qCsD+Dei70E=" + }, + "node_modules/sparse-bitfield": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", + "integrity": "sha1-/0rm5oZWBWuks+eSqzM004JzyhE=", + "optional": true, + "dependencies": { + "memory-pager": "^1.0.2" + } + }, "node_modules/split2": { "version": "3.2.2", "resolved": "https://registry.npmjs.org/split2/-/split2-3.2.2.tgz", @@ -1089,9 +1296,9 @@ "integrity": "sha512-eZxlbI8GZscaGS7kkc/trHTT5xgrjH3/1n2JDwusC9iahPKWMRvRjJSAN5mCXviuTGQ/lHnhvv8Q1YTpnfz9gA==" }, "@types/node": { - "version": "14.18.5", - "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.5.tgz", - "integrity": "sha512-LMy+vDDcQR48EZdEx5wRX1q/sEl6NdGuHXPnfeL8ixkwCOSZ2qnIyIZmcCbdX0MeRqHhAcHmX+haCbrS8Run+A==" + "version": "17.0.8", + "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.8.tgz", + "integrity": "sha512-YofkM6fGv4gDJq78g4j0mMuGMkZVxZDgtU0JRdx6FgiJDG+0fY0GKVolOV8WqVmEhLCXkQRjwDdKyPxJp/uucg==" }, "@types/pg": { "version": "8.6.3", @@ -1104,12 +1311,35 @@ "pg-types": "^2.2.0" } }, + "@types/puppeteer": { + "version": "5.4.4", + "resolved": "https://registry.npmjs.org/@types/puppeteer/-/puppeteer-5.4.4.tgz", + "integrity": "sha512-3Nau+qi69CN55VwZb0ATtdUAlYlqOOQ3OfQfq0Hqgc4JMFXiQT/XInlwQ9g6LbicDslE6loIFsXFklGh5XmI6Q==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/uuid": { "version": "8.3.4", "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-8.3.4.tgz", "integrity": "sha512-c/I8ZRb51j+pYGAu5CrFMRxqZ2ke4y2grEBO5AUjgSkSk+qT2Ea+OdWElz/OiMf5MNpn2b17kuVBwZLQJXzihw==", "dev": true }, + "@types/webidl-conversions": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-6.1.1.tgz", + "integrity": "sha512-XAahCdThVuCFDQLT7R7Pk/vqeObFNL3YqRyFZg+AqAP/W1/w3xHaIxuW7WszQqTbIBOPRcItYJIou3i/mppu3Q==" + }, + "@types/whatwg-url": { + "version": "8.2.1", + "resolved": "https://registry.npmjs.org/@types/whatwg-url/-/whatwg-url-8.2.1.tgz", + "integrity": "sha512-2YubE1sjj5ifxievI5Ge1sckb9k/Er66HyR2c+3+I6VDUUg1TLPdYYTEbQ+DjRkS4nTxMJhgWfSfMRD2sl2EYQ==", + "requires": { + "@types/node": "*", + "@types/webidl-conversions": "*" + } + }, "@types/yauzl": { "version": "2.9.2", "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.9.2.tgz", @@ -1153,8 +1383,7 @@ "base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "dev": true + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" }, "bl": { "version": "4.1.0", @@ -1177,11 +1406,18 @@ "concat-map": "0.0.1" } }, + "bson": { + "version": "4.6.1", + "resolved": "https://registry.npmjs.org/bson/-/bson-4.6.1.tgz", + "integrity": "sha512-I1LQ7Hz5zgwR4QquilLNZwbhPw0Apx7i7X9kGMBTsqPdml/03Q9NBtD9nt/19ahjlphktQImrnderxqpzeVDjw==", + "requires": { + "buffer": "^5.6.0" + } + }, "buffer": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", - "dev": true, "requires": { "base64-js": "^1.3.1", "ieee754": "^1.1.13" @@ -1225,11 +1461,15 @@ "version": "4.3.2", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==", - "dev": true, "requires": { "ms": "2.1.2" } }, + "denque": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.0.1.tgz", + "integrity": "sha512-tfiWc6BQLXNLpNiR5iGd0Ocu3P3VpxfzFiqubLgMfhfOw9WyvgJBd46CClNn9k3qfbjvT//0cf7AlYRX/OslMQ==" + }, "devtools-protocol": { "version": "0.0.937139", "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.937139.tgz", @@ -1296,6 +1536,12 @@ "path-exists": "^4.0.0" } }, + "fs": { + "version": "0.0.1-security", + "resolved": "https://registry.npmjs.org/fs/-/fs-0.0.1-security.tgz", + "integrity": "sha1-invTcYa23d84E/I4WLV+yq9eQdQ=", + "dev": true + }, "fs-constants": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", @@ -1344,8 +1590,7 @@ "ieee754": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", - "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", - "dev": true + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" }, "inflight": { "version": "1.0.6", @@ -1363,6 +1608,11 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", "dev": true }, + "kareem": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/kareem/-/kareem-2.3.3.tgz", + "integrity": "sha512-uESCXM2KdtOQ8LOvKyTUXEeg0MkYp4wGglTIpGcYHvjJcS5sn2Wkfrfit8m4xSbaNDAw2KdI9elgkOxZbrFYbg==" + }, "locate-path": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", @@ -1377,6 +1627,12 @@ "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==" }, + "memory-pager": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz", + "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==", + "optional": true + }, "minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", @@ -1397,11 +1653,93 @@ "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.1.tgz", "integrity": "sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==" }, + "mongodb": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.2.2.tgz", + "integrity": "sha512-zt8rCTnTKyMQppyt63qMnrLM5dbADgUk18ORPF1XbtHLIYCyc9hattaYHi0pqMvNxDpgGgUofSVzS+UQErgTug==", + "requires": { + "bson": "^4.6.0", + "denque": "^2.0.1", + "mongodb-connection-string-url": "^2.3.2", + "saslprep": "^1.0.3" + } + }, + "mongodb-connection-string-url": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/mongodb-connection-string-url/-/mongodb-connection-string-url-2.4.1.tgz", + "integrity": "sha512-d5Kd2bVsKcSA7YI/yo57fSTtMwRQdFkvc5IZwod1RRxJtECeWPPSo7zqcUGJELifRA//Igs4spVtYAmvFCatug==", + "requires": { + "@types/whatwg-url": "^8.2.1", + "whatwg-url": "^11.0.0" + }, + "dependencies": { + "tr46": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz", + "integrity": "sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==", + "requires": { + "punycode": "^2.1.1" + } + }, + "webidl-conversions": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", + "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==" + }, + "whatwg-url": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-11.0.0.tgz", + "integrity": "sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ==", + "requires": { + "tr46": "^3.0.0", + "webidl-conversions": "^7.0.0" + } + } + } + }, + "mongoose": { + "version": "6.1.6", + "resolved": "https://registry.npmjs.org/mongoose/-/mongoose-6.1.6.tgz", + "integrity": "sha512-wvNRJ547x/Bn3EMhwbVInNsEp2OdlMxk4Q/vfgAkw8OI+giZQ72S90ZH0H6VzzIhs4lCU9SwXHYa2O0sPicnEQ==", + "requires": { + "@types/node": "< 17.0.6", + "bson": "^4.2.2", + "kareem": "2.3.3", + "mongodb": "4.2.2", + "mpath": "0.8.4", + "mquery": "4.0.0", + "ms": "2.1.2", + "regexp-clone": "1.0.0", + "sift": "13.5.2", + "sliced": "1.0.1" + }, + "dependencies": { + "@types/node": { + "version": "17.0.5", + "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.5.tgz", + "integrity": "sha512-w3mrvNXLeDYV1GKTZorGJQivK6XLCoGwpnyJFbJVK/aTBQUxOCaa/GlFAAN3OTDFcb7h5tiFG+YXCO2By+riZw==" + } + } + }, + "mpath": { + "version": "0.8.4", + "resolved": "https://registry.npmjs.org/mpath/-/mpath-0.8.4.tgz", + "integrity": "sha512-DTxNZomBcTWlrMW76jy1wvV37X/cNNxPW1y2Jzd4DZkAaC5ZGsm8bfGfNOthcDuRJujXLqiuS6o3Tpy0JEoh7g==" + }, + "mquery": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/mquery/-/mquery-4.0.0.tgz", + "integrity": "sha512-nGjm89lHja+T/b8cybAby6H0YgA4qYC/lx6UlwvHGqvTq8bDaNeCwl1sY8uRELrNbVWJzIihxVd+vphGGn1vBw==", + "requires": { + "debug": "4.x", + "regexp-clone": "^1.0.0", + "sliced": "1.0.1" + } + }, "ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "node-fetch": { "version": "2.6.5", @@ -1588,6 +1926,11 @@ "once": "^1.3.1" } }, + "punycode": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" + }, "puppeteer": { "version": "13.0.1", "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-13.0.1.tgz", @@ -1619,6 +1962,11 @@ "util-deprecate": "^1.0.1" } }, + "regexp-clone": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/regexp-clone/-/regexp-clone-1.0.0.tgz", + "integrity": "sha512-TuAasHQNamyyJ2hb97IuBEif4qBHGjPHBS64sZwytpLEqtBQ1gPJTnOaQ6qmpET16cK14kkjbazl6+p0RRv0yw==" + }, "rimraf": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", @@ -1643,11 +1991,39 @@ "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", "dev": true }, + "saslprep": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/saslprep/-/saslprep-1.0.3.tgz", + "integrity": "sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag==", + "optional": true, + "requires": { + "sparse-bitfield": "^3.0.3" + } + }, "sax": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" }, + "sift": { + "version": "13.5.2", + "resolved": "https://registry.npmjs.org/sift/-/sift-13.5.2.tgz", + "integrity": "sha512-+gxdEOMA2J+AI+fVsCqeNn7Tgx3M9ZN9jdi95939l1IJ8cZsqS8sqpJyOkic2SJk+1+98Uwryt/gL6XDaV+UZA==" + }, + "sliced": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/sliced/-/sliced-1.0.1.tgz", + "integrity": "sha1-CzpmK10Ewxd7GSa+qCsD+Dei70E=" + }, + "sparse-bitfield": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", + "integrity": "sha1-/0rm5oZWBWuks+eSqzM004JzyhE=", + "optional": true, + "requires": { + "memory-pager": "^1.0.2" + } + }, "split2": { "version": "3.2.2", "resolved": "https://registry.npmjs.org/split2/-/split2-3.2.2.tgz", diff --git a/package.json b/package.json index 488a934..958d5f1 100644 --- a/package.json +++ b/package.json @@ -12,11 +12,11 @@ "author": "", "license": "ISC", "dependencies": { - "@types/node": "^14.18.5", "colors": "^1.4.0", "escape-goat": "^3.0.0", "esm": "^3.2.25", "moment": "^2.29.1", + "mongoose": "^6.1.6", "node-fetch": "^2.6.1", "rss-parser": "^3.10.0", "ts-node": "^10.4.0", @@ -24,9 +24,12 @@ "uuid": "^8.3.2" }, "devDependencies": { + "@types/node": "^17.0.8", "@types/pg": "^8.6.3", - "puppeteer": "^13.0.1", + "@types/puppeteer": "^5.4.4", "@types/uuid": "^8.3.4", - "pg": "^8.7.1" + "fs": "^0.0.1-security", + "pg": "^8.7.1", + "puppeteer": "^13.0.1" } } diff --git a/scrapers/ccpl.js b/scrapers/ccpl.js deleted file mode 100644 index 2b2c0ef..0000000 --- a/scrapers/ccpl.js +++ /dev/null @@ -1,31 +0,0 @@ -const puppeteer = require('puppeteer'); - -async function run() { - const browser = await puppeteer.launch({headless:false}); - const page = await browser.newPage(); - - await page.goto('https://www.cc-paysdelimours.fr/agenda'); - await page.waitFor('#widgit_results_agenda'); - - // chaque lien d'évènement est un .widgit_result - // titre: .widgit_result .widgit_title - // titre: #widgit_event_details .widgit_title - - - let sel = '#widgit_results_agenda .widgit_result .title'; - - let titles = await page.evaluate((sel) => { - let element = document.querySelectorAll(sel); - let titleList=[]; - - if(element.length){ - console.log('title', element.innerHTML); - titleList.push(element.innerHTML); - } - return titleList; - }); - - browser.close(); -} - -run(); \ No newline at end of file diff --git a/scrapers/ccpl.ts b/scrapers/ccpl.ts new file mode 100644 index 0000000..bed04ae --- /dev/null +++ b/scrapers/ccpl.ts @@ -0,0 +1,82 @@ +const puppeteer = require('puppeteer'); +import * as fs from 'fs'; + +// configure database +const mongoose = require('mongoose'); +const EventMobilizon = require('./../models/Event'); + +// get data from webpage + +async function run() { + + let options = {}; + options = {headless: false, devtools: true } + const browser = await puppeteer.launch(options); + const page = await browser.newPage(); + + + await page.goto('https://www.cc-paysdelimours.fr/agenda'); + + // chaque lien d'évènement est un .widgit_result + // titre: .widgit_result .widgit_title + // titre: #widgit_event_details .widgit_title + + + await page.evaluate(() => { + const scrapped: any = {}; + let sel = '#widgit_results_agenda .widgit_result .title'; + + let listOfElements: any = document.querySelectorAll(sel); + let titleList: any = []; + + + if (listOfElements.length) { + listOfElements.forEach((elem: any) => { + console.log('title', elem.innerHTML); + titleList.push(elem.innerHTML); + }) + } + scrapped.selector = sel; + scrapped.titleList = titleList; + // sauver les contenus dans un fichier json + let fileName = 'ccpl_scrapped.json'; + + fs.writeFile( + `./sources_examples/${fileName}`, + JSON.stringify(scrapped, null, 4), + "utf8", + (err: any) => { + if (err) { + console.log(`Error writing file: ${err}`); + } else { + console.log(`File ${fileName} is written successfully!`); + } + } + ); + console.log('titleList', titleList); + return titleList; + }); + // browser.close(); + console.log('DONE'); +} + +function upsertEventMobilizon(eventObject:any) { + + const DB_URL = 'mongodb://localhost/thal'; + + if (mongoose.connection.readyState == 0) { mongoose.connect(DB_URL); } + + // if this email exists, update the entry, don't insert + let conditions = { title: eventObject.title }; + let options = { upsert: true, new: true, setDefaultsOnInsert: true }; + + EventMobilizon.findOneAndUpdate(conditions, eventObject, options, (err:any, result:any) => { + if (err){ + console.log('result,err', result,err); + throw err; + } + }); +} +run(); + + diff --git a/sources_examples/input.csv b/sources_examples/input.csv new file mode 100644 index 0000000..d4f7f56 --- /dev/null +++ b/sources_examples/input.csv @@ -0,0 +1 @@ +title;date_start;date_end;description;url; \ No newline at end of file diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..4fc4d0d --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "lib": ["es5", "es6", "dom"], + "outDir": "./dist/", + "sourceMap": true, + "noImplicitAny": true, + "module": "commonjs", + "target": "es6", + "moduleResolution": "node", + "jsx": "react", + "types": [ + "node", + "vue-types", + ] + }, + "include": ["./**/*"] +} diff --git a/utils.ts b/utils.ts index 60d09e3..93cf596 100644 --- a/utils.ts +++ b/utils.ts @@ -100,7 +100,7 @@ class utils { /** * file management */ - writeFile = (fileName: string, data: any, formatData: any) => { + writeFile(fileName: string, data: any, formatData: any){ let dataToSave = data; if (formatData == 'json') { dataToSave = JSON.stringify(data, null, 4) @@ -293,6 +293,7 @@ class utils { } } + } export default utils; diff --git a/yarn.lock b/yarn.lock index 3e51f82..46f7e05 100644 --- a/yarn.lock +++ b/yarn.lock @@ -34,10 +34,15 @@ "resolved" "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.2.tgz" "version" "1.0.2" -"@types/node@*", "@types/node@^14.18.5": - "integrity" "sha512-LMy+vDDcQR48EZdEx5wRX1q/sEl6NdGuHXPnfeL8ixkwCOSZ2qnIyIZmcCbdX0MeRqHhAcHmX+haCbrS8Run+A==" - "resolved" "https://registry.npmjs.org/@types/node/-/node-14.18.5.tgz" - "version" "14.18.5" +"@types/node@*", "@types/node@^17.0.8": + "integrity" "sha512-YofkM6fGv4gDJq78g4j0mMuGMkZVxZDgtU0JRdx6FgiJDG+0fY0GKVolOV8WqVmEhLCXkQRjwDdKyPxJp/uucg==" + "resolved" "https://registry.npmjs.org/@types/node/-/node-17.0.8.tgz" + "version" "17.0.8" + +"@types/node@< 17.0.6": + "integrity" "sha512-w3mrvNXLeDYV1GKTZorGJQivK6XLCoGwpnyJFbJVK/aTBQUxOCaa/GlFAAN3OTDFcb7h5tiFG+YXCO2By+riZw==" + "resolved" "https://registry.npmjs.org/@types/node/-/node-17.0.5.tgz" + "version" "17.0.5" "@types/pg@^8.6.3": "integrity" "sha512-P0RrXJcw1hPS+KF0nBCC3FEEdZBfRsHbYtAzbY2QTc0gC4jFQvjQxIKXs0X/NgPhPI4DbAzdeW7WMCjaWjT7Pg==" @@ -48,11 +53,31 @@ "pg-protocol" "*" "pg-types" "^2.2.0" +"@types/puppeteer@^5.4.4": + "integrity" "sha512-3Nau+qi69CN55VwZb0ATtdUAlYlqOOQ3OfQfq0Hqgc4JMFXiQT/XInlwQ9g6LbicDslE6loIFsXFklGh5XmI6Q==" + "resolved" "https://registry.npmjs.org/@types/puppeteer/-/puppeteer-5.4.4.tgz" + "version" "5.4.4" + dependencies: + "@types/node" "*" + "@types/uuid@^8.3.4": "integrity" "sha512-c/I8ZRb51j+pYGAu5CrFMRxqZ2ke4y2grEBO5AUjgSkSk+qT2Ea+OdWElz/OiMf5MNpn2b17kuVBwZLQJXzihw==" "resolved" "https://registry.npmjs.org/@types/uuid/-/uuid-8.3.4.tgz" "version" "8.3.4" +"@types/webidl-conversions@*": + "integrity" "sha512-XAahCdThVuCFDQLT7R7Pk/vqeObFNL3YqRyFZg+AqAP/W1/w3xHaIxuW7WszQqTbIBOPRcItYJIou3i/mppu3Q==" + "resolved" "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-6.1.1.tgz" + "version" "6.1.1" + +"@types/whatwg-url@^8.2.1": + "integrity" "sha512-2YubE1sjj5ifxievI5Ge1sckb9k/Er66HyR2c+3+I6VDUUg1TLPdYYTEbQ+DjRkS4nTxMJhgWfSfMRD2sl2EYQ==" + "resolved" "https://registry.npmjs.org/@types/whatwg-url/-/whatwg-url-8.2.1.tgz" + "version" "8.2.1" + dependencies: + "@types/node" "*" + "@types/webidl-conversions" "*" + "@types/yauzl@^2.9.1": "integrity" "sha512-8uALY5LTvSuHgloDVUvWP3pIauILm+8/0pDMokuDYIoNsOkSwd5AiHBTSEJjKTDcZr5z8UpgOWZkxBF4iJftoA==" "resolved" "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.9.2.tgz" @@ -109,6 +134,13 @@ "balanced-match" "^1.0.0" "concat-map" "0.0.1" +"bson@^4.2.2", "bson@^4.6.0": + "integrity" "sha512-I1LQ7Hz5zgwR4QquilLNZwbhPw0Apx7i7X9kGMBTsqPdml/03Q9NBtD9nt/19ahjlphktQImrnderxqpzeVDjw==" + "resolved" "https://registry.npmjs.org/bson/-/bson-4.6.1.tgz" + "version" "4.6.1" + dependencies: + "buffer" "^5.6.0" + "buffer-crc32@~0.2.3": "integrity" "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=" "resolved" "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz" @@ -119,7 +151,7 @@ "resolved" "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz" "version" "2.0.0" -"buffer@^5.2.1", "buffer@^5.5.0": +"buffer@^5.2.1", "buffer@^5.5.0", "buffer@^5.6.0": "integrity" "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==" "resolved" "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz" "version" "5.7.1" @@ -147,13 +179,18 @@ "resolved" "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz" "version" "1.1.1" -"debug@^4.1.1", "debug@4", "debug@4.3.2": +"debug@^4.1.1", "debug@4", "debug@4.3.2", "debug@4.x": "integrity" "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==" "resolved" "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz" "version" "4.3.2" dependencies: "ms" "2.1.2" +"denque@^2.0.1": + "integrity" "sha512-tfiWc6BQLXNLpNiR5iGd0Ocu3P3VpxfzFiqubLgMfhfOw9WyvgJBd46CClNn9k3qfbjvT//0cf7AlYRX/OslMQ==" + "resolved" "https://registry.npmjs.org/denque/-/denque-2.0.1.tgz" + "version" "2.0.1" + "devtools-protocol@0.0.937139": "integrity" "sha512-daj+rzR3QSxsPRy5vjjthn58axO8c11j58uY0lG5vvlJk/EiOdCWOptGdkXDjtuRHr78emKq0udHCXM4trhoDQ==" "resolved" "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.937139.tgz" @@ -222,6 +259,11 @@ "resolved" "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz" "version" "1.0.0" +"fs@^0.0.1-security": + "integrity" "sha1-invTcYa23d84E/I4WLV+yq9eQdQ=" + "resolved" "https://registry.npmjs.org/fs/-/fs-0.0.1-security.tgz" + "version" "0.0.1-security" + "get-stream@^5.1.0": "integrity" "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==" "resolved" "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz" @@ -267,6 +309,11 @@ "resolved" "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz" "version" "2.0.4" +"kareem@2.3.3": + "integrity" "sha512-uESCXM2KdtOQ8LOvKyTUXEeg0MkYp4wGglTIpGcYHvjJcS5sn2Wkfrfit8m4xSbaNDAw2KdI9elgkOxZbrFYbg==" + "resolved" "https://registry.npmjs.org/kareem/-/kareem-2.3.3.tgz" + "version" "2.3.3" + "locate-path@^5.0.0": "integrity" "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==" "resolved" "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz" @@ -279,6 +326,11 @@ "resolved" "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz" "version" "1.3.6" +"memory-pager@^1.0.2": + "integrity" "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==" + "resolved" "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz" + "version" "1.5.0" + "minimatch@^3.0.4": "integrity" "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==" "resolved" "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz" @@ -296,6 +348,55 @@ "resolved" "https://registry.npmjs.org/moment/-/moment-2.29.1.tgz" "version" "2.29.1" +"mongodb-connection-string-url@^2.3.2": + "integrity" "sha512-d5Kd2bVsKcSA7YI/yo57fSTtMwRQdFkvc5IZwod1RRxJtECeWPPSo7zqcUGJELifRA//Igs4spVtYAmvFCatug==" + "resolved" "https://registry.npmjs.org/mongodb-connection-string-url/-/mongodb-connection-string-url-2.4.1.tgz" + "version" "2.4.1" + dependencies: + "@types/whatwg-url" "^8.2.1" + "whatwg-url" "^11.0.0" + +"mongodb@4.2.2": + "integrity" "sha512-zt8rCTnTKyMQppyt63qMnrLM5dbADgUk18ORPF1XbtHLIYCyc9hattaYHi0pqMvNxDpgGgUofSVzS+UQErgTug==" + "resolved" "https://registry.npmjs.org/mongodb/-/mongodb-4.2.2.tgz" + "version" "4.2.2" + dependencies: + "bson" "^4.6.0" + "denque" "^2.0.1" + "mongodb-connection-string-url" "^2.3.2" + optionalDependencies: + "saslprep" "^1.0.3" + +"mongoose@^6.1.6": + "integrity" "sha512-wvNRJ547x/Bn3EMhwbVInNsEp2OdlMxk4Q/vfgAkw8OI+giZQ72S90ZH0H6VzzIhs4lCU9SwXHYa2O0sPicnEQ==" + "resolved" "https://registry.npmjs.org/mongoose/-/mongoose-6.1.6.tgz" + "version" "6.1.6" + dependencies: + "@types/node" "< 17.0.6" + "bson" "^4.2.2" + "kareem" "2.3.3" + "mongodb" "4.2.2" + "mpath" "0.8.4" + "mquery" "4.0.0" + "ms" "2.1.2" + "regexp-clone" "1.0.0" + "sift" "13.5.2" + "sliced" "1.0.1" + +"mpath@0.8.4": + "integrity" "sha512-DTxNZomBcTWlrMW76jy1wvV37X/cNNxPW1y2Jzd4DZkAaC5ZGsm8bfGfNOthcDuRJujXLqiuS6o3Tpy0JEoh7g==" + "resolved" "https://registry.npmjs.org/mpath/-/mpath-0.8.4.tgz" + "version" "0.8.4" + +"mquery@4.0.0": + "integrity" "sha512-nGjm89lHja+T/b8cybAby6H0YgA4qYC/lx6UlwvHGqvTq8bDaNeCwl1sY8uRELrNbVWJzIihxVd+vphGGn1vBw==" + "resolved" "https://registry.npmjs.org/mquery/-/mquery-4.0.0.tgz" + "version" "4.0.0" + dependencies: + "debug" "4.x" + "regexp-clone" "^1.0.0" + "sliced" "1.0.1" + "ms@2.1.2": "integrity" "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" "resolved" "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz" @@ -452,6 +553,11 @@ "end-of-stream" "^1.1.0" "once" "^1.3.1" +"punycode@^2.1.1": + "integrity" "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" + "resolved" "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz" + "version" "2.1.1" + "puppeteer@^13.0.1": "integrity" "sha512-wqGIx59LzYqWhYcJQphMT+ux0sgatEUbjKG0lbjJxNVqVIT3ZC5m4Bvmq2gHE3qhb63EwS+rNkql08bm4BvO0A==" "resolved" "https://registry.npmjs.org/puppeteer/-/puppeteer-13.0.1.tgz" @@ -479,6 +585,11 @@ "string_decoder" "^1.1.1" "util-deprecate" "^1.0.1" +"regexp-clone@^1.0.0", "regexp-clone@1.0.0": + "integrity" "sha512-TuAasHQNamyyJ2hb97IuBEif4qBHGjPHBS64sZwytpLEqtBQ1gPJTnOaQ6qmpET16cK14kkjbazl6+p0RRv0yw==" + "resolved" "https://registry.npmjs.org/regexp-clone/-/regexp-clone-1.0.0.tgz" + "version" "1.0.0" + "rimraf@3.0.2": "integrity" "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==" "resolved" "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz" @@ -499,11 +610,35 @@ "resolved" "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz" "version" "5.2.1" +"saslprep@^1.0.3": + "integrity" "sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag==" + "resolved" "https://registry.npmjs.org/saslprep/-/saslprep-1.0.3.tgz" + "version" "1.0.3" + dependencies: + "sparse-bitfield" "^3.0.3" + "sax@>=0.6.0": "integrity" "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" "resolved" "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz" "version" "1.2.4" +"sift@13.5.2": + "integrity" "sha512-+gxdEOMA2J+AI+fVsCqeNn7Tgx3M9ZN9jdi95939l1IJ8cZsqS8sqpJyOkic2SJk+1+98Uwryt/gL6XDaV+UZA==" + "resolved" "https://registry.npmjs.org/sift/-/sift-13.5.2.tgz" + "version" "13.5.2" + +"sliced@1.0.1": + "integrity" "sha1-CzpmK10Ewxd7GSa+qCsD+Dei70E=" + "resolved" "https://registry.npmjs.org/sliced/-/sliced-1.0.1.tgz" + "version" "1.0.1" + +"sparse-bitfield@^3.0.3": + "integrity" "sha1-/0rm5oZWBWuks+eSqzM004JzyhE=" + "resolved" "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz" + "version" "3.0.3" + dependencies: + "memory-pager" "^1.0.2" + "split2@^3.1.1": "integrity" "sha512-9NThjpgZnifTkJpzTZ7Eue85S49QwpNhZTq6GRJwObb6jnLFNGB7Qm73V5HewTROPyxD0C29xqmaI68bQtV+hg==" "resolved" "https://registry.npmjs.org/split2/-/split2-3.2.2.tgz" @@ -544,6 +679,13 @@ "resolved" "https://registry.npmjs.org/through/-/through-2.3.8.tgz" "version" "2.3.8" +"tr46@^3.0.0": + "integrity" "sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==" + "resolved" "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz" + "version" "3.0.0" + dependencies: + "punycode" "^2.1.1" + "tr46@~0.0.3": "integrity" "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o=" "resolved" "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz" @@ -595,6 +737,19 @@ "resolved" "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz" "version" "3.0.1" +"webidl-conversions@^7.0.0": + "integrity" "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==" + "resolved" "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz" + "version" "7.0.0" + +"whatwg-url@^11.0.0": + "integrity" "sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ==" + "resolved" "https://registry.npmjs.org/whatwg-url/-/whatwg-url-11.0.0.tgz" + "version" "11.0.0" + dependencies: + "tr46" "^3.0.0" + "webidl-conversions" "^7.0.0" + "whatwg-url@^5.0.0": "integrity" "sha1-lmRU6HZUYuN2RNNib2dCzotwll0=" "resolved" "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz"