From 24f590d7e550dd9edb804e448007801e81b6057d Mon Sep 17 00:00:00 2001 From: Tykayn Date: Tue, 18 Apr 2023 12:23:52 +0200 Subject: [PATCH] better tags search --- firefox_places_json_to_json_activities.mjs | 4 +- google_searches_to_json_activities.mjs | 6 +- parse_orgmode_to_json.mjs | 198 ++++++++++++++------- sms_to_csv.mjs | 8 +- 4 files changed, 139 insertions(+), 77 deletions(-) diff --git a/firefox_places_json_to_json_activities.mjs b/firefox_places_json_to_json_activities.mjs index 2839655..830f5ab 100644 --- a/firefox_places_json_to_json_activities.mjs +++ b/firefox_places_json_to_json_activities.mjs @@ -1,5 +1,5 @@ import fs from "node-fs"; -import {writeFileInOuputFolder} from "./parse_orgmode_to_json.mjs"; +import {writeFileInOuputFolderFromJsonObject} from "./parse_orgmode_to_json.mjs"; import moment from "moment"; import {headersTsv} from "./utils.mjs"; @@ -63,7 +63,7 @@ function convertJsonToActivities(sourceFilePath, outputFileName) { } }) console.log('events', events.length) - writeFileInOuputFolder(outputFileName, events.join("\n")).then(r => console.log('r', r)) + writeFileInOuputFolderFromJsonObject(outputFileName, events.join("\n")).then(r => console.log('r', r)) }) } diff --git a/google_searches_to_json_activities.mjs b/google_searches_to_json_activities.mjs index 3bd562c..fe3a10f 100644 --- a/google_searches_to_json_activities.mjs +++ b/google_searches_to_json_activities.mjs @@ -1,5 +1,5 @@ import fs from "node-fs"; -import {writeFileInOuputFolder} from "./parse_orgmode_to_json.mjs"; +import {writeFileInOuputFolderFromJsonObject} from "./parse_orgmode_to_json.mjs"; import moment from "moment"; @@ -55,7 +55,7 @@ async function convertJsonToActivities(sourceFilePath, outputFileName) allEvents.push(newLine) }) console.log('events', events.length) - writeFileInOuputFolder(outputFileName, events.join("\n")) + writeFileInOuputFolderFromJsonObject(outputFileName, events.join("\n")) }) } @@ -69,7 +69,7 @@ fileArray.forEach(item=>{ convertJsonToActivities(sourceFilePath,outputFileName); }) -writeFileInOuputFolder(fileNameAllEvents, events.join("\n")) +writeFileInOuputFolderFromJsonObject(fileNameAllEvents, events.join("\n")) diff --git a/parse_orgmode_to_json.mjs b/parse_orgmode_to_json.mjs index 3a55432..3f5c7a0 100644 --- a/parse_orgmode_to_json.mjs +++ b/parse_orgmode_to_json.mjs @@ -20,6 +20,7 @@ let tasksObjectsForJsonExport = [] let headersByKind = {} let writeJsonAfterParse = false; writeJsonAfterParse = true; +moment.locale('FR') /************************************************************** * fetch the source orgmode file to read its contents @@ -59,10 +60,10 @@ let statistics = { havingNoDate: 0, oldEst: 0, mostRecent: 0, - years:{}, - weeks:{}, - months:{}, - days:{} + years: {}, + weeks: {}, + months: {}, + days: {} } } @@ -90,15 +91,17 @@ let isLogbook = false; let isFirst = true; // init first task object as empty clone -let currentTask = {...task}; +let currentTask = Object.create(task); /** * add to tasks to export and refresh current task */ function addAndRefreshCurrentTask() { tasksObjectsForJsonExport.push(currentTask) - currentTask = {...task}; + // réinitialisation de tâche pour remplir de nouveau + currentTask = Object.create(task); currentTask.dates = {}; + currentTask.tags = []; }; function makeWordsStatistics(sentence) { @@ -124,58 +127,89 @@ function statisticDateFill(keyword, dateFoundElement) { // décompte par années - let convertedDate = new Date(dateFoundElement ) + let convertedDate = new Date(dateFoundElement) let yearOfDate = convertedDate.getFullYear() - let monthOfDate = convertedDate.getFullYear()+ '-'+ convertedDate.getMonth() - let convertedMonth = convertedDate.getMonth() < 10 ? '0'+convertedDate.getMonth(): convertedDate.getMonth() - let convertedDay = convertedDate.getDay() < 10 ? '0'+convertedDate.getDay(): convertedDate.getDay() - let dayOfDate = convertedDate.getFullYear()+ '-'+ convertedMonth+ '-' + convertedDay + let monthOfDate = yearOfDate + '-' + convertedDate.getMonth() + // add zeros + let convertedWeek = moment(convertedDate).week() < 10 ? '0' + moment(convertedDate).week() : moment(convertedDate).week() + let weekOfDate = yearOfDate + '-' + convertedWeek - console.log('convertedDate', convertedDate,yearOfDate) - if(!statistics.dates.years[yearOfDate]){ + let convertedMonth = convertedDate.getMonth() < 10 ? '0' + convertedDate.getMonth() : convertedDate.getMonth() + let convertedDay = convertedDate.getDay() < 10 ? '0' + convertedDate.getDay() : convertedDate.getDay() + let dayOfDate = convertedDate.getFullYear() + '-' + convertedMonth + '-' + convertedDay + + if (!statistics.dates.years[yearOfDate]) { statistics.dates.years[yearOfDate] = { - created:0, - closed:0, + created: 0, + closed: 0, } } - if(keyword=== 'CLOSED'){ - statistics.dates.years[yearOfDate].closed++; + if (keyword === 'CLOSED') { + statistics.dates.years[yearOfDate].closed++; } - if(keyword=== 'CREATED'){ - statistics.dates.years[yearOfDate].created++; + if (keyword === 'CREATED') { + statistics.dates.years[yearOfDate].created++; + } + // par année-semaine + if (!statistics.dates.weeks[weekOfDate]) { + statistics.dates.weeks[weekOfDate] = { + created: 0, + closed: 0, + } + } + if (keyword === 'CLOSED') { + statistics.dates.weeks[weekOfDate].closed++; + } + if (keyword === 'CREATED') { + statistics.dates.weeks[weekOfDate].created++; } // décompte par mois - if(!statistics.dates.months[monthOfDate]){ + if (!statistics.dates.months[monthOfDate]) { statistics.dates.months[monthOfDate] = { - created:0, - closed:0, + created: 0, + closed: 0, } } - if(keyword=== 'CLOSED'){ + if (keyword === 'CLOSED') { statistics.dates.months[monthOfDate].closed++; } - if(keyword=== 'CREATED'){ + if (keyword === 'CREATED') { statistics.dates.months[monthOfDate].created++; } // décompte par jours - if(!statistics.dates.days[dayOfDate]){ + if (!statistics.dates.days[dayOfDate]) { statistics.dates.days[dayOfDate] = { - created:0, - closed:0, + created: 0, + closed: 0, } } - if(keyword=== 'CLOSED'){ + if (keyword === 'CLOSED') { statistics.dates.days[dayOfDate].closed++; } - if(keyword=== 'CREATED'){ + if (keyword === 'CREATED') { statistics.dates.days[dayOfDate].created++; } } +function findOldestDate(currentDate) { + // trouver la plus ancienne date + + + if (!statistics.dates.oldEst) { + statistics.dates.oldEst = currentDate; + } else { + var beginningTime = moment(statistics.dates.oldEst); + var endTime = moment(currentDate); + if (!beginningTime.isBefore(endTime)) { + statistics.dates.oldEst = currentDate; + } + } +} + /********************** * loop to parse all *********************/ @@ -217,7 +251,7 @@ fs.readFile(sourceFilePath, 'utf8', function (err, data) { headers.push(cleanHeader(line)) currentTask.header = cleanHeader(line); - makeWordsStatistics(cleanHeader(line)); + // makeWordsStatistics(cleanHeader(line)); stateKeywordList.forEach(keyword => { let keywordIsFound = lineHasKeyword(line, keyword) @@ -228,10 +262,10 @@ fs.readFile(sourceFilePath, 'utf8', function (err, data) { // trouver les tags - let tagsFound = line.match(/\:(.*)\:/g) - if (tagsFound) { - tagsFound = tagsFound[0]; - let tagList = tagsFound.split(':'); + let tagsList = line.match(/\:([\w\_]*)\:/g) + if (tagsList) { + tagsList = tagsList[0]; + let tagList = tagsList.split(':'); if (tagList.length) { tagList.forEach(tag => { @@ -269,34 +303,36 @@ fs.readFile(sourceFilePath, 'utf8', function (err, data) { currentTask.dates[keyword] = ''; } - let convertedDate = dateFound[0].substring(0, 10) + let convertedDate = dateFound[0].substring(0, 10); + + if (dateFound[0].length == 15) { + // sans heure: "2022-12-21 mer." + convertedDate = moment(dateFound[0], 'YYYY-MM-DD ddd') + + } else if (dateFound[0].length == 21) { + // avec heure: "2022-11-01 mar. 00:44" + convertedDate = moment(dateFound[0], 'YYYY-MM-DD ddd HH:mm') + } let formattedDate = moment(convertedDate).format() - statisticDateFill(keyword,convertedDate) - + statisticDateFill(keyword, convertedDate) + findOldestDate(convertedDate) currentTask.dates[keyword] = formattedDate; - console.log('formattedDate', keyword, formattedDate) - // trouver la plus ancienne date - if (!statistics.dates.oldEst) { - statistics.dates.oldEst = formattedDate; - } else { - var beginningTime = moment(statistics.dates.oldEst); - var endTime = moment(convertedDate); - if (!beginningTime.isBefore(endTime)) { - statistics.dates.oldEst = formattedDate; - } - } } else { // console.log('keyword', keyword) } }) } else { statistics.dates.havingNoDate += 1; - if (line.indexOf(dateKeywordList) !== -1 && line.indexOf(stateKeywordList) !== -1 && line.indexOf(sectionKeywordList) !== -1) { + if ( + line.indexOf(dateKeywordList) !== -1 && + line.indexOf(stateKeywordList) !== -1 && + line.indexOf(sectionKeywordList) !== -1 + ) { + - makeWordsStatistics(line) // ajouter le corps complet de la section après le header if (line.length && !isHeader) { @@ -305,6 +341,7 @@ fs.readFile(sourceFilePath, 'utf8', function (err, data) { currentTask.corpus += `${cleanedLine} ` + makeWordsStatistics(cleanedLine) } } } @@ -317,9 +354,17 @@ fs.readFile(sourceFilePath, 'utf8', function (err, data) { // ranger par valeur décroissante les tags let sorted_stats = []; - sorted_stats = Object.fromEntries( - Object.entries(statistics).sort(([, a], [, b]) => a - b) - ); + // rangement par valeur et par date + console.log('write file ', outputAbsolutePath, outputFileNameJson); + statistics.dates.years = sortByKey(statistics.dates.years) + statistics.dates.weeks = sortByKey(statistics.dates.weeks) + statistics.dates.months = sortByKey(statistics.dates.months) + statistics.dates.days = sortByKey(statistics.dates.days) + + statistics.tags = sortByValue(statistics.tags) + statistics.words = sortByValue(statistics.tags) + + sorted_stats = sortByKey(statistics) const jsonContent = { @@ -340,11 +385,22 @@ fs.readFile(sourceFilePath, 'utf8', function (err, data) { if (writeJsonAfterParse) { - writeFileInOuputFolder(outputFileNameJson, JSON.stringify(jsonContent)); + writeFileInOuputFolderFromJsonObject(outputFileNameJson, jsonContent); } }) +/** + * ranger un objet littéral selon la valeur décroissante de ses paires + * @param literalobject + * @returns {any} + */ +function sortByValue(literalobject) { + return Object.fromEntries( + Object.entries(literalobject).sort(([, a], [, b]) => b - a) + ); +} + function lineHasKeyword(line, keyword = 'TODO') { let isFound = (line.indexOf('* ' + keyword) !== -1) @@ -401,16 +457,6 @@ function searchDate(line) { } -/** - * afin de trouver la première date liée à une tâche parmi celles mentionnées, il faut comparer les dates - * @param date1 - * @param date2 - */ -function compareDatesAndKeepOldest(date1, date2) { - date1 = moment(date1) - date2 = moment(date2) -} - /** * get the cleaned content of the header * @param line @@ -428,13 +474,29 @@ function cleanHeader(line) { return line.trim(); } -export async function writeFileInOuputFolder(fileName, fileContent) { - console.log('write file ', outputAbsolutePath, fileName); - console.log('date statistics ', statistics.dates); +/** + * ranger un objet littéral par ordre alphabétique de ses clés + * @param objectStuff + * @returns {{}} + */ +function sortByKey(objectStuff) { + return Object.keys(objectStuff).sort().reduce( + (obj, key) => { + obj[key] = objectStuff[key]; + return obj; + }, + {} + ); +} + + +export async function writeFileInOuputFolderFromJsonObject(fileName, jsonObjectThing) { + + console.log('statistics.words', statistics.words) return await fs.writeFile( `${outputAbsolutePath}${fileName}`, - fileContent, + JSON.stringify(jsonObjectThing), "utf8", (err) => { if (err) { diff --git a/sms_to_csv.mjs b/sms_to_csv.mjs index 65e3b70..ad10691 100644 --- a/sms_to_csv.mjs +++ b/sms_to_csv.mjs @@ -1,6 +1,6 @@ import fs from "node-fs"; import convert from "xml-js"; -import {writeFileInOuputFolder} from "./parse_orgmode_to_json.mjs"; +import {writeFileInOuputFolderFromJsonObject} from "./parse_orgmode_to_json.mjs"; /********************** * initialize configs @@ -75,7 +75,7 @@ function convertJsonToCsv(sourceFilePath, outputFileName) { ) }) console.log('events', events) - writeFileInOuputFolder(outputFileName, events.join("\n")) + writeFileInOuputFolderFromJsonObject(outputFileName, events.join("\n")) }) } @@ -94,7 +94,7 @@ function openSourceFile() { // console.log('smses', Object.keys(jsonConversion)) console.log('jsonConversion[0]', jsonConversion['10']) - writeFileInOuputFolder(outputFileJson, jsonConversion) + writeFileInOuputFolderFromJsonObject(outputFileJson, jsonConversion) convertJsonToCsv(sourceFileJson, outputFileCsv) }) @@ -153,7 +153,7 @@ function convertJsonToCsv(sourceFilePath, outputFileName) { ) }) console.log('events', events) - writeFileInOuputFolder(outputFileName, events.join("\n")) + writeFileInOuputFolderFromJsonObject(outputFileName, events.join("\n")) }) }