/** * convertir un fichier .org vers des données structurées en json * @type {*} */ import fs from 'node-fs' import moment from 'moment' import * as emoji from 'node-emoji' /********************** * initialize configs **********************/ const sourceFileName = 'all_tasks.org' const sourceFilePath = './sources/' + sourceFileName const outputAbsolutePath = '~/Nextcloud/ressources/social sorting/output/' const outputFileNameJson = 'export_' + sourceFileName + '_parsed.json' let headers = [] let tasksObjectsForJsonExport = [] let headersByKind = {} let writeJsonAfterParse = false writeJsonAfterParse = true moment.locale('FR') const tada = emoji.get('tada') const gift = emoji.get('gift') /************************************************************** * fetch the source orgmode file to read its contents *************************************************************/ console.log('---------- parse some org file', sourceFilePath) if (!sourceFilePath) { console.error('pas de fichier à ouvrir') } fs.stat(sourceFilePath, function (err, stat) { if (err === null) { console.log(`File ${sourceFilePath} exists`) } else if (err.code === 'ENOENT') { // file does not exist console.error(`le fichier ${sourceFilePath} est introuvable. Impossible d en extraire des infos.`, err) } else { console.log('Some other error: ', err.code) } }) /********************** * search elements *********************/ let stateKeywordList = ['SOMEDAY', 'NEXT', 'TODO', 'CANCELLED', 'DONE', 'WAITING'] let dateKeywordList = ['CREATED', 'SCHEDULED', 'DEADLINE', 'CLOSED', 'Refiled'] let sectionKeywordList = ['PROPERTIES', 'LOGBOOK', 'END'] let propertiesSection = {} // TODO properties listing let logBookSection = {} // TODO logbook listing let statistics = { tags: {}, words: {}, dates: { havingDate: 0, havingNoDate: 0, oldEst: 0, mostRecent: 0, years: {}, weeks: {}, months: {}, days: {} } } let headerKeywordSearch = '[' + stateKeywordList.join('|') + ']' /** * task object example * @type {{level: string, header: string, dates: {CREATED: string, DONE: string, REFILED: string}, state: string, content: string, properties: {}, tags: [], tagsInherited: []}} */ let task = { header: '', level: '', corpus: '', state: '', tags: [], children: [], // TODO list children tasks with a reference to the parent when level is superior to previous task tagsInherited: [], // TODO inherit tags dates: { havingDate: 0, havingNoDate: 0, oldEst: 0, mostRecent: 0, years: {}, weeks: {}, months: {}, days: {}, }, logbook: {}, properties: {}, } let isHeader = false let isProperty = false let isLogbook = false let isFirst = true // init first task object as empty clone let currentTask = Object.create(task) /** * add to tasks to export and refresh current task */ function addAndRefreshCurrentTask () { makeWordsStatistics(currentTask.header.trim()) makeWordsStatistics(currentTask.corpus.trim()) tasksObjectsForJsonExport.push(currentTask) // réinitialisation de tâche pour remplir de nouveau currentTask = Object.create(task) currentTask.dates = {} currentTask.tags = [] } function makeWordsStatistics (sentence) { let split = sentence.split(' ') if (split && split.length) { split.forEach(word => { if (!statistics.words[word]) { statistics.words[word] = 0 } statistics.words[word]++ }) } } const dateStats = { created: 0, refiled: 0, closed: 0, cancelled: 0, scheduled: 0, deadline: 0, tasks_done: [] } function fillPeriodTime (periodStat, keyword) { if (keyword === 'CLOSED') { periodStat.closed++ } if (keyword === 'CREATED') { periodStat.created++ } if (keyword === 'Refiled') { periodStat.refiled++ } if (keyword === 'CANCELLED') { periodStat.cancelled++ } if (keyword === 'SCHEDULED') { periodStat.scheduled++ } if (keyword === 'DEADLINE') { periodStat.deadline++ } } /** * pour chaque période de temps, compter les tâches créées et fermées * @param keyword * @param dateFoundElement */ function statisticDateFill (keyword, dateFoundElement, header) { // décompte par années let convertedDate = new Date(dateFoundElement) let yearOfDate = convertedDate.getFullYear() let monthOfDate = yearOfDate + '-' + convertedDate.getMonth() // add zeros let convertedWeek = moment(convertedDate).week() < 10 ? '0' + moment(convertedDate).week() : moment(convertedDate).week() let weekOfDate = yearOfDate + '-' + convertedWeek let convertedMonth = convertedDate.getMonth() < 10 ? '0' + convertedDate.getMonth() : convertedDate.getMonth() let convertedDay = convertedDate.getDay() < 10 ? '0' + convertedDate.getDay() : convertedDate.getDay() let dayOfDate = convertedDate.getFullYear() + '-' + convertedMonth + '-' + convertedDay if (!statistics.dates.years[yearOfDate]) { statistics.dates.years[yearOfDate] = Object.create(dateStats) } fillPeriodTime(statistics.dates.years[yearOfDate], keyword) // par année-semaine if (!statistics.dates.weeks[weekOfDate]) { statistics.dates.weeks[weekOfDate] = Object.create(dateStats) } fillPeriodTime(statistics.dates.weeks[weekOfDate], keyword) statistics.dates.weeks[weekOfDate].tasks_done.push(header) // décompte par mois if (!statistics.dates.months[monthOfDate]) { statistics.dates.months[monthOfDate] = Object.create(dateStats) } fillPeriodTime(statistics.dates.months[monthOfDate], keyword) // décompte par jours if (!statistics.dates.days[dayOfDate]) { statistics.dates.days[dayOfDate] = Object.create(dateStats) } fillPeriodTime(statistics.dates.days[dayOfDate], keyword) statistics.dates.days[dayOfDate].tasks_done.push(header) } function findOldestDate (currentDate) { // trouver la plus ancienne date if (!statistics.dates.oldEst) { statistics.dates.oldEst = currentDate } else { var beginningTime = moment(statistics.dates.oldEst) var endTime = moment(currentDate) if (!beginningTime.isBefore(endTime)) { statistics.dates.oldEst = currentDate } } } /********************** * loop to parse all *********************/ fs.readFile(sourceFilePath, 'utf8', function (err, data) { if (err) { return console.log(err) } console.log(' parsing...') // parcourir chaque ligne du fichier org let everyline = data.split('\n') // trouver les entêtes toutes les lignes qui commencent par * et espace. everyline.forEach((line) => { // gérer la création d'objets définissant les tâches et leurs propriétés if (line.match(/^\*+? /)) { // add last task to export list if (!isFirst) { addAndRefreshCurrentTask() } else { isFirst = false } isHeader = true // compter les étoiles pour trouver le niveau du header if (line.match(/\*/g)) { let match = line.match(/\*/g) currentTask.level = match.length } // create a new task headers.push(cleanHeader(line)) currentTask.header = cleanHeader(line) stateKeywordList.forEach(keyword => { let keywordIsFound = lineHasKeyword(line, keyword) if (keywordIsFound) { currentTask.state = keyword } }) // trouver les tags let tagsList = line.match(/\:([\w\_]*)\:/g) if (tagsList) { tagsList = tagsList[0] let tagList = tagsList.split(':') if (tagList.length) { tagList.forEach(tag => { if (tag.length > 1) { if (!statistics.tags[tag]) { statistics.tags[tag] = 0 } statistics.tags[tag]++ currentTask.tags.push(tag) } }) } } // ------------- fin des recherches dans la ligne de Header ------------- } else { isHeader = false } // examen des lignes de corps de tâche, ou de corps de section suite au header. // classer les dates de création, cloture, et de logbook let dateFound = searchDate(line) if (dateFound) { /** * we have found a date in the current line */ statistics.dates.havingDate += 1 dateKeywordList.forEach(keyword => { if (lineHasSubstring(line, keyword)) { if (!currentTask.dates[keyword]) { currentTask.dates[keyword] = '' } let convertedDate = dateFound[0].substring(0, 10) if (dateFound[0].length === 15) { // sans heure: "2022-12-21 mer." convertedDate = moment(dateFound[0], 'YYYY-MM-DD ddd') } else if (dateFound[0].length === 19) { // avec secondes: "2022-11-01 00:44:12" convertedDate = moment(dateFound[0], 'YYYY-MM-DD HH:mm:ss') } else if (dateFound[0].length === 21) { // avec heure: "2022-11-01 mar. 00:44" convertedDate = moment(dateFound[0], 'YYYY-MM-DD ddd HH:mm') } let formattedDate = moment(convertedDate).format() // console.log('currentTask.header', currentTask.header) statisticDateFill(keyword, convertedDate, currentTask.header) findOldestDate(convertedDate) currentTask.dates[keyword] = formattedDate } else { // console.log('keyword', keyword) } }) } else { statistics.dates.havingNoDate += 1 if ( line.indexOf(dateKeywordList) !== -1 && line.indexOf(stateKeywordList) !== -1 && line.indexOf(sectionKeywordList) !== -1 ) { // ajouter le corps complet de la section après le header if (line.length && !isHeader) { let cleanedLine = line.replace(/\s\s/g, ' ') cleanedLine = line.replace(/ {2,}/g, ' ') cleanedLine = cleanedLine.trim() currentTask.corpus += `${cleanedLine} ` } } } }) // ajouter la dernière tâche parsée addAndRefreshCurrentTask() console.log('tasks : ', tasksObjectsForJsonExport.length) console.log(tada + ' parsing fini ' + tada) // ranger par valeur décroissante les tags let sorted_stats = [] // rangement par valeur et par date console.log('write file ', outputAbsolutePath, outputFileNameJson) statistics.dates.years = sortByKey(statistics.dates.years) statistics.dates.weeks = sortByKey(statistics.dates.weeks) statistics.dates.months = sortByKey(statistics.dates.months) statistics.dates.days = sortByKey(statistics.dates.days) statistics = sortByKey(statistics) const jsonContent = { statistics: { lines_count: everyline.length, headers_count: headers.length, statistics }, meta_data: { author: '@tykayn@mastodon.Cipherbliss.com', generated_at: new Date(), generated_from_file: sourceFilePath + sourceFileName, sources: 'https://forge.chapril.org/tykayn/org-report-stats.git' }, tasks_list: tasksObjectsForJsonExport } if (writeJsonAfterParse) { writeFileInOuputFolderFromJsonObject(outputFileNameJson, jsonContent) } }) function lineHasKeyword (line, keyword = 'TODO') { let isFound = (line.indexOf('* ' + keyword) !== -1) if (isFound) { createNewHeaderKind(keyword) headersByKind[keyword].push(line) if (!statistics[keyword]) { statistics[keyword] = 0 } statistics[keyword]++ } return isFound } function lineHasSubstring (line, keyword) { let isFound = (line.indexOf(keyword) !== -1) if (!statistics[keyword]) { statistics[keyword] = 0 } statistics[keyword]++ return isFound } function createNewHeaderKind (keyword) { if (!headersByKind[keyword]) { headersByKind[keyword] = [] } } /** * chercher des dates et heures au format * YYYY-MM-DD HH:II:SS * * @param line * @returns {*} */ function searchDate (line) { // return line.match(/[(\d{4}\-\d{2}\-\d{2} ?\d{2}?\:?\d{2}?\:?\d{2}?)(\d{4}\-\d{2}\-\d{2})]/) let simpleDay = line.match(/\d{4}\-\d{2}\-\d{2} \w{3}?\.?/) let simpleDayHour = line.match(/\d{4}\-\d{2}\-\d{2} \w{3}?\.? \d{2}\:\d{2}/) let simpleDayHourSec = line.match(/\d{4}\-\d{2}\-\d{2} \w{3}?\.? \d{2}\:\d{2}\:\d{2}/) if (simpleDayHourSec) { return simpleDayHourSec } if (simpleDayHour) { return simpleDayHour } if (simpleDay) { return simpleDay } } /** * get the cleaned content of the header * @param line */ function cleanHeader (line) { line = '' + line stateKeywordList.forEach(keyword => { line = line.replace(keyword, '') }) line = line.replace(/\** /, '') line = line.replace(/\[.*\]/g, '') line = line.replace(/\:.*\:/g, '') line = line.replace(' ', '') return line.trim() } /** * ranger un objet littéral par ordre alphabétique de ses clés * @param objectStuff * @returns {{}} */ function sortByKey (objectStuff) { return Object.keys(objectStuff).sort().reduce( (obj, key) => { obj[key] = objectStuff[key] return obj }, {} ) } /** * ranger un objet littéral selon la valeur décroissante de ses paires * @param literalobject * @returns {any} */ function sortByValue (literalobject) { let sortable = [] for (var keyName in literalobject) { sortable[keyName] = literalobject[keyName] } // return literalobject return sortable.sort(function (a, b) { return b[1] - a[1] }) } export async function writeFileInOuputFolderFromJsonObject (fileName, jsonObjectThing) { // console.log('statistics.dates', statistics.dates) return await fs.writeFile( `${outputAbsolutePath}${fileName}`, JSON.stringify(jsonObjectThing, null, 2), 'utf8', (err) => { if (err) { console.log(`Error writing file: ${err}`) } else { console.log(`\n ${gift} File ${fileName} is written successfully!`) } } ) }