mapping-geojson-osm/make_variance_from_csv.ts

/**
 prendre un CSV,
 examiner toutes les colonnes et leurs valeurs,
 garder en mémoire les valeurs uniques de chaque colonne
 faire un nouveau csv qui ne montre que les valeurs uniques pour chacune des colonnes
 et qui compte le nombre de valeurs
 **/
import utils from './mappings/utils'
import {parse} from 'csv'

const fs = require('fs')
const minimist = require('minimist')

let mini_arguments: any = minimist(process.argv.slice(2))

// interface VarianceType {
//     [key: string]: Array<string>
// }

let csv_content = 'variance de dataset\n';
let separator = ',';
let separator_fallback = ';';
let data_variance: any = {};

let folder = 'toilettes'
let inputFile = 'small.csv'
let fileNameOutput = `variance_${inputFile}`;
let inputPath = `./etalab_data/${folder}/${inputFile}`
// let inputPath = './etalab_data/toilettes/small_datas.csv'
let outputPath = `etalab_data/${folder}`
if (mini_arguments['source']) {
    inputPath = mini_arguments['source']
}
if (mini_arguments['separator']) {
    separator = mini_arguments['separator']
}
let columns_headings: Array<string> = [];
let lines_count = 0;
let longest_variance_count = 0;


function getColumnsFromRow(row: string) {
    let headings: any = []
    console.log('elem', row)

    headings = row.split(separator)
    return headings
}

console.log('open file ', inputPath)

/**
 * sort unique values ascending in each column
 */
function reorderValuesInDataVariance() {
    columns_headings.forEach((heading: string) => {

        data_variance[heading] = data_variance[heading].sort((a: any, b: any) => a - b)
    })
}

fs.readFile(inputPath, function (err: any, fileData: any) {

    if (err) {
        throw new Error(err)
    } else {
        parse(fileData, {columns: false, trim: true}, function (err: any, lines: any) {
            // Your CSV data is in an array of arrays passed to this callback as rows.

            if (err) {
                throw new Error(err)
            }
            console.log('line ', lines_count)
            console.log('lines.length', lines.length)

            lines.forEach((line: any) => {

                line = line[0]

                if (lines_count === 0) {
                    console.log('line', line)
                    if(line.indexOf(separator) === -1){
                        console.log('separator not found: ', separator, 'trying other separator')
                        if(line.indexOf(separator_fallback) === -1){
                            console.log('separator not found: ', separator_fallback)

                            throw new Error('no separator found in csv')

                        }else{
                            console.log('fallback separator found: ', separator_fallback, 'now using it ')
                            separator = separator_fallback
                        }
                    }

                    columns_headings = getColumnsFromRow(line)
                    console.log('columns_headings.length', columns_headings.length)
                    // console.log('columns_headings', columns_headings)
                    let headers = Object.keys(columns_headings)
                    columns_headings.forEach((header: string) => {
                        data_variance[header] = []
                    })
                    console.log('data_variance', data_variance)
                } else {
                    // lignes suivantes

                    let column_index = 0


                    line.split(separator).forEach((value: string) => {
                        value = value.trim()
                        let column_header_current = columns_headings[column_index]
                        // console.log('column_index', column_index)
                        // dans chaque colonne, vérifier que la valeur n'est pas déjà présente
                        // dans les index de variance
                        // si la valeur est nouvelle, l'ajouter

                        if (data_variance[column_header_current].indexOf(value) === -1) {
                            data_variance[column_header_current].push(value)
                            if (
                                data_variance[column_header_current].length > longest_variance_count
                            ) {
                                longest_variance_count = data_variance[column_header_current].length
                            }

                        } else {
                            console.log('value', value, ' déjà présente dans la collection', column_header_current)
                        }

                        column_index++

                    })

                }

                lines_count++
            })

            console.log('lines_count', lines_count)
            console.log('longest_variance_count', longest_variance_count)

            reorderValuesInDataVariance()
            utils.writeFile(fileNameOutput, writeCSVVariance(), outputPath)

        })
    }

    console.log('parsing done')
    // console.log('data_variance', data_variance)
})

/**
 * écrit un csv avec les données de variance du dataset donné
 */
function writeCSVVariance() {

    let csv_content = `;variance de ${inputPath};généré le:;${new Date()};lignes du csv original:;${lines_count};fait avec make_variance_from_csv.ts de Tykayn
`
    let columns = Object.keys(data_variance);

    // add headings
    columns_headings.forEach((heading: string) => {
        csv_content = csv_content + separator + heading
    })
    csv_content = csv_content + '\n'
    // add max length of variance for each column
    let ii = 0
    columns.forEach((column: string) => {
        // console.log('column', column, data_variance[column].length)
        csv_content = csv_content + separator + data_variance[column].length
        ii++
    })

    csv_content = csv_content + '\n\n'
    // add content of values
    for (let ii = 0; ii < longest_variance_count; ii++) {
        csv_content = csv_content + '\n'
        columns.forEach((column: any) => {
            if (ii < data_variance[column].length) {

                let currentValue = data_variance[column][ii]
                csv_content = csv_content + separator + currentValue
            } else {
                csv_content = csv_content + separator
            }
        })
    }

    return csv_content;
}