import.cpp

//**************************************************************************************************
//*  FRIDA: fast reliable interactive data analysis                          
//*  (C) Joachim Wuttke 1990-, v2(C++) 2001-                                 
//*  http://apps.jcns.fz-juelich.de/frida                                    
//**************************************************************************************************

//! \file  import.cpp
//! \brief NImport: import tables or make data ex nihilo

#include <iostream>
#include <string>
#include <vector>

#include <../trivia/string_ops.hpp>
#include <../trivia/file_ops.hpp>
#include <../readplus/ask.hpp>

#include "defs.hpp"
#include "olf.hpp"
#include "mem.hpp"
#include "obj.hpp"
#include "slice.hpp"
#include "import.hpp"


//! Create a new online file from interactive input.

void NImport::read_in()
{
    POld fout;

    // *** set file name and coordinates ***

    static string fnam;
    fout->name = wask("Save as", fnam);

    static CCoord zco;
    fout->xco.ask_and_set( "x" );
    fout->yco.ask_and_set( "y" );
    cout << "if there is only one spectrum, answer \"\\\" to the following:\n";
    zco.ask_and_set( "z" );
    if ( zco.defined() ) fout->ZCo.push_back(zco);

    // *** set input mode ***

    static int mInpX = -1;
    cout << "input modes:\n"
        "  (1) enter x pointwise\n"
        "  (2) set regular x values\n"
        "  (3) enter x-y pairs\n"
        "  (4) select x,y from multicolumn input\n"
        "  (5) select x,y,z from multicolumn input\n";
    mInpX = iask( "Option", mInpX );
    if( mInpX==5 && !zco.defined() )
        throw "need z coordinate for this option";

    static int imcx, imcy, imcz; 
    int mInpY=-1; // ask

    switch(mInpX) {
    case 0: 
        return;
    case 1: case 2:
        break;
    case 3:
        imcx = 0;
        imcy = 1;
        mInpY = 0;
        break;
    case 4: case 5:
        imcx = iask("x from column #", imcx);
        if (imcx<0) return;
        imcy = iask("y from column #", imcy);
        if (imcy<0) return;
        mInpY = 0;
        break;
    default:
        throw "invalid option";
    }

    if (mInpY<0) {
        cout << "input modes:\n"
            "  (1) enter y pointwise\n"
            "  (2) set regular y values\n"
            "  (3) select y from multicolumn input\n";
        static int mInpY_in = -1;
        mInpY_in = iask( "Option", mInpY_in );
        if ( mInpY_in<1 || mInpY_in>3 )
            throw "invalid option";
        mInpY = mInpY_in;
    }

    switch(mInpY) {
    case 0: case 1: case 2:
        break;
    case 3:
        imcy = iask("y from column #", imcy);
        if (imcy<0) return;
        break;
    default:
        throw "PROGRAM ERROR in Make: invalid switch(mInpY)";
    }

    static int mInpZ;
    if        ( mInpX==5 ){
        mInpZ = 3;
    } else if (zco.defined()) {
        cout << "at present, we allow only _one_ z value per spectrum\n";
        cout << "input modes:\n"
            "  (1) enter z per spectrum\n"
            "  (2) select z from multi-z header line\n";
        mInpZ = iask( "Option", mInpZ );
        if ( mInpZ<0 || mInpZ>2 )
            throw "invalid option";
    } else {
        mInpZ = 0;
    }

    switch (mInpZ) {
    case 0: case 1:
        break;
    case 2:
        if (! (mInpX==4 || mInpY==3) )
            throw "invalid combination of options:"
                " y is not read from multicolumn format";
        break;
    case 3:
        imcz = iask( "z0 from column", imcz );
        if( imcz<0 ) return;
        break;
    default:
        throw "PROGRAM ERROR in Make: invalid switch(mInpZ)";
    }

    static int imcy_delta=0; 
    static int imcy_last;
    cout << "DEBUG "<< mInpZ << "\n";
    if (mInpZ==2) {
        imcy_delta = iask("step in number of y column", imcy_delta);
        if (imcy_delta) 
            imcy_last = iask("last y column", imcy_last);
    } else {
        imcy_delta = 0;
    }

    // *** loop over spectra ***

    int n = 0;
    int imcm;
    string quest, line;
    double val, zold=NAN;
    vector<double> linv;

    int ns = 0;
    while(1) {
        PSpec s( new CSpec );

        switch(mInpZ) {
        case 1:
            line = sask("Enter z value", "");
            // cout << "DEBUG: line [" << line << "]\n";
            if (!triv::str2vec(line, &linv, 2) < 0) {
                cout << "-> invalid line [" << line << "]\n";
                goto store;
            }
            // printf ("DEBUG: size = %d\n", linv.size());
            if (linv.size()<1) {
                cout << "found no z in line [" << line <<
                    "] => assuming end-of-input\n";
                goto store;
            }
            // printf ("DEBUG: val = %g\n", linv[0]);
            val = linv[0];
            break;
        case 2:
            line = sask("Enter header line with z values", "");
            if (!triv::str2vec(line, &linv, imcy+1)) {
                cout << "-> bad input\n";
                goto store;
            }
            if (linv.size()<=imcy) {
                cout << "-> not enough entries in header\n";
                goto store;
            }
            val = linv[imcy];
            break;
        }
        if (mInpZ<=2)
            s->z.push_back( PObjDbl( new CObjDbl(val) ) );

        switch(mInpX) {
        case 1:
            cout << "Enter x values (or empty line to exit):\n";
            while (1) {
                quest = "[" + S(n) + "] ";
                line = sask(quest.c_str(), "");
                if (sscanf(line.c_str(), "%lg", &val)!=1) break;
                s->x.push_back(val);
                n++;
            }
            break;
        case 2:
            //S.x.Ask("x");
            //n = S.x.size();
            throw "case 2 broken";
            break;
        case 3: case 4: case 5:
            cout<<"Enter data row (or empty line to exit):\n";
            imcm = (imcx>imcy) ? imcx : imcy;
            if( mInpY==5 && imcz>imcm )
                imcm = imcz;
            while (1) {
                quest = "[" + S(n) + "] ";
                line = sask(quest.c_str(), "");
                if (line==string("")) break;
                if (!triv::str2vec(line, &linv, imcm+1)) {
                    printf(
                        " could not extract value no. %u"
                        " from string [%s]\n", imcm, line.c_str());
                    continue;
                }
                if (linv.size()<=imcm) {
                    cout << "-> not enough entries in line\n";
                    continue;
                }
                if( mInpZ==3 ){
                    if( n==0 ){
                        s->z.push_back( PObjDbl( new CObjDbl(linv[imcz]) ) );
                        zold = linv[imcz];
                    } else if( linv[imcz]!=zold ){
                        fout->V.push_back(s);
                        ns++;
                        s->z.clear();
                        s->z.push_back( PObjDbl( new CObjDbl(linv[imcz]) ) );
                        zold = linv[imcz];
                    }
                }
                s->push_xy(linv[imcx], linv[imcy]);
                n++;
            }
            break;
        }
        if (n<=0) return;

        switch(mInpY) {
        case 0:
            break;
        case 1:
            cout << "Enter " << n << " y values:\n";
            s->y.resize(n);
            for (int i=0; i<n; i++) {
                quest = "x[" + S(i) + "] = " +
                    S(s->x[i]) + " -> ";
                do {
                    line = sask(quest.c_str());
                } while (sscanf(line.c_str(), "%lg", &val)!=1);
                s->y[i] = val;
            }
            break;
        case 2:
            throw "case 2 broken, ygrid undefined";
            //ygrid.Ask("y", n);
            //ygrid.AsVec(&(S.y));
            break;
        case 3:
            cout << "Enter " << n << " lines with y values:\n";
            s->y.resize(n);
            for (int i=0; i<n; i++) {
                quest = "x[" + S(i) + "] =" +
                    S(s->x[i]) + " -> ";
                while (1) {
                    line = sask(quest.c_str());
                    if (!triv::str2vec(line, &linv, imcy+1)) {
                        printf(
                            " could not extract y value no. %u"
                            " from string [%s]\n", imcy, line.c_str());
                        continue;
                    }
                    if (linv.size()<=imcy) {
                        cout << "-> not enough entries"
                            " in line\n";
                        continue;
                    }
                    break;
                }
                s->y[i] = linv[imcy];
            }
            break;
        }
        fout->V.push_back(s);
        ns++;

        if (!mInpZ || mInpZ==3) goto store;
        if (imcy_delta) {
            imcy += imcy_delta;
            if ( ( imcy_delta>0 && imcy>imcy_last ) ||
                 ( imcy_delta<0 && imcy<imcy_last) ) goto store;
        }
    }

    // *** store new on-line file ***
store:
    // cout << "DEBUG: store\n";
    if (ns<=0) return;
    // cout << "DEBUG: sel\n";
    // cout << "DEBUG: add\n";
    NOlm::mem_store( fout );
    // cout << "DEBUG: main\n";
}


//! Create a new online file from table.

void NImport::read_tab( string qualif )
{
    // ** parse qualif **
    if( qualif.find_first_not_of("hvcsmd")!=string::npos )
        throw "ReadTab: invalid qualifier";
    char dir = qualif[0];
    if( !( dir=='h' || dir=='v' || dir=='c' ) )
        throw "ReadTab: missing qualifier h or v";
    bool horizontal = dir=='h';
    bool choosecol = dir=='c';
    static int iycol=0;
    bool fromscript = qualif.find('s')!=string::npos;
    bool multiblock = qualif.find('m')!=string::npos;
    bool with_d = qualif.find('d')!=string::npos;

    // ** input from script or from file **
    static string script;
    vector<string> inFiles;
    if( fromscript ){
        inFiles.push_back( "/ram/tab" );
        script = sask("Script (writing to "+inFiles[0]+")", script);
        triv::system( script );
    } else {
        string fnames = sask( "Read tab from file(s)" );
        triv::glob_file_list( fnames, "", &inFiles );
    }

    if( choosecol ){
        printf( "at present, x:=i\n" );
        iycol = iask( "Read y from column", iycol );
        if( iycol<0 ) return;
    }
    
    FILE *fd;
    string fdir, fshort, fext;
    for( int iF=0; iF<inFiles.size(); ++iF ) {
        if( !(fd = fopen(inFiles[iF].c_str(), "r")) )
            throw "cannot open file " + inFiles[iF];
        cout << ".. reading from " << inFiles[iF] << "\n";
        POld fout( new COld );
        if( fromscript ) {
            fout->lDoc.push_back( "ft"+qualif+" " + script );
            fout->name = script;
        } else {
            triv::fname_divide( inFiles[iF], &fdir, &fshort, &fext);
            fout->lDoc.push_back( "ft"+qualif+" " + inFiles[iF] );
            fout->name = fshort;
            if( choosecol ){
                fout->lDoc.push_back( "y from column " + S(iycol) );
                fout->name += "_" + S(iycol);
            }
        }

        // ** file-level settings **
        if( choosecol ){
            fout->xco = CCoord("i", "");
            fout->yco = CCoord("y"+S(iycol), "");
        } else {
            fout->xco = CCoord("x", "");
            fout->yco = CCoord("y", "");
        }

        // *** read input ***
        string lin, s1, s2;
        vector<double> dat, zdat;
        int n_in = -1; // line number
        int nblock = 0;
        int nline = -1; // line number within block (starting out of block)
        int nz = 0;
        int nzdat = 0;
        double val;
        PSpec s;
        while( triv::freadln(fd, &lin)>-1 ) {
            ++n_in;
            if( lin.substr(0,4)=="#rpa" ){
                triv::string_extract_word( lin.substr(4), &s1, &s2 );
                if( !triv::any2dbl( s2, &val ) )
                    printf( "invalid parameter line [%s]\n", lin.c_str() );
                else
                    fout->RPar.push_back( CParam( s2, "", val) );
                continue;
            }
            if( lin[0]=='#' )
                continue;
            
            if( nline==-1 ) { // start of block 
                if( !horizontal && nblock!=0 && s && s->size()>0 )
                    fout->V.push_back(s);
                if ( multiblock ) {
                    if( !triv::str2vec(lin, &zdat, 0, false) )
                        throw "invalid header line [" + lin +
                            "] (or legitimate break ?)";
                    if( nblock==0 ){ // first header
                        nzdat = zdat.size();
                        nz += nzdat;
                        for( int iz=0; iz<nzdat; ++iz )
                            fout->ZCo.push_back( CCoord("z"+S(iz), "") );
                    } else if( zdat.size() != nzdat )
                        throw "line " + S(n_in) +
                            ": header has length " + S(zdat.size()) +
                            " instead of " + S(nzdat);
                }
// reactivate this in future z-y mode
//                if ( nblock==0 && horizontal ) {
//                    nz += 1; 
//                    fout->ZCo.push_back( CCoord("line", "") );
//                }
                if( !horizontal ){
                    s = PSpec( new CSpec );
                    s->z.resize( nz );
                    for( int iz=0; iz<nzdat; ++iz )
                        s->z[iz] = PObjDbl( new CObjDbl( zdat[iz] ) );
                }
                nline = 0;
                ++nblock;
                if( multiblock ) // this was a header line
                    continue;
            }
            
            if( lin=="" && nline!=-1 ) { // end-of-block
                nline = -1;
                ++nblock;
                continue;
            }
            
            // regular data line
            if( !triv::str2vec(lin, &dat, 0, false) )
                throw "cannot parse line "+S(nline)+" in block "+
                    S(nblock)+" ["+lin+"]";
            if(dat.size()==0)
                throw "no data in line "+S(nline)+" in block "+
                    S(nblock)+" ["+lin+"]";
            
            if( horizontal ) { // new spectrum for every line 
                s = PSpec( new CSpec );
                s->z.resize( nz );
                for( int iz=0; iz<nzdat; ++iz )
                    s->z[iz] = PObjDbl( new CObjDbl( zdat[iz] ) );
// reactivate this in future z-y mode
//                s->z[nz-1] = nline;
                for( int i=0; i<dat.size(); ++i )
                    s->push_xy( (double)i, dat[i] );
                fout->V.push_back(s);
            } else { // vertical
                if( choosecol ){
                    if( iycol>=dat.size() )
                        throw "line "+S(n_in)+" (line "+S(nline)+
                            " in block "+S(nblock)+") ["+lin+
                            "] contains only "+S(dat.size())+" values; "+
                            "cannot read y from col "+S(iycol);
                    s->push_xy( (double)nline, dat[iycol] );
                } else {
                    if( dat.size()!=(with_d?3:2) ){
                        throw "line "+S(n_in)+" (line "+S(nline)+
                            " in block "+S(nblock)+") ["+ lin+
                            "] contains "+S(dat.size())+" values; "+
                            "at present, exactly "+S(with_d?3:2)+
                            " are required";
                    }
                    if (with_d)
                        s->push_xy( dat[0], dat[1] );
                    else
                        s->push_xyd( dat[0], dat[1], dat[2] );
                }
            }
            ++nline;
        } // end of file input loop
        
        if( !horizontal && s && s->size()>0 )
            fout->V.push_back(s);
        
        if( !(fout->nJ()) )
            throw "no input lines";
        if( fout->nJ()==1 ){
            fout->ZCo.clear();
            fout->VS(0)->z.clear();
        }
        NOlm::mem_store( fout );
    }
}


//! Create a new online file containing a regular x,z grid; let y=0.

void NImport::make_grid()
{
    POld fout( new COld );

    static int ni=1, nj=1;
    static string fnam;

    int niIn = iask("Number of points per spectrum", ni);
    if (niIn<1) return;
    ni = niIn;

    int njIn = iask("Number of spectra", nj);
    if (njIn<1) return;
    nj = njIn;

    fnam = "grid"+S(ni);
    if (nj>1) fnam += "x"+S(nj);

    fout->name = wask("Save as", fnam);
    fout->lDoc.push_back( "fm " + S(ni) + " " + S(nj) );

    // *** set coordinates ***
    fout->xco = CCoord("x", "");
    fout->yco = CCoord("y", "");
    if (nj>1)
        fout->ZCo.push_back(CCoord("z", ""));

    // *** loop over spectra ***
    for (int j=0; j<nj; ++j) {
        PSpec s( new CSpec );

        s->x.resize(ni);
        for( int i=0; i<ni; ++i )
            s->x[i] = ((double)i)/ni;
        s->y.clear();
        s->y.resize( ni, 0. );

        if (nj>1) s->z.push_back( PObjInt( new CObjInt(j) ) );

        fout->V.push_back(s);
    }

    NOlm::mem_store( fout );
}