#!/usr/bin/python3  -O
import glob
import os
from datetime import datetime
import numpy as np
import sys
import csv
from config_manager import ConfigReader

_creader = ConfigReader('regimes.ini')
_currdir='.'

def browse(): 
    global _creader,_currdir
    _currdir = os.getcwd()
    os.chdir(_creader.scavroot())
    suff = '.' + _creader.scavsuffix()
    pref = _creader.scavformat()
    f,t = _creader.dates()
    for dt in np.arange(f, t,  dtype='datetime64[M]'):
        pdir=np.datetime_as_string(dt,unit='M').replace('-','/')
        if os.path.exists(pdir):
            for file in os.listdir(pdir):
                if file.endswith(suff) and file.startswith(pref):
                    yield os.path.join(pdir,file)

def create_csv(header):
    ofile = os.path.join(_currdir,_creader.scavout())
    fcsv = open(ofile,'w')
    print(', '.join(header),file=fcsv, flush=True)
    return fcsv

def build_output():
    first = True
    rows = None
    print ('processing files ..')
    lfield = _creader.scavlistfield().split(',')
    for  i, f in enumerate(browse()):
        with open(f,'r') as fd:
            row = []
            header = ['Date']
            print('\r%2d'%i,flush=True,end='')
            for lines in fd.readlines():
                if lines.startswith('#'):
                    if lines.startswith('#DataDate'):
                        asof = np.datetime64(lines.split(':')[1].strip())
                        row = [asof]
                    continue            

                a = lines.split('|')
                if (len(lfield) > 0 and a[0] in lfield ) or lfield is None:
                    row.append(float(a[1])) 
                    header.append(a[0])
            if first:
                fcsv = create_csv(header)
                first=False 
            if rows is None:
                rows = np.array(row)
            else:
                rows = np.vstack((rows,row))
    rows = rows.T
    dsort=np.argsort(rows[0,:])
    print ('done')
    print ('flushing')
    for ii, i in enumerate(np.argsort(rows[0,:])):
        print('\r%2d'%i,flush=True,end='')
        print(np.datetime_as_string(rows[0,i],unit='D'),',',', '.join(map(str,rows[1:,i])),file=fcsv)
    print('done')
    fcsv.close()
        

if __name__ == "__main__":
    build_output()
#t,d,T = get_factors([1,2,3],datefrom='2007-01-31')
#print (T)
