sdljasdlkf

config_manager.py

@@ -7,7 +7,11 @@ class ConfigReader:
         self._config = configparser.ConfigParser()
         self._config.read(path)
         
-        
+    def server(self):
+        return self._config['tcpip']['server']
+    def port(self):
+        return int(self._config['tcpip']['port'])
+    
     def scavroot(self):
         return self._config['scav']['root']
     
@@ -66,6 +70,16 @@ class ConfigReader:
     
     def reg_threshold(self):
         return int(self._config['regimes']['threshold'])
+    def outcl(self):
+        return self._config['clusters']['output']      
+    def outreg(self):
+        return self._config['regimes']['output']
+    def incl(self):
+        return self._config['regimes']['input']
+    def clmap(self):
+        return self._config['graphs']['cmap']
+    def dataref(self):
+        return self._config['graphs']['dataref']
         
 if __name__ == "__main__":
     c = ConfigReader('regimes.ini')

distribs.py

@@ -8,6 +8,8 @@ from sklearn import cluster, covariance, manifold
 from math import *
 import dill
 import sys
+import asyncio
+
 if not sys.warnoptions:
     import warnings
     warnings.simplefilter("ignore")
@@ -64,8 +66,10 @@ class Distributions:
         self.eps=eps
         self._window=window
         self._overlap=overlap
+        self._pr = None
+        self._interrupt = asyncio.Event()
         
-    def compute_gaussians(self,window,overlap):
+    async def compute_gaussians(self,window,overlap):
         i = 0
         r = None
         k = 0
@@ -74,13 +78,13 @@ class Distributions:
         rw=[]
         avdi = np.full((self._N,),0.)
         ilist=[]
-        pr = ProgressBar(self._logret.shape[0]/(window-overlap), prefix='Covariances Computed', suffix='Complete')
+       
         while i < self._logret.shape[0]:
             iend = min(i+window,self._logret.shape[0])                
             chunk = self._logret[i:iend,:]#*100.
             ilist.append((i,iend))
             rw.append(chunk)
-            pr.iterate()
+            self._pr.iterate()
             
             mu = chunk.mean(axis=0)
             model = GraphicalLassoCV(cv=3)
@@ -102,16 +106,20 @@ class Distributions:
             else:
                 break
             
+            await asyncio.sleep(0)
+            if self._interrupt.is_set():
+                return None
         dlist  = DistList( np.arange(ret.shape[0]), rw,ret,ilist)
         return dlist
     
     def load(self,mfile,mfields):
         pd = pandas.read_csv(mfile,sep=',')
-        self._index = pd['Date'].values
+        self._index =pd['Date'].values
         self._logret = pd[mfields].values
         self._N = self._logret.shape[1]
+        self._pr = ProgressBar(self._logret.shape[0]/(self._window-self._overlap), prefix='Covariances Computed', suffix='Complete')
        
-def main():
+def init():
     creader = ConfigReader('regimes.ini')
     mfile = creader.distrmastersource()
     mfields = creader.distrfields()
@@ -119,15 +127,30 @@ def main():
     D.load(mfile, mfields)
     window = creader.distrws()
     overlap = creader.distroverlap()
-    dlist = D.compute_gaussians(window, overlap)
-    dlist._index=D._index
+    return creader,window,overlap, D
     
+async def run(creader,window,overlap,D):
+    dlist = await D.compute_gaussians(window, overlap)
+    if dlist is None:
+        return None
+    dlist._index=D._index
     mcovf = creader.mastercovlist()
-    print()
-    print('list of covariances saved in..')
     with open(mcovf, 'wb') as f:
         dill.dump(dlist,f)    
-    print (mcovf)
-        
+    return mcovf
+    
+async def display(D):
+    pr = D._pr
+    while pr._iteration < pr._total:
+        await asyncio.sleep(1)
+        print (pr._display())
+    
 if __name__ == "__main__":
-    main()
+    creader,window,overlap, D = init()
\ No newline at end of file
+    loop = asyncio.get_event_loop()
+    main = asyncio.ensure_future (run(creader, window, overlap, D))
+    prg =  asyncio.ensure_future (display(D))
+    loop.run_until_complete(asyncio.gather(main,prg))
+    
+   
+    
\ No newline at end of file

makeclusters.py

@@ -12,9 +12,9 @@ import matplotlib.pyplot as plt
 from matplotlib.pyplot import cm
 from sklearn import cluster, covariance, manifold
 from tabulate import tabulate
-
-
-
+import asyncio
+from objects import ObjectContainer
+import pandas
 class Clusters:
     
     def __init__(self,conf):
@@ -22,10 +22,18 @@ class Clusters:
         self._NCLUSTERS = conf.clusterN()
         self._THRESHOLD = 1
         distfile = conf.mastercovlist()
+        
         with open(distfile,'rb') as f:
             self._distlist = dill.load(f)
-    
-   
+        
+        index = self._distlist._index
+        index = np.array(pandas.to_datetime(index)).astype('datetime64[D]')
+        index.sort()
+        self._distlist._index = index
+        covs = self._distlist._covlist  
+        
+        self._pr = ProgressBar(covs.shape[0]*covs.shape[0], prefix='Hellinger Affinities', suffix='Completed')
+        
     def spectral_centroids(self, labels):
         centroids=[]
         for i in range(labels.max()+1):
@@ -56,7 +64,7 @@ class Clusters:
                 continue
             lxy=centroids[i][0]
             labelx,labely = lxy[0],lxy[1]
-            textx = labelx + 0.004
+            textx = labelx + 0.001
             texty = labely + 0.01
             plt.annotate(str(col),xy=(labelx,labely),xytext=(textx,texty), 
                          arrowprops=dict(arrowstyle="->",connectionstyle="arc3"),size=6)
@@ -69,9 +77,6 @@ class Clusters:
         plt.savefig('clusters')
         plt.close(fig)
         
-   
-        
-    
     def compute_clusters(self): 
         nn = self._conf.clspectral_neighbors()
         node_position_model = manifold.SpectralEmbedding(n_components=2,n_neighbors=nn,random_state=42)
@@ -85,39 +90,46 @@ class Clusters:
         es = self._conf.cleigen_solver()
         self._agg = cluster.SpectralClustering(n_clusters=N,affinity=aff,eigen_solver=es,n_neighbors=nn,random_state=rs).fit(X)
         self._clabels=self._agg.labels_
-        with open('clabels.plk','wb') as f:
-            dill.dump(self._clabels,f)
-        with open('clusterengine.plk','wb') as f:
-            dill.dump(self._agg,f)
         self._centroids = self.spectral_centroids(self._clabels)          
   
-    def cal_affinity_matrix(self):
-        if (os.path.isfile('hellinger.plk')):
-            self._hellingerd = dill.load(open('hellinger.plk','rb'))
-            return
-        cind = self._distlist._covindex
-        covs = self._distlist._covlist        
+   
+        
+class AsyncClusters:
+    def __init__(self, conf):
+        self._clusters = Clusters(conf)
+        self._interrupt = asyncio.Event()
+        self._pr = self._clusters._pr
+    def compute_clusters(self): 
+        self._clusters.compute_clusters()
+        
+    async def cal_affinity_matrix(self):
+        cind = self._clusters._distlist._covindex
+        covs =  self._clusters ._distlist._covlist        
         hellingerd = np.full((covs.shape[0],covs.shape[0]),0.)
         n = covs.shape[1]
-        br = ProgressBar(covs.shape[0]*covs.shape[0], prefix='Hellinger Affinities', suffix='Completed')
         for u in cind:
+            await asyncio.sleep(0)
             for v in cind:
-                br.iterate()
+                if self._interrupt.is_set():
+                    return None
+                self._pr.iterate()
                 if hellingerd[v,u] == 0:
-                    hellingerd[u,v] = self._distlist.hellinger(n,covs[u],covs[v],self._conf.smooth_hellinger())
+                    hellingerd[u,v] =  self._clusters._distlist.hellinger(n,covs[u],covs[v], self._clusters._conf.smooth_hellinger())
                 else:
                     hellingerd[u,v] = hellingerd[v,u]
-       
-        self._hellingerd = hellingerd
-        with open('hellinger.plk','wb') as f:
-            dill.dump(hellingerd,f)
-       
- 
+
+        self._clusters._hellingerd = hellingerd 
+        
+        
+        
+        
 class Regimes:
     def __init__(self, clusters):
         self._clusters = clusters
         self._reg_threshold = clusters._conf.reg_threshold()
-    
+        self._conf = clusters._conf
+        
+        
     def printout_centroids(self,prefix):
         #distances
         D = np.full((len(self._rcentroids),len(self._rcentroids)),0.)
@@ -135,7 +147,7 @@ class Regimes:
             rows.append(row)
         headers=[prefix + '#%d'%r for r in range(len(self._rcentroids))]
         table = tabulate(rows,headers= headers,tablefmt='orgtbl',floatfmt='.3f')
-        print (table)    
+        return table
         
     def merge_regimes(self,s,t):
         idx = np.where(self._regimes==s)
@@ -231,16 +243,54 @@ class Regimes:
         #cl._dt2rg()        
         
         
-if __name__ == "__main__":
+
+def init_clusters():
     conf = ConfigReader('regimes.ini')
-    cl = Clusters(conf)
-    cl.cal_affinity_matrix()
+    cl = AsyncClusters(conf)
+    return cl
+
+async def run_clusters(cl):
+    await cl.cal_affinity_matrix()
     cl.compute_clusters()
+    return 
+
+def save_clusters(cl):
+    ObjectContainer.getInstance().set('clusters',cl._clusters)
+    ObjectContainer.getInstance().set('hellinger',cl._clusters._hellingerd)
+    ObjectContainer.getInstance().save()
+     
+def init_regimes():
+    conf = ConfigReader('regimes.ini')
+    cl = ObjectContainer.getInstance().get('clusters')
     r = Regimes(cl)
+    return r
+   
+async def run_regimes(r):
     r.compute_inter()
     r.label_regimes()
     r.printout_centroids('regimes')
     r.filter_small()
+    return r.printout_centroids('regimes')
+    
+    
+async def display(cl):
+    pr = cl._pr
+    while pr._iteration < pr._total:
+        await asyncio.sleep(1)
+        print (pr._display())   
+        
+        
+if __name__ == "__main__":
+    
+    c = init_clusters()
+    run_clusters(c)
+    
+    r = Regimes(cl)
+    r.compute_inter()
+    r.label_regimes()
+    r.printout_centroids('clusters')
+    r.filter_small()
     r.printout_centroids('regimes')
-    r._clusters.graph_(r._rcentroids,r._regimes,'regimes')
-    
+    
+    with open(r._conf.outreg(),'wb') as f:
\ No newline at end of file
+        dill.dump(r,f)    
\ No newline at end of file

regimes.ini

+[tcpip]
+server2=localhost
+server=192.168.0.200
+port=4444
 [scav] 
 root=/home/eric/src/Data/axioma
 datefrom=1982-01-01
@@ -12,7 +16,7 @@ mastersource=RFreturns.csv
 fields=ComEnOil,ComMeGol,CurEUR,CurJPY,EqCHN,EqEME,EqGrVaUSA,EqJPN,EqLaSmUSA,EqSeFinUSA,EqSeITeUSA,EqUSA,EqWLD,FICrHAAAUSA,FICrHBAAUSA,FICreHyUSA,FIG10YUSA,FIG3MUSA
 windowsize=60
 overlap=40
-mastercovlist=mastercovlist.pkl
+mastercovlist=mastercovlist.plk
 
 [clusters]
 N=8
@@ -22,6 +26,13 @@ spectraln_neighbors=6
 n_neighbors=10
 random_states=48
 smooth hellinger=True
+output=clusters.plk
 
 [regimes]
-threshold = 4
+input=clusters.plk
\ No newline at end of file
+threshold = 10
+output=regimes.plk
+
+[graphs]
+cmap = cm.gnuplot2
+dataref = CloseFactors.csv

utils.py

@@ -14,18 +14,20 @@ class ProgressBar:
         self._percent=0
     
     def _display(self):
+       
         self._percent = ("{0:." + str(self._decimals) + "f}").format(100 * (self._iteration / float(self._total)))
         filledLength = int(self._length * self._iteration // self._total)
         bar = self._fill * filledLength + '-' * (self._length - filledLength)
-        print('\r%s |%s| %s%% %s' % (self._prefix, bar, self._percent, self._suffix), end = '\r')
+        txt = '\r%s |%s| %s%% %s' % (self._prefix, bar, self._percent, self._suffix) + '\r'
         # Print New Line on Complete
         if self._iteration == self._total: 
-            print() 
+            return txt + '\n'
+        else:
+            return txt
             
     def iterate(self, inc=1):
-        sleep(0)
         self._iteration += inc
-        self._display()
+        #self._display()
         
     
 if __name__ == "__main__":