gavindiaz/include
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
e829dab822196c3e26d04051f17a2e412db4a03a
include/Math/Alglib/dataanalysis.mqh
T
amirghadiri1987 56b382609f Add files via upload
2025-02-07 19:12:13 +03:30

26388 lines
1.1 MiB
Plaintext
Raw Blame History

//+------------------------------------------------------------------+
//| dataanalysis.mqh |
//| Copyright 2003-2022 Sergey Bochkanov (ALGLIB project) |
//| Copyright 2012-2023, MetaQuotes Ltd. |
//| https://www.mql5.com |
//+------------------------------------------------------------------+
//| Implementation of ALGLIB library in MetaQuotes Language 5 |
//| |
//| The features of the library include: |
//| - Linear algebra (direct algorithms, EVD, SVD) |
//| - Solving systems of linear and non-linear equations |
//| - Interpolation |
//| - Optimization |
//| - FFT (Fast Fourier Transform) |
//| - Numerical integration |
//| - Linear and nonlinear least-squares fitting |
//| - Ordinary differential equations |
//| - Computation of special functions |
//| - Descriptive statistics and hypothesis testing |
//| - Data analysis - classification, regression |
//| - Implementing linear algebra algorithms, interpolation, etc. |
//| in high-precision arithmetic (using MPFR) |
//| |
//| This file is free software; you can redistribute it and/or |
//| modify it under the terms of the GNU General Public License as |
//| published by the Free Software Foundation (www.fsf.org); either |
//| version 2 of the License, or (at your option) any later version. |
//| |
//| This program is distributed in the hope that it will be useful, |
//| but WITHOUT ANY WARRANTY; without even the implied warranty of |
//| MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
//| GNU General Public License for more details. |
//+------------------------------------------------------------------+
#include "ap.mqh"
#include "optimization.mqh"
#include "statistics.mqh"
#include "solvers.mqh"
//+------------------------------------------------------------------+
//| Auxiliary class for CBdSS |
//+------------------------------------------------------------------+
struct CCVReport
{
double m_RelCLSError;
double m_AvgCE;
double m_RMSError;
double m_AvgError;
double m_AvgRelError;
//---
CCVReport(void) { ZeroMemory(this); }
~CCVReport(void) {}
//---
void Copy(const CCVReport &obj);
//--- overloading
void operator=(const CCVReport &obj) { Copy(obj); }
};
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CCVReport::Copy(const CCVReport &obj)
{
m_RelCLSError=obj.m_RelCLSError;
m_AvgCE=obj.m_AvgCE;
m_RMSError=obj.m_RMSError;
m_AvgError=obj.m_AvgError;
m_AvgRelError=obj.m_AvgRelError;
}
//+------------------------------------------------------------------+
//| Data analysis |
//+------------------------------------------------------------------+
class CBdSS
{
public:
static void DSErrAllocate(const int nclasses,double &buf[]);
static void DSErrAllocate(const int nclasses,CRowDouble &buf);
static void DSErrAccumulate(double &buf[],double &y[],double &desiredy[]);
static void DSErrAccumulate(CRowDouble &buf,CRowDouble &y,CRowDouble &desiredy);
static void DSErrFinish(double &buf[]);
static void DSErrFinish(CRowDouble &buf);
static void DSNormalize(CMatrixDouble &xy,const int npoints,const int nvars,int &info,double &means[],double &sigmas[]);
static void DSNormalize(CMatrixDouble &xy,const int npoints,const int nvars,int &info,CRowDouble &means,CRowDouble &sigmas);
static void DSNormalizeC(CMatrixDouble &xy,const int npoints,const int nvars,int &info,double &means[],double &sigmas[]);
static void DSNormalizeC(CMatrixDouble &xy,const int npoints,const int nvars,int &info,CRowDouble &means,CRowDouble &sigmas);
static double DSGetMeanMindIstance(CMatrixDouble &xy,const int npoints,const int nvars);
static void DSTie(double &a[],const int n,int &ties[],int &tiecount,int &p1[],int &p2[]);
static void DSTie(CRowDouble &a,const int n,CRowInt &ties,int &tiecount,CRowInt &p1,CRowInt &p2);
static void DSTieFastI(double &a[],int &b[],const int n,int &ties[],int &tiecount,double &bufr[],int &bufi[]);
static void DSTieFastI(CRowDouble &a,CRowInt &b,const int n,CRowInt &ties,int &tiecount,CRowDouble &bufr,CRowInt &bufi);
static void DSOptimalSplit2(double &ca[],int &cc[],const int n,int &info,double &threshold,double &pal,double &pbl,double &par,double &pbr,double &cve);
static void DSOptimalSplit2(CRowDouble &ca,CRowInt &cc,const int n,int &info,double &threshold,double &pal,double &pbl,double &par,double &pbr,double &cve);
static void DSOptimalSplit2Fast(double &a[],int &c[],int &tiesbuf[],int &cntbuf[],double &bufr[],int &bufi[],const int n,const int nc,double alpha,int &info,double &threshold,double &rms,double &cvrms);
static void DSOptimalSplit2Fast(CRowDouble &a,CRowInt &c,CRowInt &tiesbuf,CRowInt &cntbuf,CRowDouble &bufr,CRowInt &bufi,const int n,const int nc,double alpha,int &info,double &threshold,double &rms,double &cvrms);
static void DSSplitK(double &ca[],int &cc[],const int n,const int nc,int kmax,int &info,double &thresholds[],int &ni,double &cve);
static void DSSplitK(CRowDouble &ca,CRowInt &cc,const int n,const int nc,int kmax,int &info,CRowDouble &thresholds,int &ni,double &cve);
static void DSOptimalSplitK(double &ca[],int &cc[],const int n,const int nc,int kmax,int &info,double &thresholds[],int &ni,double &cve);
static void DSOptimalSplitK(CRowDouble &ca,CRowInt &cc,const int n,const int nc,int kmax,int &info,CRowDouble &thresholds,int &ni,double &cve);
private:
static double XLnY(const double x,const double y);
static double GetCV(CRowInt &cnt,const int nc);
static void TieAddC(CRowInt &c,CRowInt &ties,const int ntie,const int nc,CRowInt &cnt);
static void TieSubC(CRowInt &c,CRowInt &ties,const int ntie,const int nc,CRowInt &cnt);
};
//+------------------------------------------------------------------+
//| This set of routines (DSErrAllocate, DSErrAccumulate, |
//| DSErrFinish) calculates different error functions (classification|
//| error, cross-entropy, rms, avg, avg.rel errors). |
//| 1. DSErrAllocate prepares buffer. |
//| 2. DSErrAccumulate accumulates individual errors: |
//| * Y contains predicted output (posterior probabilities for |
//| classification) |
//| * DesiredY contains desired output (class number for |
//| classification) |
//| 3. DSErrFinish outputs results: |
//| * Buf[0] contains relative classification error (zero for |
//| regression tasks) |
//| * Buf[1] contains avg. cross-entropy (zero for regression |
//| tasks) |
//| * Buf[2] contains rms error (regression, classification) |
//| * Buf[3] contains average error (regression, classification) |
//| * Buf[4] contains average relative error (regression, |
//| classification) |
//| NOTES(1): |
//| "NClasses>0" means that we have classification task. |
//| "NClasses<0" means regression task with -NClasses real |
//| outputs. |
//| NOTES(2): |
//| rms. avg, avg.rel errors for classification tasks are |
//| interpreted as errors in posterior probabilities with |
//| respect to probabilities given by training/test set. |
//+------------------------------------------------------------------+
void CBdSS::DSErrAllocate(const int nclasses,double &buf[])
{
CRowDouble Buf;
DSErrAllocate(nclasses,Buf);
Buf.ToArray(buf);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CBdSS::DSErrAllocate(const int nclasses,CRowDouble &buf)
{
//--- allocation
buf=vector<double>::Zeros(8);
//--- initialization
buf.Set(5,(double)nclasses);
}
//+------------------------------------------------------------------+
//| See DSErrAllocate for comments on this routine. |
//+------------------------------------------------------------------+
void CBdSS::DSErrAccumulate(double &buf[],double &y[],double &desiredy[])
{
CRowDouble Buf=buf;
CRowDouble Y=y;
CRowDouble DesiredY=desiredy;
DSErrAccumulate(Buf,Y,DesiredY);
Buf.ToArray(buf);
}
//+------------------------------------------------------------------+
//| See DSErrAllocate for comments on this routine. |
//+------------------------------------------------------------------+
void CBdSS::DSErrAccumulate(CRowDouble &buf,CRowDouble &y,CRowDouble &desiredy)
{
//--- create variables
int offs=5;
int nclasses=(int)MathRound(buf[offs]);
int nout=0;
int mmax=0;
int rmax=0;
int j=0;
double v=0;
//--- initialization
vector<double> ev=vector<double>::Zeros(MathAbs(nclasses));
//--- check
if(nclasses>0)
{
//--- Classification
rmax=(int)MathRound(desiredy[0]);
//--- initialization
mmax=(int)y.ArgMax();
//--- check
if(mmax!=rmax)
buf.Add(0,1);
//--- check
if(y[rmax]>0.0)
buf.Add(1,-MathLog(y[rmax]));
else
buf.Add(1,MathLog(CMath::m_maxrealnumber));
//--- calculation
if(rmax>=0 && rmax<nclasses)
{
buf.Add(4,MathAbs(1-y[rmax]));
buf.Add(offs+2,1);
ev[rmax]=1;
}
vector<double> err=y.ToVector()-ev;
buf.Add(2,MathPow(err,2.0).Sum());
buf.Add(3,MathAbs(err).Sum());
//--- change value
buf.Add(offs+1,1);
}
else
{
//--- Regression
nout=-nclasses;
//--- initialization
rmax=(int)desiredy.ArgMax();
//--- initialization
mmax=(int)y.ArgMax();
//--- check
if(mmax!=rmax)
buf.Add(0,1);
//--- calculation
ev=desiredy.ToVector();
vector<double> err=y.ToVector()-ev;
buf.Add(2,MathPow(err,2.0).Sum());
buf.Add(3,MathAbs(err).Sum());
for(j=0; j<nout; j++)
{
//--- check
if(ev[j]!=0.0)
{
buf.Add(4,MathAbs(err[j]/ev[j]));
buf.Add(offs+2,1);
}
}
//--- change value
buf.Add(offs+1,1);
}
}
//+------------------------------------------------------------------+
//| See DSErrAllocate for comments on this routine. |
//+------------------------------------------------------------------+
void CBdSS::DSErrFinish(double &buf[])
{
CRowDouble Buf=buf;
DSErrFinish(Buf);
Buf.ToArray(buf);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CBdSS::DSErrFinish(CRowDouble &buf)
{
//--- create variables
int offs=5;
int nout=MathAbs((int)MathRound(buf[offs]));
//--- check
if(buf[offs+1]!=0.0)
{
//--- change values
buf.Mul(0,1.0/buf[offs+1]);
buf.Mul(1,1.0/buf[offs+1]);
buf.Set(2,MathSqrt(buf[2]/(nout*buf[offs+1])));
buf.Mul(3,1.0/(nout*buf[offs+1]));
}
//--- check
if(buf[offs+2]!=0.0)
buf.Mul(4,1.0/buf[offs+2]);
}
//+------------------------------------------------------------------+
//| Normalize |
//+------------------------------------------------------------------+
void CBdSS::DSNormalize(CMatrixDouble &xy,const int npoints,
const int nvars,int &info,double &means[],
double &sigmas[])
{
CRowDouble Means,Sigmas;
DSNormalize(xy,npoints,nvars,info,Means,Sigmas);
Means.ToArray(means);
Sigmas.ToArray(sigmas);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CBdSS::DSNormalize(CMatrixDouble &xy,const int npoints,
const int nvars,int &info,CRowDouble &means,
CRowDouble &sigmas)
{
//--- function call
DSNormalizeC(xy,npoints,nvars,info,means,sigmas);
//--- calculation
for(int j=0; j<nvars; j++)
{
//--- change values
for(int i=0; i<npoints; i++)
xy.Set(i,j,(xy.Get(i,j)-means[j])/sigmas[j]);
}
}
//+------------------------------------------------------------------+
//| Normalize |
//+------------------------------------------------------------------+
void CBdSS::DSNormalizeC(CMatrixDouble &xy,const int npoints,
const int nvars,int &info,double &means[],
double &sigmas[])
{
CRowDouble Means,Sigmas;
DSNormalizeC(xy,npoints,nvars,info,Means,Sigmas);
Means.ToArray(means);
Sigmas.ToArray(sigmas);
}
//+------------------------------------------------------------------+
//| Normalize |
//+------------------------------------------------------------------+
void CBdSS::DSNormalizeC(CMatrixDouble &xy,const int npoints,
const int nvars,int &info,CRowDouble &means,
CRowDouble &sigmas)
{
//--- initialization
info=0;
//--- Test parameters
if(npoints<=0 || nvars<1)
{
info=-1;
return;
}
//--- copy
matrix<double> tmp=xy.ToMatrix();
tmp.Resize(npoints,nvars);
//--- change value
info=1;
//--- Standartization
means=tmp.Mean(0);
sigmas=tmp.Std(0);
//--- calculation
for(int j=0; j<nvars; j++)
if(sigmas[j]==0.0)
sigmas.Set(j,1);
}
//+------------------------------------------------------------------+
//| Method |
//+------------------------------------------------------------------+
double CBdSS::DSGetMeanMindIstance(CMatrixDouble &xy,const int npoints,
const int nvars)
{
//--- create variables
double result=0;
double v=0;
//--- creating arrays
vector<double> tmp;
vector<double> tmp2;
//--- Test parameters
if(npoints<=0 || nvars<1)
return(0);
//--- Process
tmp=vector<double>::Full(npoints,CMath::m_maxrealnumber);
//--- allocation
for(int i=0; i<npoints; i++)
{
for(int j=i+1; j<npoints; j++)
{
//--- calculation
tmp2=xy[i];
tmp2.Resize(nvars);
for(int i_=0; i_<nvars; i_++)
tmp2[i_]-=xy.Get(j,i_);
v=MathSqrt(MathPow(tmp2,2).Sum());
//--- change values
tmp[i]=MathMin(tmp[i],v);
tmp[j]=MathMin(tmp[j],v);
}
}
//--- get result
result=tmp.Sum()/npoints;
//--- return result
return(result);
}
//+------------------------------------------------------------------+
//| Method |
//+------------------------------------------------------------------+
void CBdSS::DSTie(double &a[],const int n,int &ties[],int &tiecount,
int &p1[],int &p2[])
{
CRowDouble A=a;
CRowInt Ties=ties;
CRowInt P1=p1;
CRowInt P2=p2;
DSTie(A,n,Ties,tiecount,P1,P2);
A.ToArray(a);
Ties.ToArray(ties);
P1.ToArray(p1);
P2.ToArray(p2);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CBdSS::DSTie(CRowDouble &a,const int n,CRowInt &ties,int &tiecount,
CRowInt &p1,CRowInt &p2)
{
//--- initialization
tiecount=0;
//--- Special case
if(n<=0)
{
tiecount=0;
return;
}
//--- Sort A
CTSort::TagSort(a,n,p1,p2);
//--- Process ties
//--- allocation
ties.Resize(n+1);
//--- change values
ties.Set(0,0);
tiecount=1;
for(int i=1; i<n; i++)
{
//--- check
if(a[i]!=a[i-1])
{
ties.Set(tiecount,i);
tiecount++;
}
}
//--- change value
ties.Set(tiecount,n);
ties.Resize(tiecount+1);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CBdSS::DSTieFastI(double &a[],int &b[],const int n,int &ties[],
int &tiecount,double &bufr[],int &bufi[])
{
CRowDouble A=a;
CRowInt B=b;
CRowInt Ties=ties;
CRowDouble BufR=bufr;
CRowInt BufI=bufi;
DSTieFastI(A,B,n,Ties,tiecount,BufR,BufI);
A.ToArray(a);
B.ToArray(b);
Ties.ToArray(ties);
BufR.ToArray(bufr);
BufI.ToArray(bufi);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CBdSS::DSTieFastI(CRowDouble &a,CRowInt &b,const int n,CRowInt &ties,
int &tiecount,CRowDouble &bufr,CRowInt &bufi)
{
//--- Special case
if(n<=0)
{
tiecount=0;
return;
}
//--- Sort A
CTSort::TagSortFastI(a,b,bufr,bufi,n);
//--- Process ties
ties.Set(0,0);
int k=1;
//--- calculation
for(int i=1; i<n; i++)
{
//--- check
if(a[i]!=a[i-1])
{
ties.Set(k,i);
k++;
}
}
//--- change values
ties.Set(k,n);
tiecount=k;
}
//+------------------------------------------------------------------+
//| Optimal binary classification |
//| Algorithms finds optimal (=with minimal cross-entropy) binary |
//| partition. |
//| Internal subroutine. |
//| INPUT PARAMETERS: |
//| A - array[0..N-1], variable |
//| C - array[0..N-1], class numbers (0 or 1). |
//| N - array size |
//| OUTPUT PARAMETERS: |
//| Info - completetion code: |
//| * -3, all values of A[] are same (partition is |
//| impossible) |
//| * -2, one of C[] is incorrect (<0, >1) |
//| * -1, incorrect pararemets were passed (N<=0). |
//| * 1, OK |
//| Threshold- partiton boundary. Left part contains values |
//| which are strictly less than Threshold. Right |
//| part contains values which are greater than or |
//| equal to Threshold. |
//| PAL, PBL- probabilities P(0|v<Threshold) and |
//| P(1|v<Threshold) |
//| PAR, PBR- probabilities P(0|v>=Threshold) and |
//| P(1|v>=Threshold) |
//| CVE - cross-validation estimate of cross-entropy |
//+------------------------------------------------------------------+
void CBdSS::DSOptimalSplit2(double &ca[],int &cc[],const int n,
int &info,double &threshold,double &pal,
double &pbl,double &par,double &pbr,
double &cve)
{
CRowDouble A=ca;
CRowInt C=cc;
DSOptimalSplit2(A,C,n,info,threshold,pal,pbl,par,pbr,cve);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CBdSS::DSOptimalSplit2(CRowDouble &ca,CRowInt &cc,const int n,
int &info,double &threshold,double &pal,
double &pbl,double &par,double &pbr,
double &cve)
{
//--- create variables
int i=0;
int t=0;
double s=0;
int tiecount=0;
int k=0;
int koptimal=0;
double pak=0;
double pbk=0;
double cvoptimal=0;
double cv=0;
//--- creating arrays
CRowInt ties;
CRowInt p1;
CRowInt p2;
//--- copy
CRowDouble a=ca;
CRowInt c=cc;
//--- initialization
info=0;
threshold=0;
pal=0;
pbl=0;
par=0;
pbr=0;
cve=0;
//--- Test for errors in inputs
if(n<=0)
{
info=-1;
return;
}
for(i=0; i<n; i++)
{
//--- check
if(c[i]!=0 && c[i]!=1)
{
info=-2;
return;
}
}
//--- change value
info=1;
//--- Tie
DSTie(a,n,ties,tiecount,p1,p2);
//--- swap
for(i=0; i<n; i++)
{
//--- check
if(p2[i]!=i)
c.Swap(i,p2[i]);
}
//--- Special case: number of ties is 1.
//--- NOTE: we assume that P.Get(i,j) equals to 0 or 1,
//--- intermediate values are not allowed.
if(tiecount==1)
{
info=-3;
return;
}
//--- General case,number of ties > 1
//--- NOTE: we assume that P.Get(i,j) equals to 0 or 1,
//--- intermediate values are not allowed.
pal=0;
pbl=0;
par=0;
pbr=0;
for(i=0; i<n; i++)
{
//--- check
if(c[i]==0)
par=par+1;
//--- check
if(c[i]==1)
pbr=pbr+1;
}
//--- change values
koptimal=-1;
cvoptimal=CMath::m_maxrealnumber;
for(k=0; k<tiecount-1; k++)
{
//--- first,obtain information about K-th tie which is
//--- moved from R-part to L-part
pak=0;
pbk=0;
for(i=ties[k]; i<ties[k+1]; i++)
{
//--- check
if(c[i]==0)
pak=pak+1;
//--- check
if(c[i]==1)
pbk=pbk+1;
}
//--- Calculate cross-validation CE
cv=0;
cv=cv-XLnY(pal+pak,(pal+pak)/(pal+pak+pbl+pbk+1));
cv=cv-XLnY(pbl+pbk,(pbl+pbk)/(pal+pak+1+pbl+pbk));
cv=cv-XLnY(par-pak,(par-pak)/(par-pak+pbr-pbk+1));
cv=cv-XLnY(pbr-pbk,(pbr-pbk)/(par-pak+1+pbr-pbk));
//--- Compare with best
if(cv<cvoptimal)
{
cvoptimal=cv;
koptimal=k;
}
//--- update
pal+=pak;
pbl+=pbk;
par-=pak;
pbr-=pbk;
}
//--- change values
cve=cvoptimal;
threshold=0.5*(a[ties[koptimal]]+a[ties[koptimal+1]]);
pal=0;
pbl=0;
par=0;
pbr=0;
for(i=0; i<n; i++)
{
//--- check
if(a[i]<threshold)
{
//--- check
if(c[i]==0)
pal=pal+1;
else
pbl=pbl+1;
}
else
{
//--- check
if(c[i]==0)
par=par+1;
else
pbr=pbr+1;
}
}
//--- change values
s=pal+pbl;
pal=pal/s;
pbl=pbl/s;
s=par+pbr;
par=par/s;
pbr=pbr/s;
}
//+------------------------------------------------------------------+
//| Optimal partition, internal subroutine. Fast version. |
//| Accepts: |
//| A array[0..N-1] array of attributes array[0..N-1]|
//| C array[0..N-1] array of class labels |
//| TiesBuf array[0..N] temporaries (ties) |
//| CntBuf array[0..2*NC-1] temporaries (counts) |
//| Alpha centering factor (0<=alpha<=1, |
//| recommended value - 0.05) |
//| BufR array[0..N-1] temporaries |
//| BufI array[0..N-1] temporaries |
//| Output: |
//| Info error code (">0"=OK, "<0"=bad) |
//| RMS training set RMS error |
//| CVRMS leave-one-out RMS error |
//| Note: |
//| content of all arrays is changed by subroutine; |
//| it doesn't allocate temporaries. |
//+------------------------------------------------------------------+
void CBdSS::DSOptimalSplit2Fast(double &a[],int &c[],int &tiesbuf[],
int &cntbuf[],double &bufr[],int &bufi[],
const int n,const int nc,double alpha,
int &info,double &threshold,
double &rms,double &cvrms)
{
CRowDouble A=a;
CRowInt C=c;
CRowInt TiesBuf=tiesbuf;
CRowInt CntBuf=cntbuf;
CRowDouble BufR=bufr;
CRowInt BufI=bufi;
DSOptimalSplit2Fast(A,C,TiesBuf,CntBuf,BufR,BufI,n,nc,alpha,info,threshold,rms,cvrms);
A.ToArray(a);
C.ToArray(c);
TiesBuf.ToArray(tiesbuf);
CntBuf.ToArray(cntbuf);
BufR.ToArray(bufr);
BufI.ToArray(bufi);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CBdSS::DSOptimalSplit2Fast(CRowDouble &a,CRowInt &c,CRowInt &tiesbuf,
CRowInt &cntbuf,CRowDouble &bufr,CRowInt &bufi,
const int n,const int nc,double alpha,
int &info,double &threshold,
double &rms,double &cvrms)
{
//--- create variables
int i=0;
int k=0;
int cl=0;
int tiecount=0;
double cbest=0;
double cc=0;
int koptimal=0;
int sl=0;
int sr=0;
double v=0;
double w=0;
double x=0;
//--- initialization
info=0;
threshold=0;
rms=0;
cvrms=0;
//--- Test for errors in inputs
if(n<=0 || nc<2)
{
info=-1;
return;
}
for(i=0; i<n; i++)
{
//--- check
if(c[i]<0 || c[i]>=nc)
{
info=-2;
return;
}
}
//--- change value
info=1;
//--- Tie
DSTieFastI(a,c,n,tiesbuf,tiecount,bufr,bufi);
//--- Special case: number of ties is 1.
if(tiecount==1)
{
info=-3;
return;
}
//--- General case,number of ties > 1
cntbuf.Fill(0,0,2*nc);
for(i=0; i<n; i++)
cntbuf.Add(nc+c[i],1);
//--- change values
koptimal=-1;
threshold=a[n-1];
cbest=CMath::m_maxrealnumber;
sl=0;
sr=n;
//--- calculation
for(k=0; k<tiecount-1; k++)
{
//--- first,move Kth tie from right to left
for(i=tiesbuf[k]; i<tiesbuf[k+1]; i++)
{
cl=c[i];
cntbuf.Add(cl,1);
cntbuf.Add(nc+cl,-1);
}
sl+=(tiesbuf[k+1]-tiesbuf[k]);
sr-=(tiesbuf[k+1]-tiesbuf[k]);
//--- Calculate RMS error
v=0;
for(i=0; i<nc; i++)
{
w=cntbuf[i];
v+=w*CMath::Sqr(w/sl-1);
v+=(sl-w)*CMath::Sqr(w/sl);
w=cntbuf[nc+i];
v+=w*CMath::Sqr(w/sr-1);
v+=(sr-w)*CMath::Sqr(w/sr);
}
//--- change value
v=MathSqrt(v/(nc*n));
//--- Compare with best
x=(2.0*sl)/(double)(sl+sr)-1.0;
cc=v*(1.0+alpha*(CMath::Sqr(x)-1.0));
//--- check
if(cc<cbest)
{
//--- store split
rms=v;
koptimal=k;
cbest=cc;
//--- calculate CVRMS error
cvrms=0;
for(i=0; i<nc; i++)
{
//--- check
if(sl>1)
{
w=cntbuf[i];
cvrms+=w*CMath::Sqr((w-1.0)/(sl-1.0)-1.0);
cvrms+=(sl-w)*CMath::Sqr(w/(sl-1.0));
}
else
{
w=cntbuf[i];
cvrms+=w*CMath::Sqr(1.0/(double)nc-1.0);
cvrms+=(sl-w)*CMath::Sqr(1.0/(double)nc);
}
//--- check
if(sr>1)
{
w=cntbuf[nc+i];
cvrms+=w*CMath::Sqr((w-1.0)/(sr-1.0)-1.0);
cvrms+=(sr-w)*CMath::Sqr(w/(sr-1.0));
}
else
{
w=cntbuf[nc+i];
cvrms+=w*CMath::Sqr(1.0/(double)nc-1.0);
cvrms+=(sr-w)*CMath::Sqr(1.0/(double)nc);
}
}
//--- change value
cvrms=MathSqrt(cvrms/(double)(nc*n));
}
}
//--- Calculate threshold.
//--- Code is a bit complicated because there can be such
//--- numbers that 0.5(A+B) equals to A or B (if A-B=epsilon)
threshold=0.5*(a[tiesbuf[koptimal]]+a[tiesbuf[koptimal+1]]);
//--- check
if(threshold<=a[tiesbuf[koptimal]])
threshold=a[tiesbuf[koptimal+1]];
}
//+------------------------------------------------------------------+
//| Automatic non-optimal discretization, internal subroutine. |
//+------------------------------------------------------------------+
void CBdSS::DSSplitK(double &ca[],int &cc[],const int n,const int nc,
int kmax,int &info,double &thresholds[],int &ni,
double &cve)
{
CRowDouble A=ca;
CRowInt C=cc;
CRowDouble Thresholds=thresholds;
DSSplitK(A,C,n,nc,kmax,info,Thresholds,ni,cve);
Thresholds.ToArray(thresholds);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CBdSS::DSSplitK(CRowDouble &ca,CRowInt &cc,const int n,const int nc,
int kmax,int &info,CRowDouble &thresholds,int &ni,
double &cve)
{
//--- create variables
int i=0;
int j=0;
int j1=0;
int k=0;
int tiecount=0;
double v2=0;
int bestk=0;
double bestcve=0;
double curcve=0;
//--- creating arrays
CRowInt ties;
CRowInt p1;
CRowInt p2;
CRowInt cnt;
CRowInt bestsizes;
CRowInt cursizes;
CRowDouble a=ca;
CRowInt c=cc;
a.Resize(n);
c.Resize(n);
//--- initialization
info=0;
ni=0;
cve=0;
//--- Test for errors in inputs
if((n<=0 || nc<2) || kmax<2)
{
info=-1;
return;
}
for(i=0; i<n; i++)
{
//--- check
if(c[i]<0 || c[i]>=nc)
{
info=-2;
return;
}
}
//--- change value
info=1;
//--- Tie
DSTie(a,n,ties,tiecount,p1,p2);
//--- swap
for(i=0; i<n; i++)
{
//--- check
if(p2[i]!=i)
c.Swap(i,p2[i]);
}
//--- Special cases
if(tiecount==1)
{
info=-3;
return;
}
//--- General case:
//--- 0. allocate arrays
kmax=MathMin(kmax,tiecount);
//--- allocation
bestsizes.Resize(kmax);
cursizes.Resize(kmax);
cnt.Resize(nc);
//--- General case:
//--- 1. prepare "weak" solution (two subintervals,divided at median)
v2=CMath::m_maxrealnumber;
j=-1;
for(i=1; i<tiecount; i++)
{
//--- check
if(MathAbs(ties[i]-0.5*(n-1))<v2)
{
v2=MathAbs(ties[i]-0.5*n);
j=i;
}
}
//--- check
if(!CAp::Assert(j>0,__FUNCTION__+": internal error #1!"))
return;
//--- change values
bestk=2;
bestsizes.Set(0,ties[j]);
bestsizes.Set(1,n-j);
bestcve=0;
//--- calculation
cnt.Fill(0);
for(i=0; i<j; i++)
TieAddC(c,ties,i,nc,cnt);
bestcve+=GetCV(cnt,nc);
//--- calculation
cnt.Fill(0);
for(i=j; i<=tiecount-1; i++)
TieAddC(c,ties,i,nc,cnt);
bestcve+=GetCV(cnt,nc);
//--- General case:
//--- 2. Use greedy algorithm to find sub-optimal split in O(KMax*N) time
for(k=2; k<=kmax; k++)
{
//--- Prepare greedy K-interval split
cursizes.Fill(0,0,k);
//--- change values
i=0;
j=0;
//--- cycle
while(j<tiecount && i<k)
{
//--- Rule: I-th bin is empty,fill it
if(cursizes[i]==0)
{
cursizes.Set(i,ties[j+1]-ties[j]);
j++;
continue;
}
//--- Rule: (K-1-I) bins left,(K-1-I) ties left (1 tie per bin);next bin
if((tiecount-j)==(k-1-i))
{
i++;
continue;
}
//--- Rule: last bin,always place in current
if(i==(k-1))
{
cursizes.Add(i,ties[j+1]-ties[j]);
j++;
continue;
}
//--- Place J-th tie in I-th bin,or leave for I+1-th bin.
if(MathAbs(cursizes[i]+ties[j+1]-ties[j]-(double)n/(double)k)<MathAbs(cursizes[i]-(double)n/(double)k))
{
cursizes.Add(i,ties[j+1]-ties[j]);
j++;
}
else
i++;
}
//--- check
if(!CAp::Assert(cursizes[k-1]!=0 && j==tiecount,__FUNCTION__+": internal error #1"))
return;
//--- Calculate CVE
curcve=0;
j=0;
for(i=0; i<k; i++)
{
//--- calculation
cnt.Fill(0,0,nc);
for(j1=j; j1<j+cursizes[i]; j1++)
cnt.Add(c[j1],1);
curcve+=GetCV(cnt,nc);
j+=cursizes[i];
}
//--- Choose best variant
if(curcve<bestcve)
{
bestsizes.Copy(cursizes,0,0,k);
bestcve=curcve;
bestk=k;
}
}
//--- Transform from sizes to thresholds
cve=bestcve;
ni=bestk;
//--- allocation
thresholds.Resize(ni-1);
j=bestsizes[0];
//--- calculation
for(i=1; i<bestk; i++)
{
thresholds.Set(i-1,0.5*(a[j-1]+a[j]));
j+=bestsizes[i];
}
}
//+------------------------------------------------------------------+
//| Automatic optimal discretization, internal subroutine. |
//+------------------------------------------------------------------+
void CBdSS::DSOptimalSplitK(double &ca[],int &cc[],const int n,
const int nc,int kmax,int &info,
double &thresholds[],int &ni,double &cve)
{
CRowDouble A=ca;
CRowInt C=cc;
CRowDouble Thresholds=thresholds;
DSOptimalSplitK(A,C,n,nc,kmax,info,Thresholds,ni,cve);
Thresholds.ToArray(thresholds);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CBdSS::DSOptimalSplitK(CRowDouble &ca,CRowInt &cc,const int n,
const int nc,int kmax,int &info,
CRowDouble &thresholds,int &ni,double &cve)
{
//--- Test for errors in inputs
if((n<=0 || nc<2) || kmax<2)
{
info=-1;
return;
}
//--- create variables
int i=0;
int j=0;
int s=0;
int jl=0;
int jr=0;
double v2=0;
int tiecount=0;
double cvtemp=0;
int k=0;
int koptimal=0;
double cvoptimal=0;
//--- creating arrays
CRowInt ties;
CRowInt p1;
CRowInt p2;
CRowInt cnt;
CRowInt cnt2;
//--- copy
CRowDouble a=ca;
CRowInt c=cc;
a.Resize(n);
c.Resize(n);
//--- create matrix
CMatrixDouble cv;
CMatrixInt splits;
//--- initialization
info=0;
ni=0;
cve=0;
for(i=0; i<n; i++)
{
//--- check
if(c[i]<0 || c[i]>=nc)
{
info=-2;
return;
}
}
//--- change value
info=1;
//--- Tie
DSTie(a,n,ties,tiecount,p1,p2);
//--- swap
for(i=0; i<n; i++)
//--- check
if(p2[i]!=i)
c.Swap(i,p2[i]);
//--- Special cases
if(tiecount==1)
{
info=-3;
return;
}
//--- General case
//--- Use dynamic programming to find best split in O(KMax*NC*TieCount^2) time
kmax=MathMin(kmax,tiecount);
//--- allocation
cv.Resize(kmax,tiecount);
splits.Resize(kmax,tiecount);
cnt.Resize(nc);
cnt2.Resize(nc);
//--- calculation
cnt.Fill(0);
for(j=0; j<tiecount; j++)
{
TieAddC(c,ties,j,nc,cnt);
splits.Set(0,j,0);
cv.Set(0,j,GetCV(cnt,nc));
}
for(k=1; k<=kmax-1; k++)
{
cnt.Fill(0);
//--- Subtask size J in [K..TieCount-1]:
//--- optimal K-splitting on ties from 0-th to J-th.
for(j=k; j<=tiecount-1; j++)
{
//--- Update Cnt - let it contain classes of ties from K-th to J-th
TieAddC(c,ties,j,nc,cnt);
//--- Search for optimal split point S in [K..J]
cnt2=cnt;
cv.Set(k,j,cv.Get(k-1,j-1)+GetCV(cnt2,nc));
splits.Set(k,j,j);
//--- calculation
for(s=k+1; s<=j; s++)
{
//--- Update Cnt2 - let it contain classes of ties from S-th to J-th
TieSubC(c,ties,s-1,nc,cnt2);
//--- Calculate CVE
cvtemp=cv.Get(k-1,s-1)+GetCV(cnt2,nc);
//--- check
if(cvtemp<cv.Get(k,j))
{
cv.Set(k,j,cvtemp);
splits.Set(k,j,s);
}
}
}
}
//--- Choose best partition,output result
koptimal=-1;
cvoptimal=CMath::m_maxrealnumber;
for(k=0; k<kmax; k++)
{
//--- check
if(cv.Get(k,tiecount-1)<cvoptimal)
{
cvoptimal=cv.Get(k,tiecount-1);
koptimal=k;
}
}
//--- check
if(!CAp::Assert(koptimal>=0,__FUNCTION__+": internal error #1!"))
return;
//--- check
if(koptimal==0)
{
//--- Special case: best partition is one big interval.
//--- Even 2-partition is not better.
//--- This is possible when dealing with "weak" predictor variables.
//--- Make binary split as close to the median as possible.
v2=CMath::m_maxrealnumber;
j=-1;
for(i=1; i<=tiecount-1; i++)
{
//--- check
if(MathAbs(ties[i]-0.5*(n-1))<v2)
{
v2=MathAbs(ties[i]-0.5*(n-1));
j=i;
}
}
//--- check
if(!CAp::Assert(j>0,__FUNCTION__+": internal error #2!"))
return;
//--- allocation
thresholds.Resize(1);
//--- change values
thresholds.Set(0,0.5*(a[ties[j-1]]+a[ties[j]]));
ni=2;
cve=0;
//--- calculation
cnt.Fill(0);
for(i=0; i<j; i++)
TieAddC(c,ties,i,nc,cnt);
cve+=GetCV(cnt,nc);
cnt.Fill(0);
for(i=j; i<=tiecount-1; i++)
TieAddC(c,ties,i,nc,cnt);
cve+=GetCV(cnt,nc);
}
else
{
//--- General case: 2 or more intervals
thresholds.Resize(koptimal);
ni=koptimal+1;
cve=cv.Get(koptimal,tiecount-1);
jl=splits.Get(koptimal,tiecount-1);
jr=tiecount-1;
//--- calculation
for(k=koptimal; k>=1; k--)
{
thresholds.Set(k-1,0.5*(a[ties[jl-1]]+a[ties[jl]]));
jr=jl-1;
jl=splits.Get(k-1,jl-1);
}
}
}
//+------------------------------------------------------------------+
//| Internal function |
//+------------------------------------------------------------------+
double CBdSS::XLnY(const double x,const double y)
{
//--- check
if(x==0.0)
return(0.0);
//--- return result
return(x*MathLog(y));
}
//+------------------------------------------------------------------+
//| Internal function, |
//| returns number of samples of class I in Cnt[I] |
//+------------------------------------------------------------------+
double CBdSS::GetCV(CRowInt &cnt,const int nc)
{
//--- create variables
double result=0;
int i=0;
double s=0;
//--- calculation
s=0;
for(i=0; i<nc; i++)
s+=cnt[i];
//--- get result
result=0;
for(i=0; i<nc; i++)
result-=XLnY(cnt[i],cnt[i]/(s+nc-1));
//--- return result
return(result);
}
//+------------------------------------------------------------------+
//| Internal function,adds number of samples of class I in tie NTie |
//| to Cnt[I] |
//+------------------------------------------------------------------+
void CBdSS::TieAddC(CRowInt &c,CRowInt &ties,const int ntie,const int nc,
CRowInt &cnt)
{
for(int i=ties[ntie]; i<ties[ntie+1]; i++)
cnt.Add(c[i],1);
}
//+------------------------------------------------------------------+
//| Internal function,subtracts number of samples of class I in tie |
//| NTie to Cnt[I] |
//+------------------------------------------------------------------+
void CBdSS::TieSubC(CRowInt &c,CRowInt &ties,const int ntie,const int nc,
CRowInt &cnt)
{
for(int i=ties[ntie]; i<ties[ntie+1]; i++)
cnt.Add(c[i],-1);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
struct CDFWorkBuf
{
CRowInt m_classpriors;
CRowInt m_varpool;
int m_varpoolsize;
CRowInt m_trnset;
int m_trnsize;
CRowDouble m_trnlabelsr;
CRowInt m_trnlabelsi;
CRowInt m_oobset;
int m_oobsize;
CRowDouble m_ooblabelsr;
CRowInt m_ooblabelsi;
CRowDouble m_treebuf;
CRowDouble m_curvals;
CRowDouble m_bestvals;
CRowInt m_tmp0i;
CRowInt m_tmp1i;
CRowDouble m_tmp0r;
CRowDouble m_tmp1r;
CRowDouble m_tmp2r;
CRowDouble m_tmp3r;
CRowInt m_tmpnrms2;
CRowInt m_classtotals0;
CRowInt m_classtotals1;
CRowInt m_classtotals01;
//--- constructor / destructor
CDFWorkBuf(void) { m_trnsize=0; m_oobsize=0; }
~CDFWorkBuf(void) {}
void Copy(const CDFWorkBuf &obj);
//--- overloading
void operator=(const CDFWorkBuf &obj) { Copy(obj); }
};
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CDFWorkBuf::Copy(const CDFWorkBuf &obj)
{
m_classpriors=obj.m_classpriors;
m_varpool=obj.m_varpool;
m_varpoolsize=obj.m_varpoolsize;
m_trnset=obj.m_trnset;
m_trnsize=obj.m_trnsize;
m_trnlabelsr=obj.m_trnlabelsr;
m_trnlabelsi=obj.m_trnlabelsi;
m_oobset=obj.m_oobset;
m_oobsize=obj.m_oobsize;
m_ooblabelsr=obj.m_ooblabelsr;
m_ooblabelsi=obj.m_ooblabelsi;
m_treebuf=obj.m_treebuf;
m_curvals=obj.m_curvals;
m_bestvals=obj.m_bestvals;
m_tmp0i=obj.m_tmp0i;
m_tmp1i=obj.m_tmp1i;
m_tmp0r=obj.m_tmp0r;
m_tmp1r=obj.m_tmp1r;
m_tmp2r=obj.m_tmp2r;
m_tmp3r=obj.m_tmp3r;
m_tmpnrms2=obj.m_tmpnrms2;
m_classtotals0=obj.m_classtotals0;
m_classtotals1=obj.m_classtotals1;
m_classtotals01=obj.m_classtotals01;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
struct CDFVoteBuf
{
CRowDouble m_trntotals;
CRowDouble m_oobtotals;
CRowInt m_trncounts;
CRowInt m_oobcounts;
CRowDouble m_giniimportances;
//--- constructor / destructor
CDFVoteBuf(void) {}
~CDFVoteBuf(void) {}
//---
void Copy(const CDFVoteBuf &obj);
//--- overloading
void operator=(const CDFVoteBuf &obj) { Copy(obj); }
};
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CDFVoteBuf::Copy(const CDFVoteBuf &obj)
{
m_trntotals=obj.m_trntotals;
m_oobtotals=obj.m_oobtotals;
m_trncounts=obj.m_trncounts;
m_oobcounts=obj.m_oobcounts;
m_giniimportances=obj.m_giniimportances;
}
//+------------------------------------------------------------------+
//| Permutation importance buffer object, stores permutation - |
//| related losses for some subset of the dataset + some temporaries |
//| Losses - array[NVars + 2], stores sum of squared |
//| residuals for each permutation type: |
//| * Losses[0..NVars - 1] stores losses for |
//| permutation in J-th variable |
//| * Losses[NVars] stores loss for all variables |
//| being randomly perturbed |
//| * Losses[NVars + 1] stores loss for unperturbed |
//| dataset |
//+------------------------------------------------------------------+
struct CDFPermimpBuf
{
CRowDouble m_losses;
CRowDouble m_xraw;
CRowDouble m_xdist;
CRowDouble m_xcur;
CRowDouble m_y;
CRowDouble m_yv;
CRowDouble m_targety;
CRowInt m_startnodes;
//--- constructor / destructor
CDFPermimpBuf(void) {}
~CDFPermimpBuf(void) {}
//---
void Copy(const CDFPermimpBuf &obj);
//--- overloading
void operator=(const CDFPermimpBuf &obj) { Copy(obj); }
};
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CDFPermimpBuf::Copy(const CDFPermimpBuf &obj)
{
m_losses=obj.m_losses;
m_xraw=obj.m_xraw;
m_xdist=obj.m_xdist;
m_xcur=obj.m_xcur;
m_y=obj.m_y;
m_yv=obj.m_yv;
m_targety=obj.m_targety;
m_startnodes=obj.m_startnodes;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
struct CDFTreeBuf
{
CRowDouble m_treebuf;
int m_treeidx;
//--- constructor / destructor
CDFTreeBuf(void) { m_treeidx=-1; }
~CDFTreeBuf(void) {}
//---
void Copy(const CDFTreeBuf &obj);
//--- overloading
void operator=(const CDFTreeBuf &obj) { Copy(obj); }
};
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CDFTreeBuf::Copy(const CDFTreeBuf &obj)
{
m_treebuf=obj.m_treebuf;
m_treeidx=obj.m_treeidx;
}
//+------------------------------------------------------------------+
//| A random forest (decision forest) builder object. |
//| Used to store dataset and specify decision forest training |
//| algorithm settings. |
//+------------------------------------------------------------------+
struct CDecisionForestBuilder
{
int m_DSType;
int m_NPoints;
int m_NVars;
int m_NClasses;
CRowDouble m_DSData;
CRowDouble m_DSRVal;
CRowInt m_DSIVal;
int m_RDFAlgo;
double m_RDFRatio;
double m_RDFVars;
int m_RDFGlobalSeed;
int m_RDFSplitStrength;
int m_RDFImportance;
CRowDouble m_DSMin;
CRowDouble m_DSMax;
bool m_DSBinary[];
double m_DSRAvg;
CRowInt m_DSCTotals;
int m_RDFProgress;
int m_RDFTotal;
bool m_NeedIOBMatrix;
CMatrixInt m_IOBMatrix;
CRowInt m_VarImpShuffle2;
CDFWorkBuf m_WorkBuf[];
CDFVoteBuf m_VoteBuf[];
CDFTreeBuf m_TreeBuf[];
//--- constructor / destructor
CDecisionForestBuilder(void);
~CDecisionForestBuilder(void) {}
//---
void Copy(const CDecisionForestBuilder &obj);
//--- overloading
void operator=(const CDecisionForestBuilder &obj) { Copy(obj); }
};
//+------------------------------------------------------------------+
//| Constructor |
//+------------------------------------------------------------------+
CDecisionForestBuilder::CDecisionForestBuilder(void)
{
m_DSType=0;
m_NPoints=0;
m_NVars=0;
m_NClasses=0;
m_RDFAlgo=0;
m_RDFRatio=0;
m_RDFVars=0;
m_RDFGlobalSeed=0;
m_RDFSplitStrength=0;
m_RDFImportance=0;
m_DSRAvg=0;
m_RDFProgress=0;
m_RDFTotal=0;
m_NeedIOBMatrix=false;
}
//+------------------------------------------------------------------+
//| Copy |
//+------------------------------------------------------------------+
void CDecisionForestBuilder::Copy(const CDecisionForestBuilder &obj)
{
m_DSType=obj.m_DSType;;
m_NPoints=obj.m_NPoints;
m_NVars=obj.m_NVars;
m_NClasses=obj.m_NClasses;
m_DSData=obj.m_DSData;
m_DSRVal=obj.m_DSRVal;
m_DSIVal=obj.m_DSIVal;
m_RDFAlgo=obj.m_RDFAlgo;
m_RDFRatio=obj.m_RDFRatio;
m_RDFVars=obj.m_RDFVars;
m_RDFGlobalSeed=obj.m_RDFGlobalSeed;
m_RDFSplitStrength=obj.m_RDFSplitStrength;
m_RDFImportance=obj.m_RDFImportance;
m_DSMin=obj.m_DSMin;
m_DSMax=obj.m_DSMax;
ArrayCopy(m_DSBinary,obj.m_DSBinary);
m_DSRAvg=obj.m_DSRAvg;
m_DSCTotals=obj.m_DSCTotals;
m_RDFProgress=obj.m_RDFProgress;
m_RDFTotal=obj.m_RDFTotal;
m_NeedIOBMatrix=obj.m_NeedIOBMatrix;
m_IOBMatrix=obj.m_IOBMatrix;
m_VarImpShuffle2=obj.m_VarImpShuffle2;
int total=ArraySize(obj.m_WorkBuf);
ArrayResize(m_WorkBuf,total);
ArrayResize(m_VoteBuf,total);
ArrayResize(m_TreeBuf,total);
for(int i=0; i<total; i++)
{
m_WorkBuf[i]=obj.m_WorkBuf[i];
m_VoteBuf[i]=obj.m_VoteBuf[i];
m_TreeBuf[i]=obj.m_TreeBuf[i];
}
}
//+------------------------------------------------------------------+
//| Buffer object which is used to perform various requests(usually |
//| model inference) in the multithreaded mode(multiple threads |
//| working with same DF object). |
//| This object should be created with DFCreateBuffer(). |
//+------------------------------------------------------------------+
struct CDecisionForestBuffer
{
CRowDouble m_x;
CRowDouble m_y;
//--- constructor / destructor
CDecisionForestBuffer(void) {}
~CDecisionForestBuffer(void) {}
//---
void Copy(const CDecisionForestBuffer &obj);
//--- overloading
void operator=(const CDecisionForestBuffer &obj) { Copy(obj); }
};
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CDecisionForestBuffer::Copy(const CDecisionForestBuffer &obj)
{
m_x=obj.m_x;
m_y=obj.m_y;
}
//+------------------------------------------------------------------+
//| Decision forest (random forest) model. |
//+------------------------------------------------------------------+
class CDecisionForest
{
public:
int m_ForestFormat;
bool m_UseMantissa8;
int m_NVars;
int m_NClasses;
int m_NTrees;
int m_BufSize;
CRowDouble m_Trees;
CDecisionForestBuffer m_Buffer;
CRowInt m_Trees8;
//--- constructor, destructor
CDecisionForest(void) { m_ForestFormat=0; m_UseMantissa8=false; m_NVars=0; m_NClasses=0; m_NTrees=0; m_BufSize=0; }
~CDecisionForest(void) {}
//--- copy
void Copy(const CDecisionForest &obj);
//--- overloading
void operator=(const CDecisionForest &obj) { Copy(obj); }
};
//+------------------------------------------------------------------+
//| Copy |
//+------------------------------------------------------------------+
void CDecisionForest::Copy(const CDecisionForest &obj)
{
//--- copy variables
m_ForestFormat=obj.m_ForestFormat;
m_UseMantissa8=obj.m_UseMantissa8;
m_NVars=obj.m_NVars;
m_NClasses=obj.m_NClasses;
m_NTrees=obj.m_NTrees;
m_BufSize=obj.m_BufSize;
m_Trees=obj.m_Trees;
m_Buffer=obj.m_Buffer;
m_Trees8=obj.m_Trees8;
}
//+------------------------------------------------------------------+
//| This class is a shell for class CDecisionForest |
//+------------------------------------------------------------------+
class CDecisionForestShell
{
private:
CDecisionForest m_innerobj;
public:
//--- constructors, destructor
CDecisionForestShell(void) {}
CDecisionForestShell(CDecisionForest &obj) { m_innerobj.Copy(obj); }
~CDecisionForestShell(void) {}
//--- method
CDecisionForest *GetInnerObj(void) { return(GetPointer(m_innerobj)); }
};
//+------------------------------------------------------------------+
//| Decision forest training report. |
//| === training/oob errors ======================================== |
//| Following fields store training set errors: |
//| * relclserror - fraction of misclassified cases, [0,1] |
//| * avgce - average cross-entropy in bits per symbol |
//| * rmserror - root-mean-square error |
//| * avgerror - average error |
//| * avgrelerror - average relative error |
//| Out-of-bag estimates are stored in fields with same names, but |
//| "oob" prefix. |
//| For classification problems: |
//| * RMS, AVG and AVGREL errors are calculated for posterior |
//| probabilities |
//| For regression problems: |
//| * RELCLS and AVGCE errors are zero |
//| === variable importance ======================================== |
//| Following fields are used to store variable importance |
//| information: |
//| * topvars - variables ordered from the most important |
//| to less important ones (according to |
//| current choice of importance raiting). |
//| For example, topvars[0] contains index of |
//| the most important variable, and |
//| topvars[0:2] are indexes of 3 most |
//| important ones and so on. |
//| * varimportances - array[nvars], ratings (the larger, the |
//| more important the variable is, always in |
//| [0,1] range). By default, filled by zeros |
//| (no importance ratings are provided unless|
//| you explicitly request them). Zero rating |
//| means that variable is not important, |
//| however you will rarely encounter such |
//| a thing, in many cases unimportant |
//| variables produce nearly-zero (but |
//| nonzero) ratings. Variable importance |
//| report must be EXPLICITLY requested by |
//| calling: |
//| * DFBuilderSetImportanceGini() function, |
//| if you need out-of-bag Gini-based |
//| importance rating also known as MDI |
//| (fast to calculate, resistant to |
//| overfitting issues, but has some bias |
//| towards continuous and high-cardinality |
//| categorical variables) |
//| * DFBuilderSetImportanceTrnGini() function|
//| if you need training set Gini-based |
//| importance rating (what other packages |
//| typically report). |
//| * DFBuilderSetImportancePermutation() |
//| function, if you need permutation-based |
//| importance rating also known as MDA |
//| (slower to calculate, but less biased) |
//| * DFBuilderSetImportancenOne() function, |
//| if you do not need importance ratings - |
//| ratings will be zero, topvars[] will be |
//| [0,1,2,...] |
//| Different importance ratings (Gini or permutation) produce |
//| non-comparable values. Although in all cases rating values lie in|
//| [0,1] range, there are exist differences: |
//| * informally speaking, Gini importance rating tends to divide |
//| "unit amount of importance" between several important |
//| variables, i.e. it produces estimates which roughly sum |
//| to 1.0 (or less than 1.0, if your task can not be solved |
//| exactly). If all variables are equally important, they will |
//| have same rating, roughly 1/NVars, even if every variable is |
//| critically important. |
//| * from the other side, permutation importance tells us what |
//| percentage of the model predictive power will be ruined by |
//| permuting this specific variable. It does not produce |
//| estimates which sum to one. Critically important variable |
//| will have rating close to 1.0, and you may have multiple |
//| variables with such a rating. |
//| More information on variable importance ratings can be found in |
//| comments on the DFBuilderSetImportanceGini() and |
//| DFBuilderSetImportancePermutation() functions. |
//+------------------------------------------------------------------+
class CDFReport
{
public:
//--- variables
double m_RelCLSError;
double m_AvgCE;
double m_RMSError;
double m_AvgError;
double m_AvgRelError;
double m_oobrelclserror;
double m_oobavgce;
double m_oobrmserror;
double m_oobavgerror;
double m_oobavgrelerror;
CRowInt m_topvars;
CRowDouble m_varimportances;
//--- constructor, destructor
CDFReport(void);
~CDFReport(void) {}
//--- copy
void Copy(const CDFReport &obj);
//--- overloading
void operator=(const CDFReport &obj) { Copy(obj); }
};
//+------------------------------------------------------------------+
//| Constructor |
//+------------------------------------------------------------------+
CDFReport::CDFReport(void)
{
m_RelCLSError=0;
m_AvgCE=0;
m_RMSError=0;
m_AvgError=0;
m_AvgRelError=0;
m_oobrelclserror=0;
m_oobavgce=0;
m_oobrmserror=0;
m_oobavgerror=0;
m_oobavgrelerror=0;
}
//+------------------------------------------------------------------+
//| Copy |
//+------------------------------------------------------------------+
void CDFReport::Copy(const CDFReport &obj)
{
//--- copy variables
m_RelCLSError=obj.m_RelCLSError;
m_AvgCE=obj.m_AvgCE;
m_RMSError=obj.m_RMSError;
m_AvgError=obj.m_AvgError;
m_AvgRelError=obj.m_AvgRelError;
m_oobrelclserror=obj.m_oobrelclserror;
m_oobavgce=obj.m_oobavgce;
m_oobrmserror=obj.m_oobrmserror;
m_oobavgerror=obj.m_oobavgerror;
m_oobavgrelerror=obj.m_oobavgrelerror;
m_topvars=obj.m_topvars;
m_varimportances=obj.m_varimportances;
}
//+------------------------------------------------------------------+
//| This class is a shell for class CDFReport |
//+------------------------------------------------------------------+
class CDFReportShell
{
private:
CDFReport m_innerobj;
public:
//--- constructors, destructor
CDFReportShell(void) {}
CDFReportShell(CDFReport &obj) { m_innerobj.Copy(obj); }
~CDFReportShell(void) {}
//--- methods
double GetRelClsError(void);
void SetRelClsError(const double d);
double GetAvgCE(void);
void SetAvgCE(const double d);
double GetRMSError(void);
void SetRMSError(const double d);
double GetAvgError(void);
void SetAvgError(const double d);
double GetAvgRelError(void);
void SetAvgRelError(const double d);
double GetOOBRelClsError(void);
void SetOOBRelClsError(const double d);
double GetOOBAvgCE(void);
void SetOOBAvgCE(const double d);
double GetOOBRMSError(void);
void SetOOBRMSError(const double d);
double GetOOBAvgError(void);
void SetOOBAvgError(const double d);
double GetOOBAvgRelError(void);
void SetOOBAvgRelError(const double d);
CDFReport *GetInnerObj(void);
};
//+------------------------------------------------------------------+
//| Returns the value of the variable relclserror |
//+------------------------------------------------------------------+
double CDFReportShell::GetRelClsError(void)
{
return(m_innerobj.m_RelCLSError);
}
//+------------------------------------------------------------------+
//| Changing the value of the variable relclserror |
//+------------------------------------------------------------------+
void CDFReportShell::SetRelClsError(const double d)
{
m_innerobj.m_RelCLSError=d;
}
//+------------------------------------------------------------------+
//| Returns the value of the variable avgce |
//+------------------------------------------------------------------+
double CDFReportShell::GetAvgCE(void)
{
return(m_innerobj.m_AvgCE);
}
//+------------------------------------------------------------------+
//| Changing the value of the variable avgce |
//+------------------------------------------------------------------+
void CDFReportShell::SetAvgCE(const double d)
{
m_innerobj.m_AvgCE=d;
}
//+------------------------------------------------------------------+
//| Returns the value of the variable rmserror |
//+------------------------------------------------------------------+
double CDFReportShell::GetRMSError(void)
{
return(m_innerobj.m_RMSError);
}
//+------------------------------------------------------------------+
//| Changing the value of the variable rmserror |
//+------------------------------------------------------------------+
void CDFReportShell::SetRMSError(const double d)
{
m_innerobj.m_RMSError=d;
}
//+------------------------------------------------------------------+
//| Returns the value of the variable avgerror |
//+------------------------------------------------------------------+
double CDFReportShell::GetAvgError(void)
{
return(m_innerobj.m_AvgError);
}
//+------------------------------------------------------------------+
//| Changing the value of the variable avgerror |
//+------------------------------------------------------------------+
void CDFReportShell::SetAvgError(const double d)
{
m_innerobj.m_AvgError=d;
}
//+------------------------------------------------------------------+
//| Returns the value of the variable avgrelerror |
//+------------------------------------------------------------------+
double CDFReportShell::GetAvgRelError(void)
{
return(m_innerobj.m_AvgRelError);
}
//+------------------------------------------------------------------+
//| Changing the value of the variable avgrelerror |
//+------------------------------------------------------------------+
void CDFReportShell::SetAvgRelError(const double d)
{
m_innerobj.m_AvgRelError=d;
}
//+------------------------------------------------------------------+
//| Returns the value of the variable oobrelclserror |
//+------------------------------------------------------------------+
double CDFReportShell::GetOOBRelClsError(void)
{
return(m_innerobj.m_oobrelclserror);
}
//+------------------------------------------------------------------+
//| Changing the value of the variable oobrelclserror |
//+------------------------------------------------------------------+
void CDFReportShell::SetOOBRelClsError(const double d)
{
m_innerobj.m_oobrelclserror=d;
}
//+------------------------------------------------------------------+
//| Returns the value of the variable oobavgce |
//+------------------------------------------------------------------+
double CDFReportShell::GetOOBAvgCE(void)
{
return(m_innerobj.m_oobavgce);
}
//+------------------------------------------------------------------+
//| Changing the value of the variable oobavgce |
//+------------------------------------------------------------------+
void CDFReportShell::SetOOBAvgCE(const double d)
{
m_innerobj.m_oobavgce=d;
}
//+------------------------------------------------------------------+
//| Returns the value of the variable oobrmserror |
//+------------------------------------------------------------------+
double CDFReportShell::GetOOBRMSError(void)
{
return(m_innerobj.m_oobrmserror);
}
//+------------------------------------------------------------------+
//| Changing the value of the variable oobrmserror |
//+------------------------------------------------------------------+
void CDFReportShell::SetOOBRMSError(const double d)
{
m_innerobj.m_oobrmserror=d;
}
//+------------------------------------------------------------------+
//| Returns the value of the variable oobavgerror |
//+------------------------------------------------------------------+
double CDFReportShell::GetOOBAvgError(void)
{
return(m_innerobj.m_oobavgerror);
}
//+------------------------------------------------------------------+
//| Changing the value of the variable oobavgerror |
//+------------------------------------------------------------------+
void CDFReportShell::SetOOBAvgError(const double d)
{
m_innerobj.m_oobavgerror=d;
}
//+------------------------------------------------------------------+
//| Returns the value of the variable oobavgrelerror |
//+------------------------------------------------------------------+
double CDFReportShell::GetOOBAvgRelError(void)
{
return(m_innerobj.m_oobavgrelerror);
}
//+------------------------------------------------------------------+
//| Changing the value of the variable oobavgrelerror |
//+------------------------------------------------------------------+
void CDFReportShell::SetOOBAvgRelError(const double d)
{
m_innerobj.m_oobavgrelerror=d;
}
//+------------------------------------------------------------------+
//| Return object of class |
//+------------------------------------------------------------------+
CDFReport *CDFReportShell::GetInnerObj(void)
{
return(GetPointer(m_innerobj));
}
//+------------------------------------------------------------------+
//| Auxiliary class for CDForest |
//+------------------------------------------------------------------+
struct CDFInternalBuffers
{
//--- arrays
CRowDouble m_treebuf;
CRowInt m_idxbuf;
CRowDouble m_tmpbufr;
CRowDouble m_tmpbufr2;
CRowInt m_tmpbufi;
CRowInt m_classibuf;
CRowDouble m_sortrbuf;
CRowDouble m_sortrbuf2;
CRowInt m_sortibuf;
CRowInt m_varpool;
bool m_evsbin[];
CRowDouble m_evssplits;
//--- constructor, destructor
CDFInternalBuffers(void) {}
~CDFInternalBuffers(void) {}
//---
void Copy(const CDFInternalBuffers &obj);
//--- overloading
void operator=(const CDFInternalBuffers &obj) { Copy(obj); }
};
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CDFInternalBuffers::Copy(const CDFInternalBuffers &obj)
{
//---
m_treebuf=obj.m_treebuf;
m_idxbuf=obj.m_idxbuf;
m_tmpbufr=obj.m_tmpbufr;
m_tmpbufr2=obj.m_tmpbufr2;
m_tmpbufi=obj.m_tmpbufi;
m_classibuf=obj.m_classibuf;
m_sortrbuf=obj.m_sortrbuf;
m_sortrbuf2=obj.m_sortrbuf2;
m_sortibuf=obj.m_sortibuf;
m_varpool=obj.m_varpool;
m_evssplits=obj.m_evssplits;
//---
ArrayCopy(m_evsbin,obj.m_evsbin);
}
//+------------------------------------------------------------------+
//| Decision forest class |
//+------------------------------------------------------------------+
class CDForest
{
public:
//--- class constants
static const int m_InnerNodeWidth;
static const int m_LeafNodeWdth;
static const int m_DFUseStrongSplits;
static const int m_DFUseEVS;
static const int m_DFUncompressedV0;
static const int m_DFCompressedV0;
static const int m_NeedTrnGini;
static const int m_NeedOOBGini;
static const int m_NeedPermutation;
static const int m_PermutationImportanceBatchSize;
//--- public methods
static void DFCreateBuffer(CDecisionForest &model,CDecisionForestBuffer &buf);
static void DFBuilderCreate(CDecisionForestBuilder &s);
static void DFBuilderSetDataset(CDecisionForestBuilder &s,CMatrixDouble &xy,int npoints,int nvars,int nclasses);
static void DFBuilderSetRndVars(CDecisionForestBuilder &s,int rndvars);
static void DFBuilderSetRndVarsRatio(CDecisionForestBuilder &s,double f);
static void DFBuilderSetRndVarsAuto(CDecisionForestBuilder &s);
static void DFBuilderSetSubsampleRatio(CDecisionForestBuilder &s,double f);
static void DFBuilderSetSeed(CDecisionForestBuilder &s,int seedval);
static void DFBuilderSetRDFAlgo(CDecisionForestBuilder &s,int algotype);
static void DFBuilderSetRDFSplitStrength(CDecisionForestBuilder &s,int splitstrength);
static void DFBuilderSetImportanceTrnGini(CDecisionForestBuilder &s);
static void DFBuilderSetImportanceOOBGini(CDecisionForestBuilder &s);
static void DFBuilderSetImportancePermutation(CDecisionForestBuilder &s);
static void DFBuilderSetImportanceNone(CDecisionForestBuilder &s);
static double DFBuilderGetProgress(CDecisionForestBuilder &s);
static double DFBuilderPeekProgress(CDecisionForestBuilder &s);
static void DFBuilderBuildRandomForest(CDecisionForestBuilder &s,int ntrees,CDecisionForest &df,CDFReport &rep);
static double DFBinaryCompression(CDecisionForest &df);
static double DFBinaryCompression8(CDecisionForest &df);
static void DFProcess(CDecisionForest &df,double &x[],double &y[]);
static void DFProcess(CDecisionForest &df,CRowDouble &x,CRowDouble &y);
static void DFProcessI(CDecisionForest &df,double &x[],double &y[]);
static void DFProcessI(CDecisionForest &df,CRowDouble &x,CRowDouble &y);
static double DFProcess0(CDecisionForest &model,CRowDouble &x);
static int DFClassify(CDecisionForest &model,CRowDouble &x);
static double DFRelClsError(CDecisionForest &df,CMatrixDouble &xy,const int npoints);
static double DFAvgCE(CDecisionForest &df,CMatrixDouble &xy,const int npoints);
static double DFRMSError(CDecisionForest &df,CMatrixDouble &xy,const int npoints);
static double DFAvgError(CDecisionForest &df,CMatrixDouble &xy,const int npoints);
static double DFAvgRelError(CDecisionForest &df,CMatrixDouble &xy,const int npoints);
static void DFCopy(CDecisionForest &df1,CDecisionForest &df2);
static void DFAlloc(CSerializer &s,CDecisionForest &forest);
static void DFSerialize(CSerializer &s,CDecisionForest &forest);
static void DFUnserialize(CSerializer &s,CDecisionForest &forest);
//---
static void DFBuildRandomDecisionForest(CMatrixDouble &xy,int npoints,int nvars,int nclasses,int ntrees,double r,int &info,CDecisionForest &df,CDFReport &rep);
static void DFBuildRandomDecisionForestX1(CMatrixDouble &xy,int npoints,int nvars,int nclasses,int ntrees,int nrndvars,double r,int &info,CDecisionForest &df,CDFReport &rep);
static void DFBuildInternal(CMatrixDouble &xy,int npoints,int nvars,int nclasses,int ntrees,int samplesize,int nfeatures,int flags,int &info,CDecisionForest &df,CDFReport &rep);
private:
static void BuildRandomTree(CDecisionForestBuilder &s,int treeidx0,int treeidx1);
static void BuildRandomTreeRec(CDecisionForestBuilder &s,CDFWorkBuf &workbuf,int workingset,int varstoselect,CRowDouble &treebuf,CDFVoteBuf &votebuf,CHighQualityRandState &rs,int idx0,int idx1,int oobidx0,int oobidx1,double meanloss,double topmostmeanloss,int &treesize);
static void EstimateVariableImportance(CDecisionForestBuilder &s,int sessionseed,CDecisionForest &df,int ntrees,CDFReport &rep);
static void EstimatePermutationImportances(CDecisionForestBuilder &s,CDecisionForest &df,int ntrees,CDFPermimpBuf &permimpbuf,int idx0,int idx1);
static void CleanReport(CDecisionForestBuilder &s,CDFReport &rep);
static double MeanNRMS2(int nclasses,CRowInt &trnlabelsi,CRowDouble &trnlabelsr,int trnidx0,int trnidx1,CRowInt &tstlabelsi,CRowDouble &tstlabelsr,int tstidx0,int tstidx1,CRowInt &tmpi);
static void ChooseCurrentSplitDense(CDecisionForestBuilder &s,CDFWorkBuf &workbuf,int &varsinpool,int varstoselect,CHighQualityRandState &rs,int idx0,int idx1,int &varbest,double &splitbest);
static void EvaluateDenseSplit(CDecisionForestBuilder &s,CDFWorkBuf &workbuf,CHighQualityRandState &rs,int splitvar,int idx0,int idx1,int &info,double &split,double &rms);
static void ClassifierSplit(CDecisionForestBuilder &s,CDFWorkBuf &workbuf,CRowDouble &x,CRowInt &c,int n,CHighQualityRandState &rs,int &info,double &threshold,double &e,CRowDouble &sortrbuf,CRowInt &sortibuf);
static void RegressionSplit(CDecisionForestBuilder &s,CDFWorkBuf &workbuf,CRowDouble &x,CRowDouble &y,int n,int &info,double &threshold,double &e,CRowDouble &sortrbuf,CRowDouble &sortrbuf2);
static double GetSplit(CDecisionForestBuilder &s,double a,double b,CHighQualityRandState &rs);
static void OutputLeaf(CDecisionForestBuilder &s,CDFWorkBuf &workbuf,CRowDouble &treebuf,CDFVoteBuf &votebuf,int idx0,int idx1,int oobidx0,int oobidx1,int &treesize,double leafval);
static void AnalyzeAndPreprocessDataset(CDecisionForestBuilder &s);
static void MergeTrees(CDecisionForestBuilder &s,CDecisionForest &df);
static void ProcessVotingResults(CDecisionForestBuilder &s,int ntrees,CDFVoteBuf &buf,CDFReport &rep);
static double BinaryCompression(CDecisionForest &df,bool usemantissa8);
static int ComputeCompressedSizeRec(CDecisionForest &df,bool usemantissa8,int treeroot,int treepos,CRowInt &compressedsizes,bool savecompressedsizes);
static void CompressRec(CDecisionForest &df,bool usemantissa8,int treeroot,int treepos,CRowInt &compressedsizes,CRowInt &buf,int &dstoffs);
static int ComputeCompressedUintSize(int v);
static void StreamUint(CRowInt &buf,int &offs,int v);
static int UnstreamUint(CRowInt &buf,int &offs);
static void StreamFloat(CRowInt &buf,bool usemantissa8,int &offs,double v);
static double UnstreamFloat(CRowInt &buf,bool usemantissa8,int &off);
static int DFClsError(CDecisionForest &df,CMatrixDouble &xy,const int npoints);
static void DFProcessInternalUncompressed(CDecisionForest &df,int subtreeroot,int nodeoffs,CRowDouble &x,CRowDouble &y);
static void DFProcessInternalCompressed(CDecisionForest &df,int offs,CRowDouble &x,CRowDouble &y);
};
//+------------------------------------------------------------------+
//| Initialize constants |
//+------------------------------------------------------------------+
const int CDForest::m_InnerNodeWidth=3;
const int CDForest::m_LeafNodeWdth=2;
const int CDForest::m_DFUseStrongSplits=1;
const int CDForest::m_DFUseEVS=2;
const int CDForest::m_DFUncompressedV0=0;
const int CDForest::m_DFCompressedV0=1;
const int CDForest::m_NeedTrnGini=1;
const int CDForest::m_NeedOOBGini=2;
const int CDForest::m_NeedPermutation=3;
const int CDForest::m_PermutationImportanceBatchSize=512;
//+------------------------------------------------------------------+
//| This function creates buffer structure which can be used to |
//| perform parallel inference requests. |
//| DF subpackage provides two sets of computing functions - ones |
//| which use internal buffer of DF model (these functions are |
//| single-threaded because they use same buffer, which can not |
//| shared between threads), and ones which use external buffer. |
//| This function is used to initialize external buffer. |
//| INPUT PARAMETERS: |
//| Model - DF model which is associated with newly created |
//| buffer |
//| OUTPUT PARAMETERS: |
//| Buf - external buffer. |
//| IMPORTANT: buffer object should be used only with model which was|
//| used to initialize buffer. Any attempt to use buffer |
//| with different object is dangerous - you may get |
//| integrity check failure (exception) because sizes of |
//| internal arrays do not fit to dimensions of the model |
//| structure. |
//+------------------------------------------------------------------+
void CDForest::DFCreateBuffer(CDecisionForest &model,
CDecisionForestBuffer &buf)
{
buf.m_x.Resize(model.m_NVars);
buf.m_y.Resize(model.m_NClasses);
}
//+------------------------------------------------------------------+
//| This subroutine creates CDecisionForestBuilder object which is |
//| used to train decision forests. |
//| By default, new builder stores empty dataset and some reasonable |
//| default settings. At the very least, you should specify dataset |
//| prior to building decision forest. You can also tweak settings of|
//| the forest construction algorithm (recommended, although default |
//| setting should work well). |
//| Following actions are mandatory: |
//| * calling DFBuilderSetDataset() to specify dataset |
//| * calling DFBuilderBuildRandomForest() to build decision forest|
//| using current dataset and default settings |
//| Additionally, you may call: |
//| * DFBuilderSetRndVars() or DFBuilderSetRndVarsRatio() to |
//| specify number of variables randomly chosen for each split |
//| * DFBuilderSetSubsampleRatio() to specify fraction of the |
//| dataset randomly subsampled to build each tree |
//| * DFBuilderSetSeed() to control random seed chosen for tree |
//| construction |
//| INPUT PARAMETERS: |
//| none |
//| OUTPUT PARAMETERS: |
//| S - decision forest builder |
//+------------------------------------------------------------------+
void CDForest::DFBuilderCreate(CDecisionForestBuilder &s)
{
//--- Empty dataset
s.m_DSType=-1;
s.m_NPoints=0;
s.m_NVars=0;
s.m_NClasses=1;
//--- Default training settings
s.m_RDFAlgo=0;
s.m_RDFRatio=0.5;
s.m_RDFVars=0.0;
s.m_RDFGlobalSeed=0;
s.m_RDFSplitStrength=2;
s.m_RDFImportance=0;
//--- Other fields
s.m_RDFProgress=0;
s.m_RDFTotal=1;
}
//+------------------------------------------------------------------+
//| This subroutine adds dense dataset to the internal storage of the|
//| builder object. Specifying your dataset in the dense format means|
//| that the dense version of the forest construction algorithm will |
//| be invoked. |
//| INPUT PARAMETERS: |
//| S - decision forest builder object |
//| XY - array[NPoints,NVars+1] (minimum size; actual size |
//| can be larger, only leading part is used anyway), |
//| dataset: |
//| * first NVars elements of each row store values of |
//| the independent variables |
//| * last column store class number(in 0...NClasses-1)|
//| or real value of the dependent variable |
//| NPoints - number of rows in the dataset, NPoints>=1 |
//| NVars - number of independent variables, NVars>=1 |
//| NClasses - indicates type of the problem being solved: |
//| * NClasses>=2 means that classification problem is |
//| solved (last column of the dataset stores class |
//| number) |
//| * NClasses=1 means that regression problem is |
//| solved (last column of the dataset stores |
//| variable value) |
//| OUTPUT PARAMETERS: |
//| S - decision forest builder |
//+------------------------------------------------------------------+
void CDForest::DFBuilderSetDataset(CDecisionForestBuilder &s,
CMatrixDouble &xy,
int npoints,
int nvars,
int nclasses)
{
//--- create variables
int i=0;
int j=0;
//--- Check parameters
if(!CAp::Assert(npoints>=1,__FUNCTION__": npoints<1"))
return;
if(!CAp::Assert(nvars>=1,__FUNCTION__": nvars<1"))
return;
if(!CAp::Assert(nclasses>=1,__FUNCTION__": nclasses<1"))
return;
if(!CAp::Assert(xy.Rows()>=npoints,__FUNCTION__": rows(xy)<npoints"))
return;
if(!CAp::Assert(xy.Cols()>nvars,__FUNCTION__": cols(xy)<nvars+1"))
return;
if(!CAp::Assert(CApServ::IsFiniteMatrix(xy,npoints,nvars+1),__FUNCTION__": xy parameter contains INFs or NANs"))
return;
//---
if(nclasses>1)
for(i=0; i<npoints; i++)
{
j=(int)MathRound(xy.Get(i,nvars));
if(!CAp::Assert(j>=0 && j<nclasses,__FUNCTION__": last column of xy contains invalid class number"))
return;
}
//--- Set dataset
s.m_DSType=0;
s.m_NPoints=npoints;
s.m_NVars=nvars;
s.m_NClasses=nclasses;
CApServ::RVectorSetLengthAtLeast(s.m_DSData,npoints*nvars);
for(i=0; i<npoints; i++)
for(j=0; j<nvars; j++)
s.m_DSData.Set(j*npoints+i,xy.Get(i,j));
if(nclasses>1)
{
CApServ::IVectorSetLengthAtLeast(s.m_DSIVal,npoints);
for(i=0; i<npoints; i++)
s.m_DSIVal.Set(i,(int)MathRound(xy.Get(i,nvars)));
}
else
s.m_DSRVal=xy.Col(nvars)+0;
}
//+------------------------------------------------------------------+
//| This function sets number of variables (in [1,NVars] range) used |
//| by decision forest construction algorithm. |
//| The default option is to use roughly sqrt(NVars) variables. |
//| INPUT PARAMETERS: |
//| S - decision forest builder object |
//| RndVars - number of randomly selected variables; values |
//| outside of [1,NVars] range are silently clipped. |
//| OUTPUT PARAMETERS: |
//| S - decision forest builder |
//+------------------------------------------------------------------+
void CDForest::DFBuilderSetRndVars(CDecisionForestBuilder &s,
int rndvars)
{
s.m_RDFVars=MathMax(rndvars,1);
}
//+------------------------------------------------------------------+
//| This function sets number of variables used by decision forest |
//| construction algorithm as a fraction of total variable count |
//| (0,1) range. |
//| The default option is to use roughly sqrt(NVars) variables. |
//| INPUT PARAMETERS: |
//| S - decision forest builder object |
//| F - round(NVars*F) variables are selected |
//| OUTPUT PARAMETERS: |
//| S - decision forest builder |
//+------------------------------------------------------------------+
void CDForest::DFBuilderSetRndVarsRatio(CDecisionForestBuilder &s,
double f)
{
if(!CAp::Assert(MathIsValidNumber(f),__FUNCTION__": F is INF or NAN"))
return;
s.m_RDFVars=-MathMax(f,CMath::m_machineepsilon);
}
//+------------------------------------------------------------------+
//| This function tells decision forest builder to automatically |
//| choose number of variables used by decision forest construction |
//| algorithm. Roughly sqrt(NVars) variables will be used. |
//| INPUT PARAMETERS: |
//| S - decision forest builder object |
//| OUTPUT PARAMETERS: |
//| S - decision forest builder |
//+------------------------------------------------------------------+
void CDForest::DFBuilderSetRndVarsAuto(CDecisionForestBuilder &s)
{
s.m_RDFVars=0;
}
//+------------------------------------------------------------------+
//| This function sets size of dataset subsample generated the |
//| decision forest construction algorithm. Size is specified as a |
//| fraction of total dataset size. |
//| The default option is to use 50% of the dataset for training, |
//| 50% for the OOB estimates. You can decrease fraction F down to |
//| 10%, 1% or even below in order to reduce overfitting. |
//| INPUT PARAMETERS: |
//| S - decision forest builder object |
//| F - fraction of the dataset to use, in (0,1] range. |
//| Values outside of this range will be silently |
//| clipped. At least one element is always selected |
//| for the training set. |
//| OUTPUT PARAMETERS: |
//| S - decision forest builder |
//+------------------------------------------------------------------+
void CDForest::DFBuilderSetSubsampleRatio(CDecisionForestBuilder &s,
double f)
{
if(!CAp::Assert(MathIsValidNumber(f),__FUNCTION__": F is INF or NAN"))
return;
s.m_RDFRatio=MathMax(f,CMath::m_machineepsilon);
}
//+------------------------------------------------------------------+
//| This function sets seed used by internal RNG for random |
//| subsampling and random selection of variable subsets. |
//| By default random seed is used, i.e. every time you build |
//| decision forest, we seed generator with new value obtained from |
//| system-wide RNG. Thus, decision forest builder returns |
//| non-deterministic results. You can change such behavior by |
//| specyfing fixed positive seed value. |
//| INPUT PARAMETERS: |
//| S - decision forest builder object |
//| SeedVal - seed value: |
//| * positive values are used for seeding RNG with |
//| fixed seed, i.e. subsequent runs on same data |
//| will return same decision forests |
//| * non-positive seed means that random seed is used |
//| for every run of builder, i.e. subsequent runs |
//| on same datasets will return slightly different |
//| decision forests |
//| OUTPUT PARAMETERS: |
//| S - decision forest builder, see |
//+------------------------------------------------------------------+
void CDForest::DFBuilderSetSeed(CDecisionForestBuilder &s,
int seedval)
{
s.m_RDFGlobalSeed=seedval;
}
//+------------------------------------------------------------------+
//| This function sets random decision forest construction algorithm.|
//| As for now, only one decision forest construction algorithm is |
//| supported-a dense "baseline" RDF algorithm. |
//| INPUT PARAMETERS: |
//| S - decision forest builder object |
//| AlgoType - algorithm type: |
//| * 0 = baseline dense RDF |
//| OUTPUT PARAMETERS: |
//| S - decision forest builder, see |
//+------------------------------------------------------------------+
void CDForest::DFBuilderSetRDFAlgo(CDecisionForestBuilder &s,
int algotype)
{
if(!CAp::Assert(algotype==0,__FUNCTION__": unexpected algotype"))
return;
s.m_RDFAlgo=algotype;
}
//+------------------------------------------------------------------+
//| This function sets split selection algorithm used by decision |
//| forest classifier. You may choose several algorithms, with |
//| different speed and quality of the results. |
//| INPUT PARAMETERS: |
//| S - decision forest builder object |
//| SplitStrength - split type: |
//| * 0 = split at the random position, fastest one |
//| * 1 = split at the middle of the range |
//| * 2 = strong split at the best point of the range |
//| (default) |
//| OUTPUT PARAMETERS: |
//| S - decision forest builder, see |
//+------------------------------------------------------------------+
void CDForest::DFBuilderSetRDFSplitStrength(CDecisionForestBuilder &s,
int splitstrength)
{
if(!CAp::Assert(splitstrength==0 || splitstrength==1 || splitstrength==2,__FUNCTION__": unexpected split type"))
return;
s.m_RDFSplitStrength=splitstrength;
}
//+------------------------------------------------------------------+
//| This function tells decision forest construction algorithm to use|
//| Gini impurity based variable importance estimation (also known as|
//| MDI). |
//| This version of importance estimation algorithm analyzes mean |
//| decrease in impurity (MDI) on training sample during splits. The |
//| result is divided by impurity at the root node in order to |
//| produce estimate in [0,1] range. |
//| Such estimates are fast to calculate and beautifully normalized |
//| (sum to one) but have following downsides: |
//| * They ALWAYS sum to 1.0, even if output is completely |
//| unpredictable. I.e. MDI allows to order variables by |
//| importance, but does not tell us about "absolute" |
//| importances of variables |
//| * there exist some bias towards continuous and high-cardinality|
//| categorical variables |
//| NOTE: informally speaking, MDA (permutation importance) rating |
//| answers the question "what part of the model |
//| predictive power is ruined by permuting k-th variable?" |
//| while MDI tells us "what part of the model predictive power|
//| was achieved due to usage of k-th variable". |
//| Thus, MDA rates each variable independently at "0 to 1" scale |
//| while MDI (and OOB-MDI too) tends to divide "unit amount of |
//| importance" between several important variables. |
//| If all variables are equally important, they will have same |
//| MDI/OOB-MDI rating, equal (for OOB-MDI: roughly equal) to |
//| 1/NVars. However, roughly same picture will be produced for |
//| the "all variables provide information no one is critical" |
//| situation and for the "all variables are critical, drop any one, |
//| everything is ruined" situation. |
//| Contrary to that, MDA will rate critical variable as ~1.0 |
//| important, and important but non-critical variable will have less|
//| than unit rating. |
//| NOTE: quite an often MDA and MDI return same results. It |
//| generally happens on problems with low test set error |
//| (a few percents at most) and large enough training set |
//| to avoid overfitting. |
//| The difference between MDA, MDI and OOB-MDI becomes important |
//| only on "hard" tasks with high test set error and/or small |
//| training set. |
//| INPUT PARAMETERS: |
//| S - decision forest builder object |
//| OUTPUT PARAMETERS: |
//| S - decision forest builder object. Next call to the |
//| forest construction function will produce: |
//| * importance estimates in rep.varimportances field |
//| * variable ranks in rep.topvars field |
//+------------------------------------------------------------------+
void CDForest::DFBuilderSetImportanceTrnGini(CDecisionForestBuilder &s)
{
s.m_RDFImportance=m_NeedTrnGini;
}
//+------------------------------------------------------------------+
//| This function tells decision forest construction algorithm to use|
//| out-of-bag version of Gini variable importance estimation (also |
//| known as OOB-MDI). |
//| This version of importance estimation algorithm analyzes mean |
//| decrease in impurity (MDI) on out-of-bag sample during splits. |
//| The result is divided by impurity at the root node in order to |
//| produce estimate in [0,1] range. |
//| Such estimates are fast to calculate and resistant to overfitting|
//| issues (thanks to the out-of-bag estimates used). However, OOB |
//| Gini rating has following downsides: |
//| * there exist some bias towards continuous and |
//| high-cardinality categorical variables |
//| * Gini rating allows us to order variables by importance,|
//| but it is hard to define importance of the variable by |
//| itself. |
//| NOTE: informally speaking, MDA (permutation importance) rating |
//| answers the question "what part of the model predictive |
//| power is ruined by permuting k-th variable?" while MDI |
//| tells us "what part of the model predictive power was |
//| achieved due to usage of k-th variable". |
//| Thus, MDA rates each variable independently at "0 to 1" scale |
//| while MDI (and OOB-MDI too) tends to divide "unit amount of |
//| importance" between several important variables. |
//| If all variables are equally important, they will have same |
//| MDI/OOB-MDI rating, equal (for OOB-MDI: roughly equal) to |
//| 1/NVars. However, roughly same picture will be produced for the |
//| "all variables provide information no one is critical" situation |
//| and for the "all variables are critical, drop any one, everything|
//| is ruined" situation. |
//| Contrary to that, MDA will rate critical variable as ~1.0 |
//| important, and important but non-critical variable will have less|
//| than unit rating. |
//| NOTE: quite an often MDA and MDI return same results. It |
//| generally happens on problems with low test set error |
//| (a few percents at most) and large enough training set to |
//| avoid overfitting. |
//| The difference between MDA, MDI and OOB-MDI becomes important |
//| only on "hard" tasks with high test set error and/or small |
//| training set. |
//| INPUT PARAMETERS: |
//| S - decision forest builder object |
//| OUTPUT PARAMETERS: |
//| S - decision forest builder object. Next call to the|
//| forest construction function will produce: |
//| * importance estimates in rep.varimportances field |
//| * variable ranks in rep.topvars field |
//+------------------------------------------------------------------+
void CDForest::DFBuilderSetImportanceOOBGini(CDecisionForestBuilder &s)
{
s.m_RDFImportance=m_NeedOOBGini;
}
//+------------------------------------------------------------------+
//| This function tells decision forest construction algorithm to use|
//| permutation variable importance estimator (also known as MDA). |
//| This version of importance estimation algorithm analyzes mean |
//| increase in out-of-bag sum of squared residuals after random |
//| permutation of J-th variable. The result is divided by error |
//| computed with all variables being perturbed in order to produce |
//| R-squared-like estimate in [0,1] range. |
//| Such estimate is slower to calculate than Gini-based rating |
//| because it needs multiple inference runs for each of variables |
//| being studied. |
//| MDA rating has following benefits over Gini-based ones: |
//| * no bias towards specific variable types |
//| *ability to directly evaluate "absolute" importance of some|
//| variable at "0 to 1" scale (contrary to Gini-based rating,|
//| which returns comparative importances). |
//| NOTE: informally speaking, MDA (permutation importance) rating |
//| answers the question "what part of the model predictive |
//| power is ruined by permuting k-th variable?" while MDI |
//| tells us "what part of the model predictive power was |
//| achieved due to usage of k-th variable". |
//| Thus, MDA rates each variable independently at "0 to 1" scale |
//| while MDI (and OOB-MDI too) tends to divide "unit amount of |
//| importance" between several important variables. |
//| If all variables are equally important, they will have same |
//| MDI/OOB-MDI rating, equal (for OOB-MDI: roughly equal) to |
//| 1/NVars. However, roughly same picture will be produced forthe |
//| "all variables provide information no one is critical" situation |
//| and for the "all variables are critical, drop any one, everything|
//| is ruined" situation. |
//| Contrary to that, MDA will rate critical variable as ~1.0 |
//| important, and important but non-critical variable will have less|
//| than unit rating. |
//| NOTE: quite an often MDA and MDI return same results. It |
//| generally happens on problems with low test set error |
//| (a few percents at most) and large enough training set |
//| to avoid overfitting. |
//| The difference between MDA, MDI and OOB-MDI becomes important |
//| only on "hard" tasks with high test set error and/or small |
//| training set. |
//| INPUT PARAMETERS: |
//| S - decision forest builder object |
//| OUTPUT PARAMETERS: |
//| S - decision forest builder object. Next call to |
//| the forest construction function will produce: |
//| * importance estimates in rep.varimportances field |
//| * variable ranks in rep.topvars field |
//+------------------------------------------------------------------+
void CDForest::DFBuilderSetImportancePermutation(CDecisionForestBuilder &s)
{
s.m_RDFImportance=m_NeedPermutation;
}
//+------------------------------------------------------------------+
//| This function tells decision forest construction algorithm to |
//| skip variable importance estimation. |
//| INPUT PARAMETERS: |
//| S - decision forest builder object |
//| OUTPUT PARAMETERS: |
//| S - decision forest builder object. Next call to the |
//| forest construction function will result in forest |
//| being built without variable importance estimation.|
//+------------------------------------------------------------------+
void CDForest::DFBuilderSetImportanceNone(CDecisionForestBuilder &s)
{
s.m_RDFImportance=0;
}
//+------------------------------------------------------------------+
//| This function is an alias for dfbuilderpeekprogress(), left in |
//| ALGLIB for backward compatibility reasons. |
//+------------------------------------------------------------------+
double CDForest::DFBuilderGetProgress(CDecisionForestBuilder &s)
{
return(DFBuilderPeekProgress(s));
}
//+------------------------------------------------------------------+
//| This function is used to peek into decision forest construction |
//| process from some other thread and get current progress indicator|
//| It returns value in [0,1]. |
//| INPUT PARAMETERS: |
//| S - decision forest builder object used to build forest|
//| in some other thread |
//| RESULT: |
//| progress value, in [0,1] |
//+------------------------------------------------------------------+
double CDForest::DFBuilderPeekProgress(CDecisionForestBuilder &s)
{
double result=0;
//--- calculation
result=s.m_RDFProgress/MathMax(s.m_RDFTotal,1);
result=MathMax(result,0);
result=MathMin(result,1);
//--- return result
return(result);
}
//+------------------------------------------------------------------+
//| This subroutine builds decision forest according to current |
//| settings using dataset internally stored in the builder object. |
//| Dense algorithm is used. |
//| NOTE: this function uses dense algorithm for forest construction |
//| independently from the dataset format (dense or sparse). |
//| NOTE: forest built with this function is stored in-memory using |
//| 64-bit data structures for offsets/indexes/split values. It|
//| is possible to convert forest into more memory-efficient |
//| compressed binary representation. Depending on the problem |
//| properties, 3.7x-5.7x compression factors are possible. |
//| The downsides of compression are (a) slight reduction in the |
//| model accuracy and (b) ~1.5x reduction in the inference speed |
//| (due to increased complexity of the storage format). |
//| See comments on DFBinaryCompression() for more info. |
//| Default settings are used by the algorithm; you can tweak them |
//| with the help of the following functions: |
//| * DFBuilderSetRFactor() - to control a fraction of the |
//| dataset used for subsampling |
//| * DFBuilderSetRandomVars() - to control number of variables |
//| randomly chosen for decision rule |
//| creation |
//| INPUT PARAMETERS: |
//| S - decision forest builder object |
//| NTrees - NTrees>=1, number of trees to train |
//| OUTPUT PARAMETERS: |
//| D - decision forest. You can compress this forest to |
//| more compact 16-bit representation with |
//| DFBinaryCompression() |
//| Rep - report, see below for information on its fields. |
//| == report information produced by forest construction function = |
//| Decision forest training report includes following information: |
//| * training set errors |
//| * out-of-bag estimates of errors |
//| * variable importance ratings |
//| Following fields are used to store information: |
//| * training set errors are stored in rep.RelCLSError, rep.AvgCE,|
//| rep.RMSError, rep.AvgError and rep.AvgRelError |
//| * out-of-bag estimates of errors are stored in |
//| rep.oobrelclserror, rep.oobavgce, rep.oobrmserror, |
//| rep.oobavgerror and rep.oobavgrelerror |
//| Variable importance reports, if requested by |
//| DFBuilderSetImportanceGini(), DFBuilderSetImportanceTrnGini() or |
//| DFBuilderSetImportancePermutation() call, are stored in: |
//| * rep.varimportances field stores importance ratings |
//| * rep.topvars stores variable indexes ordered from the most |
//| important to less important ones |
//| You can find more information about report fields in: |
//| * comments on CDFReport structure |
//| * comments on DFBuilderSetImportanceGini function |
//| * comments on DFBuilderSetImportanceTrnGini function |
//| * comments on DFBuilderDetImportancePermutation function |
//+------------------------------------------------------------------+
void CDForest::DFBuilderBuildRandomForest(CDecisionForestBuilder &s,
int ntrees,
CDecisionForest &df,
CDFReport &rep)
{
//--- create variables
int nvars=0;
int nclasses=0;
int npoints=0;
int trnsize=0;
int maxtreesize=0;
int sessionseed=0;
CDFVoteBuf buf;
//--- check
if(!CAp::Assert(ntrees>=1,__FUNCTION__": ntrees<1"))
return;
CleanReport(s,rep);
npoints=s.m_NPoints;
nvars=s.m_NVars;
nclasses=s.m_NClasses;
//--- Set up progress counter
s.m_RDFProgress=0;
s.m_RDFTotal=ntrees*npoints;
if(s.m_RDFImportance==m_NeedPermutation)
s.m_RDFTotal+=ntrees*npoints;
//--- Quick exit for empty dataset
if(s.m_DSType==-1 || npoints==0)
{
if(!CAp::Assert(m_LeafNodeWdth==2,__FUNCTION__": integrity check failed"))
return;
df.m_ForestFormat=m_DFUncompressedV0;
df.m_NVars=s.m_NVars;
df.m_NClasses=s.m_NClasses;
df.m_NTrees=1;
df.m_BufSize=1+m_LeafNodeWdth;
df.m_Trees.Resize(1+m_LeafNodeWdth);
df.m_Trees.Set(0,1+m_LeafNodeWdth);
df.m_Trees.Set(1,-1.0);
df.m_Trees.Set(2,0.0);
DFCreateBuffer(df,df.m_Buffer);
return;
}
if(!CAp::Assert(npoints>0,__FUNCTION__": integrity check failed"))
return;
//--- Analyze dataset statistics, perform preprocessing
AnalyzeAndPreprocessDataset(s);
//--- Prepare "work", "vote" and "tree" pools and other settings
trnsize=(int)MathRound(npoints*s.m_RDFRatio);
trnsize=MathMax(trnsize,1);
trnsize=MathMin(trnsize,npoints);
maxtreesize=1+m_InnerNodeWidth*(trnsize-1)+m_LeafNodeWdth*trnsize;
//--- Allocation
ResetLastError();
if(!CAp::Assert(ArrayResize(s.m_WorkBuf,ntrees)==ntrees,StringFormat("%s: resize buffer failed (%d)",__FUNCTION__,GetLastError())))
return;
if(!CAp::Assert(ArrayResize(s.m_VoteBuf,ntrees)==ntrees,StringFormat("%s: resize buffer failed (%d)",__FUNCTION__,GetLastError())))
return;
if(!CAp::Assert(ArrayResize(s.m_TreeBuf,ntrees)==ntrees,StringFormat("%s: resize buffer failed (%d)",__FUNCTION__,GetLastError())))
return;
s.m_WorkBuf[0].m_varpool.Resize(nvars);
s.m_WorkBuf[0].m_trnset.Resize(trnsize);
s.m_WorkBuf[0].m_oobset.Resize(npoints-trnsize);
s.m_WorkBuf[0].m_tmp0i.Resize(npoints);
s.m_WorkBuf[0].m_tmp1i.Resize(npoints);
s.m_WorkBuf[0].m_tmp0r.Resize(npoints);
s.m_WorkBuf[0].m_tmp1r.Resize(npoints);
s.m_WorkBuf[0].m_tmp2r.Resize(npoints);
s.m_WorkBuf[0].m_tmp3r.Resize(npoints);
s.m_WorkBuf[0].m_trnlabelsi.Resize(npoints);
s.m_WorkBuf[0].m_trnlabelsr.Resize(npoints);
s.m_WorkBuf[0].m_ooblabelsi.Resize(npoints);
s.m_WorkBuf[0].m_ooblabelsr.Resize(npoints);
s.m_WorkBuf[0].m_curvals.Resize(npoints);
s.m_WorkBuf[0].m_bestvals.Resize(npoints);
s.m_WorkBuf[0].m_classpriors.Resize(nclasses);
s.m_WorkBuf[0].m_classtotals0.Resize(nclasses);
s.m_WorkBuf[0].m_classtotals1.Resize(nclasses);
s.m_WorkBuf[0].m_classtotals01.Resize(2*nclasses);
s.m_WorkBuf[0].m_treebuf.Resize(maxtreesize);
s.m_WorkBuf[0].m_trnsize=trnsize;
s.m_WorkBuf[0].m_oobsize=npoints-trnsize;
s.m_VoteBuf[0].m_trntotals=vector<double>::Zeros(npoints*nclasses);
s.m_VoteBuf[0].m_oobtotals=s.m_VoteBuf[0].m_trntotals;
s.m_VoteBuf[0].m_trncounts.Resize(npoints);
s.m_VoteBuf[0].m_oobcounts.Resize(npoints);
s.m_VoteBuf[0].m_trncounts.Fill(0);
s.m_VoteBuf[0].m_oobcounts.Fill(0);
s.m_VoteBuf[0].m_giniimportances=vector<double>::Zeros(nvars);
for(int treeIdx=1; treeIdx<ntrees; treeIdx++)
{
s.m_WorkBuf[treeIdx]=s.m_WorkBuf[0];
s.m_VoteBuf[treeIdx]=s.m_VoteBuf[0];
}
//--- Select session seed (individual trees are constructed using
//--- combination of session and local seeds).
sessionseed=s.m_RDFGlobalSeed;
if(s.m_RDFGlobalSeed<=0)
sessionseed=CMath::RandomInteger(30000);
//--- Prepare In-and-Out-of-Bag matrix, if needed
s.m_NeedIOBMatrix=(s.m_RDFImportance==m_NeedPermutation);
if(s.m_NeedIOBMatrix)
{
//--- Prepare default state of In-and-Out-of-Bag matrix
s.m_IOBMatrix.Resize(ntrees,npoints);
s.m_IOBMatrix.Fill((int)false);
}
//--- Build tree
BuildRandomTree(s,0,ntrees);
//--- Merge tree and output result
MergeTrees(s,df);
//--- Process voting results and output training set and OOB errors.
//--- Finalize tree construction.
ProcessVotingResults(s,ntrees,buf,rep);
DFCreateBuffer(df,df.m_Buffer);
//--- Perform variable importance estimation
EstimateVariableImportance(s,sessionseed,df,ntrees,rep);
//--- Update progress counter
s.m_RDFProgress=s.m_RDFTotal;
}
//+------------------------------------------------------------------+
//| This function performs binary compression of the decision forest.|
//| Original decision forest produced by the forest builder is stored|
//| using 64-bit representation for all numbers - offsets, variable |
//| indexes, split points. |
//| It is possible to significantly reduce model size by means of: |
//| * using compressed dynamic encoding for integers (offsets and |
//| variable indexes), which uses just 1 byte to store small ints|
//| (less than 128), just 2 bytes for larger values (less than |
//| 128^2) and so on |
//| * storing floating point numbers using 8-bit exponent and |
//| 16-bit mantissa |
//| As result, model needs significantly less memory (compression |
//| factor depends on variable and class counts). In particular: |
//| * NVars<128 and NClasses<128 result in 4.4x-5.7x model size |
//| reduction |
//| * NVars<16384 and NClasses<128 result in 3.7x-4.5x model size |
//| reduction |
//| Such storage format performs lossless compression of all integers|
//| but compression of floating point values (split values) is lossy,|
//| with roughly 0.01% relative error introduced during rounding. |
//| Thus, we recommend you to re-evaluate model accuracy after |
//| compression. |
//| Another downside of compression is ~1.5x reduction in the |
//| inference speed due to necessity of dynamic decompression of the |
//| compressed model. |
//| INPUT PARAMETERS: |
//| DF - decision forest built by forest builder |
//| OUTPUT PARAMETERS: |
//| DF - replaced by compressed forest |
//| RESULT: |
//| compression factor (in-RAM size of the compressed model vs than |
//| of the uncompressed one), positive number larger than 1.0 |
//+------------------------------------------------------------------+
double CDForest::DFBinaryCompression(CDecisionForest &df)
{
return(BinaryCompression(df,false));
}
//+------------------------------------------------------------------+
//| This is a 8-bit version of DFBinaryCompression. |
//| Not recommended for external use because it is too lossy. |
//+------------------------------------------------------------------+
double CDForest::DFBinaryCompression8(CDecisionForest &df)
{
return(BinaryCompression(df,true));
}
//+------------------------------------------------------------------+
//| Procesing |
//| INPUT PARAMETERS: |
//| DF - decision forest model |
//| X - input vector, array[0..NVars-1]. |
//| OUTPUT PARAMETERS: |
//| Y - result. Regression estimate when solving |
//| regression task, vector of posterior |
//| probabilities for classification task. |
//| See also DFProcessI. |
//+------------------------------------------------------------------+
void CDForest::DFProcess(CDecisionForest &df,double &x[],double &y[])
{
CRowDouble X=x;
CRowDouble Y;
DFProcess(df,X,Y);
Y.ToArray(y);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CDForest::DFProcess(CDecisionForest &df,CRowDouble &x,CRowDouble &y)
{
//--- create variables
int offs=0;
int i=0;
double v=0;
int treesize=0;
bool processed=false;
//--- Proceed
//--- Although comments above warn you about thread-unsafety of this
//--- function, it is de facto thread-safe. However, thread safety is
//--- an accidental side-effect of the specific inference algorithm
//--- being used. It may disappear in the future versions of the DF
//--- models, so you should NOT rely on it.
y.Resize(df.m_NClasses);
//--- initialization
y.Fill(0);
//--- calculation
if(df.m_ForestFormat==m_DFUncompressedV0)
{
//--- Process trees stored in uncompressed format
for(i=0; i<df.m_NTrees; i++)
{
DFProcessInternalUncompressed(df,offs,offs+1,x,y);
offs+=(int)MathRound(df.m_Trees[offs]);
}
processed=true;
}
if(df.m_ForestFormat==m_DFCompressedV0)
{
//--- Process trees stored in compressed format
offs=0;
for(i=0; i<df.m_NTrees; i++)
{
treesize=UnstreamUint(df.m_Trees8,offs);
DFProcessInternalCompressed(df,offs,x,y);
offs+=treesize;
}
processed=true;
}
if(!CAp::Assert(processed,__FUNCTION__": integrity check failed (unexpected format?)"))
return;
y*=1.0/(double)df.m_NTrees;
}
//+------------------------------------------------------------------+
//| 'interactive' variant of DFProcess for languages like Python |
//| which support constructs like "Y = DFProcessI(DF,X)" and |
//| interactive mode of interpreter |
//| This function allocates new array on each call, so it is |
//| significantly slower than its 'non-interactive' counterpart, but |
//| it is more convenient when you call it from command line. |
//+------------------------------------------------------------------+
void CDForest::DFProcessI(CDecisionForest &df,double &x[],double &y[])
{
DFProcess(df,x,y);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CDForest::DFProcessI(CDecisionForest &df,CRowDouble &x,CRowDouble &y)
{
DFProcess(df,x,y);
}
//+------------------------------------------------------------------+
//| This function returns first component of the inferred vector |
//| (i.e. one with index #0). |
//| It is a convenience wrapper for dfprocess() intended for either: |
//| * 1-dimensional regression problems |
//| * 2-class classification problems |
//| In the former case this function returns inference result as |
//| scalar, which is definitely more convenient that wrapping it as |
//| vector. In the latter case it returns probability of object |
//| belonging to class #0. |
//| If you call it for anything different from two cases above, it |
//| will work as defined, i.e. return y[0], although it is of less |
//| use in such cases. |
//| INPUT PARAMETERS: |
//| Model - DF model |
//| X - input vector, array[0..NVars-1]. |
//| RESULT: |
//| Y[0] |
//+------------------------------------------------------------------+
double CDForest::DFProcess0(CDecisionForest &model,CRowDouble &x)
{
//--- copy
model.m_Buffer.m_x=x;
//--- function call
DFProcess(model,model.m_Buffer.m_x,model.m_Buffer.m_y);
//--- return result
return(model.m_Buffer.m_y[0]);
}
//+------------------------------------------------------------------+
//| This function returns most probable class number for an input X. |
//| It is same as calling DFProcess(model,x,y), then determining |
//| i=ArgMax(y[i]) and returning i. |
//| A class number in [0,NOut) range in returned for classification |
//| problems, -1 is returned when this function is called for |
//| regression problems. |
//| INPUT PARAMETERS: |
//| Model - decision forest model |
//| X - input vector, array[0..NVars-1]. |
//| RESULT: |
//| class number, -1 for regression tasks |
//+------------------------------------------------------------------+
int CDForest::DFClassify(CDecisionForest &model,CRowDouble &x)
{
//--- check
if(model.m_NClasses<2)
return(-1);
model.m_Buffer.m_x=x;
DFProcess(model,model.m_Buffer.m_x,model.m_Buffer.m_y);
//--- return result
return (int)model.m_Buffer.m_y.ArgMax();
}
//+------------------------------------------------------------------+
//| Relative classification error on the test set |
//| INPUT PARAMETERS: |
//| DF - decision forest model |
//| XY - test set |
//| NPoints - test set size |
//| RESULT: |
//| percent of incorrectly classified cases. |
//| Zero if model solves regression task. |
//+------------------------------------------------------------------+
double CDForest::DFRelClsError(CDecisionForest &df,CMatrixDouble &xy,
const int npoints)
{
return((double)DFClsError(df,xy,npoints)/(double)npoints);
}
//+------------------------------------------------------------------+
//| Average cross-entropy (in bits per element) on the test set |
//| INPUT PARAMETERS: |
//| DF - decision forest model |
//| XY - test set |
//| NPoints - test set size |
//| RESULT: |
//| CrossEntropy/(NPoints*LN(2)). |
//| Zero if model solves regression task. |
//+------------------------------------------------------------------+
double CDForest::DFAvgCE(CDecisionForest &df,CMatrixDouble &xy,
const int npoints)
{
//--- create variables
double result=0;
CRowDouble x;
CRowDouble y;
for(int i=0; i<npoints; i++)
{
x=xy[i]+0;
x.Resize(df.m_NVars);
//--- function call
DFProcess(df,x,y);
//--- check
if(df.m_NClasses>1)
{
//--- classification-specific code
int k=(int)MathRound(xy.Get(i,df.m_NVars));
//--- check
if(y[k]!=0.0)
result-=MathLog(y[k]);
else
result-=MathLog(CMath::m_minrealnumber);
}
}
//--- return result
return(result/npoints);
}
//+------------------------------------------------------------------+
//| RMS error on the test set |
//| INPUT PARAMETERS: |
//| DF - decision forest model |
//| XY - test set |
//| NPoints - test set size |
//| RESULT: |
//| root mean square error. |
//| Its meaning for regression task is obvious. As for |
//| classification task,RMS error means error when estimating |
//| posterior probabilities. |
//+------------------------------------------------------------------+
double CDForest::DFRMSError(CDecisionForest &df,CMatrixDouble &xy,
const int npoints)
{
//--- create variables
double result=0;
CRowDouble x;
CRowDouble y;
for(int i=0; i<npoints; i++)
{
x=xy[i]+0;
x.Resize(df.m_NVars);
//--- function call
DFProcess(df,x,y);
//--- check
if(df.m_NClasses>1)
{
//--- classification-specific code
int k=(int)MathRound(xy.Get(i,df.m_NVars));
y.Add(k,-1);
result+=MathPow(y.ToVector()+0,2.0).Sum();
}
else
{
//--- regression-specific code
result+=CMath::Sqr(y[0]-xy.Get(i,df.m_NVars));
}
}
//--- return result
return(MathSqrt(result/(npoints*df.m_NClasses)));
}
//+------------------------------------------------------------------+
//| Average error on the test set |
//| INPUT PARAMETERS: |
//| DF - decision forest model |
//| XY - test set |
//| NPoints - test set size |
//| RESULT: |
//| Its meaning for regression task is obvious. As for |
//| classification task, it means average error when estimating |
//| posterior probabilities. |
//+------------------------------------------------------------------+
double CDForest::DFAvgError(CDecisionForest &df,CMatrixDouble &xy,
const int npoints)
{
//--- create variables
double result=0;
CRowDouble x;
CRowDouble y;
for(int i=0; i<npoints; i++)
{
//--- copy
x=xy[i]+0;
x.Resize(df.m_NVars);
//--- function call
DFProcess(df,x,y);
//--- check
if(df.m_NClasses>1)
{
//--- classification-specific code
int k=(int)MathRound(xy.Get(i,df.m_NVars));
y.Add(k,-1);
result+=(y.Abs()+0).Sum();
}
else
{
//--- regression-specific code
result+=MathAbs(y[0]-xy.Get(i,df.m_NVars));
}
}
//--- return result
return(result/(npoints*df.m_NClasses));
}
//+------------------------------------------------------------------+
//| Average relative error on the test set |
//| INPUT PARAMETERS: |
//| DF - decision forest model |
//| XY - test set |
//| NPoints - test set size |
//| RESULT: |
//| Its meaning for regression task is obvious. As for |
//| classification task, it means average relative error when |
//| estimating posterior probability of belonging to the correct |
//| class. |
//+------------------------------------------------------------------+
double CDForest::DFAvgRelError(CDecisionForest &df,CMatrixDouble &xy,
const int npoints)
{
//--- create variables
double result=0;
int relcnt=0;
//--- creating arrays
CRowDouble x;
CRowDouble y;
//--- initialization
for(int i=0; i<npoints; i++)
{
//--- copy
x=xy[i]+0;
x.Resize(df.m_NVars);
//--- function call
DFProcess(df,x,y);
//--- check
if(df.m_NClasses>1)
{
//--- classification-specific code
int k=(int)MathRound(xy.Get(i,df.m_NVars));
if(k<0 || k>=df.m_NVars)
continue;
result+=MathAbs(y[k]-1);
relcnt++;
}
else
{
//--- regression-specific code
if(xy.Get(i,df.m_NVars)!=0.0)
{
result+=MathAbs((y[0]-xy.Get(i,df.m_NVars))/xy.Get(i,df.m_NVars));
relcnt ++;
}
}
}
//--- check
if(relcnt>0)
result=result/relcnt;
//--- return result
return(result);
}
//+------------------------------------------------------------------+
//| Copying of DecisionForest strucure |
//| INPUT PARAMETERS: |
//| DF1 - original |
//| OUTPUT PARAMETERS: |
//| DF2 - copy |
//+------------------------------------------------------------------+
void CDForest::DFCopy(CDecisionForest &df1,CDecisionForest &df2)
{
if(df1.m_ForestFormat==m_DFUncompressedV0)
{
df2.m_ForestFormat=df1.m_ForestFormat;
df2.m_NVars=df1.m_NVars;
df2.m_NClasses=df1.m_NClasses;
df2.m_NTrees=df1.m_NTrees;
df2.m_BufSize=df1.m_BufSize ;
df2.m_Trees=df1.m_Trees;
DFCreateBuffer(df2,df2.m_Buffer);
return;
}
if(df1.m_ForestFormat==m_DFCompressedV0)
{
df2.m_ForestFormat=df1.m_ForestFormat;
df2.m_NVars=df1.m_NVars;
df2.m_NClasses=df1.m_NClasses;
df2.m_NTrees=df1.m_NTrees;
df2.m_BufSize=df1.m_BufSize ;
df2.m_UseMantissa8=df1.m_UseMantissa8;
df2.m_Trees8=df1.m_Trees8;
DFCreateBuffer(df2,df2.m_Buffer);
return;
}
CAp::Assert(false,__FUNCTION__": unexpected forest format");
}
//+------------------------------------------------------------------+
//| Serializer: allocation |
//+------------------------------------------------------------------+
void CDForest::DFAlloc(CSerializer &s,CDecisionForest &forest)
{
//--- preparation to serialize
if(forest.m_ForestFormat==m_DFUncompressedV0)
{
s.Alloc_Entry();
s.Alloc_Entry();
s.Alloc_Entry();
s.Alloc_Entry();
s.Alloc_Entry();
s.Alloc_Entry();
CApServ::AllocRealArray(s,forest.m_Trees,forest.m_BufSize);
return;
}
if(forest.m_ForestFormat==m_DFCompressedV0)
{
s.Alloc_Entry();
s.Alloc_Entry();
s.Alloc_Entry();
s.Alloc_Entry();
s.Alloc_Entry();
s.Alloc_Entry();
CApServ::AllocIntegerArray(s,forest.m_Trees8,forest.m_BufSize);
return;
}
CAp::Assert(false,__FUNCTION__": unexpected forest format");
}
//+------------------------------------------------------------------+
//| Serializer: serialization |
//+------------------------------------------------------------------+
void CDForest::DFSerialize(CSerializer &s,CDecisionForest &forest)
{
//--- serializetion
if(forest.m_ForestFormat==m_DFUncompressedV0)
{
s.Serialize_Int(CSCodes::GetRDFSerializationCode());
s.Serialize_Int(m_DFUncompressedV0);
s.Serialize_Int(forest.m_NVars);
s.Serialize_Int(forest.m_NClasses);
s.Serialize_Int(forest.m_NTrees);
s.Serialize_Int(forest.m_BufSize);
CApServ::SerializeRealArray(s,forest.m_Trees,forest.m_BufSize);
return;
}
if(forest.m_ForestFormat==m_DFCompressedV0)
{
s.Serialize_Int(CSCodes::GetRDFSerializationCode());
s.Serialize_Int(forest.m_ForestFormat);
s.Serialize_Bool(forest.m_UseMantissa8);
s.Serialize_Int(forest.m_NVars);
s.Serialize_Int(forest.m_NClasses);
s.Serialize_Int(forest.m_NTrees);
CApServ::SerializeIntegerArray(s,forest.m_Trees8,forest.m_Trees8.Size());
return;
}
CAp::Assert(false,__FUNCTION__": unexpected forest format");
}
//+------------------------------------------------------------------+
//| Serializer: unserialization |
//+------------------------------------------------------------------+
void CDForest::DFUnserialize(CSerializer &s,CDecisionForest &forest)
{
bool processed=false;
//--- check correctness of header
int i0=s.Unserialize_Int();
if(!CAp::Assert(i0==CSCodes::GetRDFSerializationCode(),__FUNCTION__": stream header corrupted"))
return;
//--- Read forest
int forestformat=s.Unserialize_Int();
if(forestformat==m_DFUncompressedV0)
{
//--- Unserialize data
forest.m_ForestFormat=forestformat;
forest.m_NVars=s.Unserialize_Int();
forest.m_NClasses=s.Unserialize_Int();
forest.m_NTrees=s.Unserialize_Int();
forest.m_BufSize=s.Unserialize_Int();
CApServ::UnserializeRealArray(s,forest.m_Trees);
processed=true;
}
if(forestformat==m_DFCompressedV0)
{
//--- Unserialize data
forest.m_ForestFormat=forestformat;
forest.m_UseMantissa8=s.Unserialize_Bool();
forest.m_NVars=s.Unserialize_Int();
forest.m_NClasses=s.Unserialize_Int();
forest.m_NTrees=s.Unserialize_Int();
CApServ::UnserializeIntegerArray(s,forest.m_Trees8);
processed=true;
}
if(!CAp::Assert(processed,__FUNCTION__": unexpected forest format"))
return;
//--- Prepare buffer
DFCreateBuffer(forest,forest.m_Buffer);
}
//+------------------------------------------------------------------+
//| This subroutine builds random decision forest. |
//| ---- DEPRECATED VERSION! USE DECISION FOREST BUILDER OBJECT ---- |
//+------------------------------------------------------------------+
void CDForest::DFBuildRandomDecisionForest(CMatrixDouble &xy,
int npoints,
int nvars,
int nclasses,
int ntrees,
double r,
int &info,
CDecisionForest &df,
CDFReport &rep)
{
info=0;
//--- check
if(r<=0.0 || r>1.0)
{
info=-1;
return;
}
int samplesize=MathMax((int)MathRound(r*npoints),1);
DFBuildInternal(xy,npoints,nvars,nclasses,ntrees,samplesize,MathMax(nvars/2,1),m_DFUseStrongSplits+m_DFUseEVS,info,df,rep);
}
//+------------------------------------------------------------------+
//| This subroutine builds random decision forest. |
//| ---- DEPRECATED VERSION! USE DECISION FOREST BUILDER OBJECT ---- |
//+------------------------------------------------------------------+
void CDForest::DFBuildRandomDecisionForestX1(CMatrixDouble &xy,
int npoints,
int nvars,
int nclasses,
int ntrees,
int nrndvars,
double r,
int &info,
CDecisionForest &df,
CDFReport &rep)
{
info=0;
//--- check
if(r<=0.0 || r>1.0)
{
info=-1;
return;
}
if(nrndvars<=0 || nrndvars>nvars)
{
info=-1;
return;
}
int samplesize=MathMax((int)MathRound(r*npoints),1);
DFBuildInternal(xy,npoints,nvars,nclasses,ntrees,samplesize,nrndvars,m_DFUseStrongSplits+m_DFUseEVS,info,df,rep);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CDForest::DFBuildInternal(CMatrixDouble &xy,
int npoints,
int nvars,
int nclasses,
int ntrees,
int samplesize,
int nfeatures,
int flags,
int &info,
CDecisionForest &df,
CDFReport &rep)
{
CDecisionForestBuilder builder;
info=0;
//--- Test for inputs
if(npoints<1 || samplesize<1 || samplesize>npoints || nvars<1 || nclasses<1 || ntrees<1 || nfeatures<1)
{
info=-1;
return;
}
if(nclasses>1)
for(int i=0; i<=npoints-1; i++)
if(MathRound(xy.Get(i,nvars))<0 || MathRound(xy.Get(i,nvars))>=nclasses)
{
info=-2;
return;
}
info=1;
DFBuilderCreate(builder);
DFBuilderSetDataset(builder,xy,npoints,nvars,nclasses);
DFBuilderSetSubsampleRatio(builder,(double)samplesize/(double)npoints);
DFBuilderSetRndVars(builder,nfeatures);
DFBuilderBuildRandomForest(builder,ntrees,df,rep);
}
//+------------------------------------------------------------------+
//| Builds a range of random trees [TreeIdx0, TreeIdx1) using |
//| decision forest algorithm. Tree index is used to seed per - tree |
//| RNG. |
//+------------------------------------------------------------------+
void CDForest::BuildRandomTree(CDecisionForestBuilder &s,int treeidx0,
int treeidx1)
{
//--- create variables
int i=0;
int j=0;
int npoints=0;
int nvars=0;
int nclasses=0;
CHighQualityRandState rs;
int treesize=0;
int varstoselect=0;
int workingsetsize=0;
double meanloss=0;
//--- Prepare
npoints=s.m_NPoints;
nvars=s.m_NVars;
nclasses=s.m_NClasses;
//---
if(treeidx0>treeidx1)
CApServ::Swap(treeidx0,treeidx1);
if(treeidx0==treeidx1)
treeidx1++;
for(int treeidx=treeidx0; treeidx<treeidx1; treeidx++)
{
if(s.m_RDFGlobalSeed>0)
{
CHighQualityRand::HQRndSeed(s.m_RDFGlobalSeed,1+treeidx,rs);
}
else
{
CHighQualityRand::HQRndSeed(CMath::RandomInteger(30000),1+treeidx,rs);
}
//--- Prepare everything for tree construction.
if(!CAp::Assert(s.m_WorkBuf[treeidx].m_trnsize>=1,__FUNCTION__": integrity check failed (34636)"))
return;
if(!CAp::Assert(s.m_WorkBuf[treeidx].m_oobsize>=0,__FUNCTION__": integrity check failed (45745)"))
return;
if(!CAp::Assert(s.m_WorkBuf[treeidx].m_trnsize+s.m_WorkBuf[treeidx].m_oobsize==npoints,__FUNCTION__": integrity check failed (89415)"))
return;
workingsetsize=-1;
s.m_WorkBuf[treeidx].m_varpoolsize=0;
for(i=0; i<nvars; i++)
if(s.m_DSMin[i]!=s.m_DSMax[i])
{
s.m_WorkBuf[treeidx].m_varpool.Set(s.m_WorkBuf[treeidx].m_varpoolsize,i);
s.m_WorkBuf[treeidx].m_varpoolsize++;
}
workingsetsize=s.m_WorkBuf[treeidx].m_varpoolsize;
if(!CAp::Assert(workingsetsize>=0,__FUNCTION__": integrity check failed (73f5)"))
return;
for(i=0; i<npoints; i++)
s.m_WorkBuf[treeidx].m_tmp0i.Set(i,i);
for(i=0; i<s.m_WorkBuf[treeidx].m_trnsize; i++)
{
j=CHighQualityRand::HQRndUniformI(rs,npoints-i);
s.m_WorkBuf[treeidx].m_tmp0i.Swap(i,i+j);
s.m_WorkBuf[treeidx].m_trnset.Set(i,s.m_WorkBuf[treeidx].m_tmp0i[i]);
if(nclasses>1)
s.m_WorkBuf[treeidx].m_trnlabelsi.Set(i,s.m_DSIVal[s.m_WorkBuf[treeidx].m_tmp0i[i]]);
else
s.m_WorkBuf[treeidx].m_trnlabelsr.Set(i,s.m_DSRVal[s.m_WorkBuf[treeidx].m_tmp0i[i]]);
if(s.m_NeedIOBMatrix)
s.m_IOBMatrix.Set(treeidx,s.m_WorkBuf[treeidx].m_trnset[i],(int)true);
}
for(i=0; i<s.m_WorkBuf[treeidx].m_oobsize; i++)
{
j=s.m_WorkBuf[treeidx].m_tmp0i[s.m_WorkBuf[treeidx].m_trnsize+i];
s.m_WorkBuf[treeidx].m_oobset.Set(i,j);
if(nclasses>1)
s.m_WorkBuf[treeidx].m_ooblabelsi.Set(i,s.m_DSIVal[j]);
else
s.m_WorkBuf[treeidx].m_ooblabelsr.Set(i,s.m_DSRVal[j]);
}
varstoselect=(int)MathRound(MathSqrt(nvars));
if(s.m_RDFVars>0.0)
varstoselect=(int)MathRound(s.m_RDFVars);
else
if(s.m_RDFVars<0.0)
{
varstoselect=(int)MathRound(-(nvars*s.m_RDFVars));
}
varstoselect=MathMax(varstoselect,1);
varstoselect=MathMin(varstoselect,nvars);
//--- Perform recurrent construction
if(s.m_RDFImportance==m_NeedTrnGini)
meanloss=MeanNRMS2(nclasses,s.m_WorkBuf[treeidx].m_trnlabelsi,s.m_WorkBuf[treeidx].m_trnlabelsr,0,s.m_WorkBuf[treeidx].m_trnsize,s.m_WorkBuf[treeidx].m_trnlabelsi,s.m_WorkBuf[treeidx].m_trnlabelsr,0,s.m_WorkBuf[treeidx].m_trnsize,s.m_WorkBuf[treeidx].m_tmpnrms2);
else
meanloss=MeanNRMS2(nclasses,s.m_WorkBuf[treeidx].m_trnlabelsi,s.m_WorkBuf[treeidx].m_trnlabelsr,0,s.m_WorkBuf[treeidx].m_trnsize,s.m_WorkBuf[treeidx].m_ooblabelsi,s.m_WorkBuf[treeidx].m_ooblabelsr,0,s.m_WorkBuf[treeidx].m_oobsize,s.m_WorkBuf[treeidx].m_tmpnrms2);
treesize=1;
BuildRandomTreeRec(s,s.m_WorkBuf[treeidx],workingsetsize,varstoselect,s.m_WorkBuf[treeidx].m_treebuf,s.m_VoteBuf[treeidx],rs,0,s.m_WorkBuf[treeidx].m_trnsize,0,s.m_WorkBuf[treeidx].m_oobsize,meanloss,meanloss,treesize);
s.m_WorkBuf[treeidx].m_treebuf.Set(0,treesize);
//--- Store tree
s.m_TreeBuf[treeidx].m_treebuf=s.m_WorkBuf[treeidx].m_treebuf;
s.m_TreeBuf[treeidx].m_treeidx=treeidx;
}
}
//+------------------------------------------------------------------+
//| Recurrent tree construction function using caller - allocated |
//| buffers and caller - initialized RNG. |
//| Following iterms are processed: |
//| * items [Idx0, Idx1) of WorkBuf.TrnSet |
//| * items [OOBIdx0, OOBIdx1) of WorkBuf.OOBSet |
//| TreeSize on input must be 1(header element of the tree), on |
//| output it contains size of the tree. |
//| OOBLoss on input must contain value of MeanNRMS2(...) computed |
//| for entire dataset. |
//| Variables from #0 to #WorkingSet - 1 from WorkBuf.VarPool are |
//| used(for block algorithm: blocks, not vars) |
//+------------------------------------------------------------------+
void CDForest::BuildRandomTreeRec(CDecisionForestBuilder &s,
CDFWorkBuf &workbuf,
int workingset,
int varstoselect,
CRowDouble &treebuf,
CDFVoteBuf &votebuf,
CHighQualityRandState &rs,
int idx0,
int idx1,
int oobidx0,
int oobidx1,
double meanloss,
double topmostmeanloss,
int &treesize)
{
//---create variables
int npoints=0;
int nclasses=0;
int i=0;
int j=0;
int j0=0;
double v=0;
bool labelsaresame;
int offs=0;
int varbest=0;
double splitbest=0;
int i1=0;
int i2=0;
int idxtrn=0;
int idxoob=0;
double meanloss0=0;
double meanloss1=0;
//--- check
if(!CAp::Assert(s.m_DSType==0,__FUNCTION__": not supported skbdgfsi!"))
return;
if(!CAp::Assert(idx0<idx1,__FUNCTION__": integrity check failed (3445)"))
return;
if(!CAp::Assert(oobidx0<=oobidx1,__FUNCTION__": integrity check failed (7452)"))
return;
npoints=s.m_NPoints;
nclasses=s.m_NClasses;
//--- Check labels: all same or not?
if(nclasses>1)
{
labelsaresame=true;
workbuf.m_classpriors.Fill(0);
j0=workbuf.m_trnlabelsi[idx0];
for(i=idx0; i<idx1; i++)
{
j=workbuf.m_trnlabelsi[i];
workbuf.m_classpriors.Add(j,1);
labelsaresame=labelsaresame && j0==j;
}
}
else
labelsaresame=false;
//--- Leaf node
if(idx1-idx0==1 || labelsaresame)
{
if(nclasses==1)
OutputLeaf(s,workbuf,treebuf,votebuf,idx0,idx1,oobidx0,oobidx1,treesize,workbuf.m_trnlabelsr[idx0]);
else
OutputLeaf(s,workbuf,treebuf,votebuf,idx0,idx1,oobidx0,oobidx1,treesize,workbuf.m_trnlabelsi[idx0]);
return;
}
//--- Non-leaf node.
//--- Investigate possible splits.
if(!CAp::Assert(s.m_RDFAlgo==0,__FUNCTION__": unexpected algo"))
return;
ChooseCurrentSplitDense(s,workbuf,workingset,varstoselect,rs,idx0,idx1,varbest,splitbest);
if(varbest<0)
{
//--- No good split was found; make leaf (label is randomly chosen) and exit.
if(nclasses>1)
v=workbuf.m_trnlabelsi[idx0+CHighQualityRand::HQRndUniformI(rs,idx1-idx0)];
else
v=workbuf.m_trnlabelsr[idx0+CHighQualityRand::HQRndUniformI(rs,idx1-idx0)];
OutputLeaf(s,workbuf,treebuf,votebuf,idx0,idx1,oobidx0,oobidx1,treesize,v);
return;
}
//--- Good split WAS found, we can perform it:
//--- * first, we split training set
//--- * then, we similarly split OOB set
if(!CAp::Assert(s.m_DSType==0,__FUNCTION__": not supported 54bfdh"))
return;
offs=npoints*varbest;
i1=idx0;
i2=idx1-1;
while(i1<=i2)
{
//--- Reorder indexes so that left partition is in [Idx0..I1),
//--- and right partition is in [I2+1..Idx1)
if(workbuf.m_bestvals[i1]<splitbest)
{
i1 ++;
continue;
}
if(workbuf.m_bestvals[i2]>=splitbest)
{
i2--;
continue;
}
workbuf.m_trnset.Swap(i1,i2);
if(nclasses>1)
workbuf.m_trnlabelsi.Swap(i1,i2);
else
workbuf.m_trnlabelsr.Swap(i1,i2);
i1++;
i2--;
}
if(!CAp::Assert(i1==i2+1,__FUNCTION__": integrity check failed (45rds3)"))
return;
idxtrn=i1;
if(oobidx0<oobidx1)
{
//--- Unlike the training subset, the out-of-bag subset corresponding to the
//--- current sequence of decisions can be empty; thus, we have to explicitly
//--- handle situation of zero OOB subset.
i1=oobidx0;
i2=oobidx1-1;
while(i1<=i2)
{
//--- Reorder indexes so that left partition is in [Idx0..I1),
//--- and right partition is in [I2+1..Idx1)
if(s.m_DSData[offs+workbuf.m_oobset[i1]]<splitbest)
{
i1++;
continue;
}
if(s.m_DSData[offs+workbuf.m_oobset[i2]]>=splitbest)
{
i2--;
continue;
}
workbuf.m_oobset.Swap(i1,i2);
if(nclasses>1)
workbuf.m_ooblabelsi.Swap(i1,i2);
else
workbuf.m_ooblabelsr.Swap(i1,i2);
i1++;
i2--;
}
if(!CAp::Assert(i1==i2+1,__FUNCTION__": integrity check failed (643fs3)"))
return;
idxoob=i1;
}
else
idxoob=oobidx0;
//--- Compute estimates of NRMS2 loss over TRN or OOB subsets, update Gini importances
if(s.m_RDFImportance==m_NeedTrnGini)
{
meanloss0=MeanNRMS2(nclasses,workbuf.m_trnlabelsi,workbuf.m_trnlabelsr,idx0,idxtrn,workbuf.m_trnlabelsi,workbuf.m_trnlabelsr,idx0,idxtrn,workbuf.m_tmpnrms2);
meanloss1=MeanNRMS2(nclasses,workbuf.m_trnlabelsi,workbuf.m_trnlabelsr,idxtrn,idx1,workbuf.m_trnlabelsi,workbuf.m_trnlabelsr,idxtrn,idx1,workbuf.m_tmpnrms2);
}
else
{
meanloss0=MeanNRMS2(nclasses,workbuf.m_trnlabelsi,workbuf.m_trnlabelsr,idx0,idxtrn,workbuf.m_ooblabelsi,workbuf.m_ooblabelsr,oobidx0,idxoob,workbuf.m_tmpnrms2);
meanloss1=MeanNRMS2(nclasses,workbuf.m_trnlabelsi,workbuf.m_trnlabelsr,idxtrn,idx1,workbuf.m_ooblabelsi,workbuf.m_ooblabelsr,idxoob,oobidx1,workbuf.m_tmpnrms2);
}
votebuf.m_giniimportances.Add(varbest,(meanloss-(meanloss0+meanloss1))/(topmostmeanloss+1.0e-20));
//--- Generate tree node and subtrees (recursively)
treebuf.Set(treesize,varbest);
treebuf.Set(treesize+1,splitbest);
i=treesize;
treesize=treesize+m_InnerNodeWidth;
BuildRandomTreeRec(s,workbuf,workingset,varstoselect,treebuf,votebuf,rs,idx0,idxtrn,oobidx0,idxoob,meanloss0,topmostmeanloss,treesize);
treebuf.Set(i+2,treesize);
BuildRandomTreeRec(s,workbuf,workingset,varstoselect,treebuf,votebuf,rs,idxtrn,idx1,idxoob,oobidx1,meanloss1,topmostmeanloss,treesize);
}
//+------------------------------------------------------------------+
//| Estimates permutation variable importance ratings for a range of |
//| dataset points. |
//| Initial call to this function should span entire range of the |
//| dataset, [Idx0, Idx1) = [0, NPoints), because function performs |
//| initialization of some internal structures when called with these|
//| arguments. |
//+------------------------------------------------------------------+
void CDForest::EstimateVariableImportance(CDecisionForestBuilder &s,
int sessionseed,
CDecisionForest &df,
int ntrees,
CDFReport &rep)
{
//--- create variables
int npoints=0;
int nvars=0;
int nclasses=0;
int nperm=0;
int i=0;
int j=0;
int k=0;
CRowDouble tmpr0;
CRowDouble tmpr1;
CRowInt tmpi0;
vector<double> losses;
CDFPermimpBuf permseed;
double nopermloss=0;
double totalpermloss=0;
CHighQualityRandState varimprs;
npoints=s.m_NPoints;
nvars=s.m_NVars;
nclasses=s.m_NClasses;
//--- No importance rating
if(s.m_RDFImportance==0)
return;
//--- Gini importance
if(s.m_RDFImportance==m_NeedTrnGini || s.m_RDFImportance==m_NeedOOBGini)
{
//--- Merge OOB Gini importances computed during tree generation
int total=ArraySize(s.m_VoteBuf);
losses=s.m_VoteBuf[0].m_giniimportances.ToVector();
for(int vote=1; vote<total; vote++)
losses+=s.m_VoteBuf[vote].m_giniimportances.ToVector();
losses.Resize(nvars);
losses=losses/(double)ntrees;
losses.Clip(0,1);
rep.m_varimportances=losses;
//--- Compute topvars[] array
tmpr0=losses*(-1);
for(i=0; i<nvars; i++)
rep.m_topvars.Set(i,i);
CTSort::TagSortFastI(tmpr0,rep.m_topvars,tmpr1,tmpi0,nvars);
return;
}
//--- Permutation importance
if(s.m_RDFImportance==m_NeedPermutation)
{
if(!CAp::Assert(df.m_ForestFormat==m_DFUncompressedV0,__FUNCTION__": integrity check failed (ff)"))
return;
if(!CAp::Assert(s.m_IOBMatrix.Rows()>=ntrees && s.m_IOBMatrix.Cols()>=npoints,__FUNCTION__": integrity check failed (IOB)"))
return;
//--- Generate packed representation of the shuffle which is applied to all variables
//--- Ideally we want to apply different permutations to different variables,
//--- i.e. we have to generate and store NPoints*NVars random numbers.
//--- However due to performance and memory restrictions we prefer to use compact
//--- representation:
//---*we store one "reference" permutation P_ref in VarImpShuffle2[0:NPoints-1]
//--- * a permutation P_j applied to variable J is obtained by circularly shifting
//--- elements in P_ref by VarImpShuffle2[NPoints+J]
CHighQualityRand::HQRndSeed(sessionseed,1117,varimprs);
s.m_VarImpShuffle2.Resize(npoints+nvars);
for(i=0; i<npoints; i++)
s.m_VarImpShuffle2.Set(i,i);
for(i=0; i<npoints-1; i++)
{
j=i+CHighQualityRand::HQRndUniformI(varimprs,npoints-i);
s.m_VarImpShuffle2.Swap(i,j);
}
for(i=0; i<nvars; i++)
s.m_VarImpShuffle2.Set(npoints+i,CHighQualityRand::HQRndUniformI(varimprs,npoints));
//--- Prepare buffer object, seed pool
nperm=nvars+2;
permseed.m_losses=vector<double>::Zeros(nperm);
permseed.m_yv.Resize(nperm*nclasses);
permseed.m_xraw.Resize(nvars);
permseed.m_xdist.Resize(nvars);
permseed.m_xcur.Resize(nvars);
permseed.m_targety.Resize(nclasses);
permseed.m_startnodes.Resize(nvars);
permseed.m_y.Resize(nclasses);
//--- Recursively split subset
EstimatePermutationImportances(s,df,ntrees,permseed,0,npoints);
//--- Merge results
losses=permseed.m_losses+vector<double>::Full(nperm,1.0e-20);
//--- Compute importances
nopermloss=losses[nvars+1];
totalpermloss=losses[nvars];
losses= losses/totalpermloss-nopermloss/totalpermloss;
losses.Clip(0,1);
losses.Resize(nvars);
rep.m_varimportances=losses;
//--- Compute topvars[] array
tmpr0=losses*(-1);
rep.m_topvars.Resize(nvars);
for(j=0; j<nvars; j++)
rep.m_topvars.Set(j,j);
CTSort::TagSortFastI(tmpr0,rep.m_topvars,tmpr1,tmpi0,nvars);
return;
}
CAp::Assert(false,__FUNCTION__": unexpected importance type");
}
//+------------------------------------------------------------------+
//| Estimates permutation variable importance ratings for a range of |
//| dataset points. |
//| Initial call to this function should span entire range of the |
//| dataset, [Idx0, Idx1) = [0, NPoints), because function performs |
//| initialization of some internal structures when called with these|
//| arguments. |
//+------------------------------------------------------------------+
void CDForest::EstimatePermutationImportances(CDecisionForestBuilder &s,
CDecisionForest &df,
int ntrees,
CDFPermimpBuf &permimpbuf,
int idx0,
int idx1)
{
//--- create variables
int nperm=0;
int i=0;
int j=0;
int k=0;
double v=0;
int treeroot=0;
int nodeoffs=0;
double prediction=0;
int varidx=0;
int oobcounts=0;
int srcidx=0;
int npoints=s.m_NPoints;
int nvars=s.m_NVars;
int nclasses=s.m_NClasses;
//--- check
if(!CAp::Assert(df.m_ForestFormat==m_DFUncompressedV0,__FUNCTION__": integrity check failed (ff)"))
return;
if(!CAp::Assert(idx0>=0 && idx0<=idx1 && idx1<=npoints,__FUNCTION__": integrity check failed (idx)"))
return;
if(!CAp::Assert(s.m_IOBMatrix.Rows()>=ntrees && s.m_IOBMatrix.Cols()>=npoints,__FUNCTION__": integrity check failed (IOB)"))
return;
//--- main loop
nperm=nvars+2;
//--- Process range of points [idx0,idx1)
for(i=idx0; i<idx1-1; i++)
{
if(!CAp::Assert(s.m_DSType==0,__FUNCTION__": unexpected dataset type"))
return;
for(j=0; j<nvars; j++)
{
permimpbuf.m_xraw.Set(j,s.m_DSData[j*npoints+i]);
srcidx=s.m_VarImpShuffle2[(i+s.m_VarImpShuffle2[npoints+j])%npoints];
permimpbuf.m_xdist.Set(j,s.m_DSData[j*npoints+srcidx]);
}
if(nclasses>1)
{
permimpbuf.m_targety=vector<double>::Zeros(nclasses);
permimpbuf.m_targety.Set(s.m_DSIVal[i],1);
}
else
permimpbuf.m_targety.Set(0,s.m_DSRVal[i]);
//--- Process all trees, for each tree compute NPerm losses corresponding
//--- to various permutations of variable values
permimpbuf.m_yv=vector<double>::Zeros(nperm*nclasses);
oobcounts=0;
treeroot=0;
for(k=0; k<ntrees; k++)
{
if(!s.m_IOBMatrix.Get(k,i))
{
//--- Process original (unperturbed) point and analyze path from the
//--- tree root to the final leaf. Output prediction to RawPrediction.
//--- Additionally, for each variable in [0,NVars-1] save offset of
//--- the first split on this variable. It allows us to quickly compute
//--- tree decision when perturbation does not change decision path.
if(!CAp::Assert(df.m_ForestFormat==m_DFUncompressedV0,__FUNCTION__": integrity check failed (ff)"))
return;
nodeoffs=treeroot+1;
permimpbuf.m_startnodes.Fill(-1);
prediction=0;
while(true)
{
if(df.m_Trees[nodeoffs]==-1.0)
{
prediction=df.m_Trees[nodeoffs+1];
break;
}
j=(int)MathRound(df.m_Trees[nodeoffs]);
if(permimpbuf.m_startnodes[j]<0)
permimpbuf.m_startnodes.Set(j,nodeoffs);
if(permimpbuf.m_xraw[j]<df.m_Trees[nodeoffs+1])
nodeoffs+=m_InnerNodeWidth;
else
nodeoffs=treeroot+(int)MathRound(df.m_Trees[nodeoffs+2]);
}
//--- Save loss for unperturbed point
varidx=nvars+1;
if(nclasses>1)
{
j=(int)MathRound(prediction);
permimpbuf.m_yv.Add(varidx*nclasses+j,1);
}
else
permimpbuf.m_yv.Add(varidx,prediction);
//--- Save loss for all variables being perturbed (XDist).
//--- This loss is used as a reference loss when we compute R-squared.
varidx=nvars;
permimpbuf.m_y=vector<double>::Zeros(nclasses);
DFProcessInternalUncompressed(df,treeroot,treeroot+1,permimpbuf.m_xdist,permimpbuf.m_y);
permimpbuf.m_yv.Add(varidx*nclasses+j,permimpbuf.m_y[j]);
//--- Compute losses for variable #VarIdx being perturbed. Quite an often decision
//--- process does not actually depend on the variable #VarIdx (path from the tree
//--- root does not include splits on this variable). In such cases we perform
//--- quick exit from the loop with precomputed value.
permimpbuf.m_xcur=permimpbuf.m_xraw;
for(varidx=0; varidx<nvars; varidx++)
{
if(permimpbuf.m_startnodes[varidx]>=0)
{
//--- Path from tree root to the final leaf involves split on variable #VarIdx.
//--- Restart computation from the position first split on #VarIdx.
if(!CAp::Assert(df.m_ForestFormat==m_DFUncompressedV0,__FUNCTION__": integrity check failed (ff)"))
return;
permimpbuf.m_xcur.Set(varidx,permimpbuf.m_xdist[varidx]);
nodeoffs=permimpbuf.m_startnodes[varidx];
while(true)
{
if(df.m_Trees[nodeoffs]==-1.0)
{
if(nclasses>1)
{
j=(int)MathRound(df.m_Trees[nodeoffs+1]);
permimpbuf.m_yv.Add(varidx*nclasses+j,1.0);
}
else
permimpbuf.m_yv.Add(varidx,df.m_Trees[nodeoffs+1]);
break;
}
j=(int)MathRound(df.m_Trees[nodeoffs]);
if(permimpbuf.m_xcur[j]<df.m_Trees[nodeoffs+1])
nodeoffs+=m_InnerNodeWidth;
else
nodeoffs=treeroot+(int)MathRound(df.m_Trees[nodeoffs+2]);
}
permimpbuf.m_xcur.Set(varidx,permimpbuf.m_xraw[varidx]);
}
else
{
//--- Path from tree root to the final leaf does NOT involve split on variable #VarIdx.
//--- Permutation does not change tree output, reuse already computed value.
if(nclasses>1)
{
j=(int)MathRound(prediction);
permimpbuf.m_yv.Add(varidx*nclasses+j,1.0);
}
else
permimpbuf.m_yv.Add(varidx,prediction);
}
}
//--- update OOB counter
oobcounts++;
}
treeroot+=(int)MathRound(df.m_Trees[treeroot]);
}
//--- Now YV[] stores NPerm versions of the forest output for various permutations of variable values.
//--- Update losses.
for(j=0; j<nperm; j++)
{
if(oobcounts!=0)
for(k=0; k<nclasses ; k++)
permimpbuf.m_yv.Mul(j*nclasses+k,1.0/(double)oobcounts);
v=0;
for(k=0; k<=nclasses-1; k++)
v=v+CMath::Sqr(permimpbuf.m_yv[j*nclasses+k]-permimpbuf.m_targety[k]);
permimpbuf.m_losses.Add(j,v);
}
}
}
//+------------------------------------------------------------------+
//| Sets report fields to their default values |
//+------------------------------------------------------------------+
void CDForest::CleanReport(CDecisionForestBuilder &s,CDFReport &rep)
{
rep.m_RelCLSError=0;
rep.m_AvgCE=0;
rep.m_RMSError=0;
rep.m_AvgError=0;
rep.m_AvgRelError=0;
rep.m_oobrelclserror=0;
rep.m_oobavgce=0;
rep.m_oobrmserror=0;
rep.m_oobavgerror=0;
rep.m_oobavgrelerror=0;
rep.m_topvars.Resize(s.m_NVars);
rep.m_varimportances=vector<double>::Zeros(s.m_NVars);
for(int i=0; i<=s.m_NVars-1; i++)
rep.m_topvars.Set(i,i);
}
//+------------------------------------------------------------------+
//| This function returns NRMS2 loss(sum of squared residuals) for a |
//| constant - output model: |
//| * model output is a mean over TRN set being passed (for |
//| classification problems - NClasses - dimensional vector |
//| of class probabilities) |
//| * model is evaluated over TST set being passed, with L2 loss |
//| being returned |
//| Input parameters: |
//| NClasses - ">1" for classification, "=1" for regression |
//| TrnLabelsI - training set labels, class indexes (for |
//| NClasses > 1) |
//| TrnLabelsR - training set output values (for NClasses = 1) |
//| TrnIdx0, |
//| TrnIdx1 - a range [Idx0, Idx1) of elements in LabelsI/R |
//| is considered |
//| TstLabelsI - training set labels, class indexes (for |
//| NClasses > 1) |
//| TstLabelsR - training set output values (for NClasses = 1) |
//| TstIdx0, |
//| TstIdx1 - a range [Idx0, Idx1) of elements in LabelsI/R |
//| is considered |
//| TmpI - temporary array, reallocated as needed |
//| Result: sum of squared residuals; for NClasses >= 2 it coincides |
//| with Gini impurity times(Idx1 - Idx0) |
//| Following fields of WorkBuf are used as temporaries: |
//| * TmpMeanNRMS2 |
//+------------------------------------------------------------------+
double CDForest::MeanNRMS2(int nclasses,CRowInt &trnlabelsi,
CRowDouble &trnlabelsr,int trnidx0,
int trnidx1,CRowInt &tstlabelsi,
CRowDouble &tstlabelsr,int tstidx0,
int tstidx1,CRowInt &tmpi)
{
//--- create variables
double result=0;
int i=0;
int k=0;
int ntrn=trnidx1-trnidx0;
int ntst=tstidx1-tstidx0;
double v=0;
double vv=0;
double invntrn=0;
double pitrn=0;
double nitst=0;
//--- check
if(!CAp::Assert(trnidx0<=trnidx1,__FUNCTION__": integrity check failed (8754)"))
return(result);
if(!CAp::Assert(tstidx0<=tstidx1,__FUNCTION__": integrity check failed (8754)"))
return(result);
//--- Quick exit
if(ntrn==0 || ntst==0)
return(result);
invntrn=1.0/(double)ntrn;
if(nclasses>1)
{
//--- Classification problem
tmpi.Resize(2*nclasses);
tmpi.Fill(0);
for(i=trnidx0; i<trnidx1; i++)
tmpi.Add(trnlabelsi[i],1);
for(i=tstidx0; i<tstidx1; i++)
tmpi.Add(tstlabelsi[i]+nclasses,1);
for(i=0; i<nclasses; i++)
{
pitrn=tmpi[i]*invntrn;
nitst=tmpi[i+nclasses];
result+=nitst+pitrn*(ntst*pitrn-2*nitst);
}
}
else
{
//--- regression-specific code
v=0;
for(i=trnidx0; i<trnidx1; i++)
v+=trnlabelsr[i];
v=v*invntrn;
for(i=tstidx0; i<tstidx1 ; i++)
{
vv=tstlabelsr[i]-v;
result+=vv*vv;
}
}
//--- return result
return(result);
}
//+------------------------------------------------------------------+
//| This function is a part of the recurrent tree construction |
//| function; it selects variable for splitting according to current |
//| tree construction algorithm. |
//| Note: modifies VarsInPool, may decrease it if some variables |
//| become non-informative and leave the pool. |
//+------------------------------------------------------------------+
void CDForest::ChooseCurrentSplitDense(CDecisionForestBuilder &s,
CDFWorkBuf &workbuf,
int &varsinpool,
int varstoselect,
CHighQualityRandState &rs,
int idx0,
int idx1,
int &varbest,
double &splitbest)
{
//--- create variables
int npoints=0;
double errbest=0;
int varstried=0;
int varcur=0;
bool valuesaresame;
int offs=0;
double split=0;
int i=0;
double v=0;
double v0=0;
double currms=0;
int info=0;
varbest=0;
splitbest=0;
//---check
if(!CAp::Assert(s.m_DSType==0,__FUNCTION__": sparsity is not supported 4terg!"))
return;
if(!CAp::Assert(s.m_RDFAlgo==0,__FUNCTION__": integrity check failed (1657)"))
return;
if(!CAp::Assert(idx0<idx1,__FUNCTION__": integrity check failed (3445)"))
return;
npoints=s.m_NPoints;
//--- Select split according to dense direct RDF algorithm
varbest=-1;
errbest=CMath::m_maxrealnumber;
splitbest=0;
varstried=0;
while(varstried<MathMin(varstoselect,varsinpool))
{
//--- select variables from pool
workbuf.m_varpool.Swap(varstried,varstried+CHighQualityRand::HQRndUniformI(rs,varsinpool-varstried));
varcur=workbuf.m_varpool[varstried];
//--- Load variable values to working array.
//--- If all variable values are same, variable is excluded from pool and we re-run variable selection.
valuesaresame=true;
if(!CAp::Assert(s.m_DSType==0,"not supported segsv34fs"))
return;
offs=npoints*varcur;
v0=s.m_DSData[offs+workbuf.m_trnset[idx0]];
for(i=idx0; i<idx1; i++)
{
v=s.m_DSData[offs+workbuf.m_trnset[i]];
workbuf.m_curvals.Set(i,v);
valuesaresame=valuesaresame && v==v0;
}
if(valuesaresame)
{
//--- Variable does not change across current subset.
//--- Exclude variable from pool, go to the next iteration.
//--- VarsTried is not increased.
//--- NOTE: it is essential that updated VarsInPool is passed
//--- down to children but not up to caller - it is
//--- possible that one level higher this variable is
//--- not-fixed.
workbuf.m_varpool.Swap(varstried,varsinpool-1);
varsinpool=varsinpool-1;
continue;
}
//--- Now we are ready to infer the split
EvaluateDenseSplit(s,workbuf,rs,varcur,idx0,idx1,info,split,currms);
if(info>0 && (varbest<0 || currms<=errbest))
{
errbest=currms;
varbest=varcur;
splitbest=split;
for(i=idx0; i<idx1; i++)
workbuf.m_bestvals.Set(i,workbuf.m_curvals[i]);
}
//--- Next iteration
varstried=varstried+1;
}
}
//+------------------------------------------------------------------+
//| This function performs split on some specific dense variable |
//| whose values are stored in WorkBuf.CurVals[Idx0, Idx1) and labels|
//| are stored in WorkBuf.TrnLabelsR / I[Idx0, Idx1). |
//| It returns split value and associated RMS error. It is |
//| responsibility of the caller to make sure that variable has at |
//| least two distinct values, i.e. it is possible to make a split. |
//| Precomputed values of following fields of WorkBuf are used: |
//| * ClassPriors |
//| Following fields of WorkBuf are used as temporaries: |
//| * ClassTotals0, 1, 01 |
//| * Tmp0I, Tmp1I, Tmp0R, Tmp1R, Tmp2R, Tmp3R |
//+------------------------------------------------------------------+
void CDForest::EvaluateDenseSplit(CDecisionForestBuilder &s,
CDFWorkBuf &workbuf,
CHighQualityRandState &rs,
int splitvar,
int idx0,
int idx1,
int &info,
double &split,
double &rms)
{
//--- create variables
int nclasses=0;
int i=0;
int j=0;
int k0=0;
int k1=0;
double v=0;
double v0=0;
double v1=0;
double v2=0;
int sl=0;
int sr=0;
info=0;
split=0;
rms=0;
if(!CAp::Assert(idx0<idx1,__FUNCTION__": integrity check failed (8754)"))
return;
nclasses=s.m_NClasses;
if(s.m_DSBinary[splitvar])
{
//--- Try simple binary split, if possible
//--- Split can be inferred from minimum/maximum values, just calculate RMS error
info=1;
split=GetSplit(s,s.m_DSMin[splitvar],s.m_DSMax[splitvar],rs);
if(nclasses>1)
{
//--- Classification problem
workbuf.m_classtotals0.Fill(0);
sl=0;
for(i=idx0; i<idx1; i++)
{
if(workbuf.m_curvals[i]<split)
{
j=workbuf.m_trnlabelsi[i];
workbuf.m_classtotals0.Add(j,1);
sl++;
}
}
sr=idx1-idx0-sl;
if(!CAp::Assert(sl!=0 && sr!=0,__FUNCTION__": something strange,impossible failure!"))
return;
v0=1.0/(double)sl;
v1=1.0/(double)sr;
rms=0;
for(j=0; j<nclasses; j++)
{
k0=workbuf.m_classtotals0[j];
k1=workbuf.m_classpriors[j]-k0;
rms+=k0*(1-v0*k0)+k1*(1-v1*k1);
}
rms=MathSqrt(rms/(nclasses*(idx1-idx0+1)));
}
else
{
//--- regression-specific code
sl=0;
sr=0;
v1=0;
v2=0;
for(j=idx0; j<idx1; j++)
{
if(workbuf.m_curvals[j]<split)
{
v1+=workbuf.m_trnlabelsr[j];
sl++;
}
else
{
v2+=workbuf.m_trnlabelsr[j];
sr++;
}
}
if(!CAp::Assert(sl!=0 && sr!=0,__FUNCTION__": something strange,impossible failure!"))
return;
v1=v1/sl;
v2=v2/sr;
rms=0;
for(j=0; j<idx1-idx0; j++)
{
v=workbuf.m_trnlabelsr[idx0+j];
if(workbuf.m_curvals[j]<split)
v-=v1;
else
v-=v2;
rms+=v*v;
}
rms=MathSqrt(rms/(idx1-idx0+1));
}
}
else
{
//--- General split
info=0;
if(nclasses>1)
{
for(i=0; i<idx1-idx0; i++)
{
workbuf.m_tmp0r.Set(i,workbuf.m_curvals[idx0+i]);
workbuf.m_tmp0i.Set(i,workbuf.m_trnlabelsi[idx0+i]);
}
ClassifierSplit(s,workbuf,workbuf.m_tmp0r,workbuf.m_tmp0i,idx1-idx0,rs,info,split,rms,workbuf.m_tmp1r,workbuf.m_tmp1i);
}
else
{
for(i=0; i<idx1-idx0; i++)
{
workbuf.m_tmp0r.Set(i,workbuf.m_curvals[idx0+i]);
workbuf.m_tmp1r.Set(i,workbuf.m_trnlabelsr[idx0+i]);
}
RegressionSplit(s,workbuf,workbuf.m_tmp0r,workbuf.m_tmp1r,idx1-idx0,info,split,rms,workbuf.m_tmp2r,workbuf.m_tmp3r);
}
}
}
//+------------------------------------------------------------------+
//| Classifier split |
//+------------------------------------------------------------------+
void CDForest::ClassifierSplit(CDecisionForestBuilder &s,
CDFWorkBuf &workbuf,
CRowDouble &x,
CRowInt &c,
int n,
CHighQualityRandState &rs,
int &info,
double &threshold,
double &e,
CRowDouble &sortrbuf,
CRowInt &sortibuf)
{
//--- create variables
int i=0;
int j=0;
int k=0;
int n0=0;
int n0prev=0;
double v=0;
int advanceby=0;
double rms=0;
int k0=0;
int k1=0;
double v0=0;
double v1=0;
int nclasses=0;
double vmin=0;
double vmax=0;
info=0;
threshold=0;
e=0;
//--- check
if(!CAp::Assert(s.m_RDFSplitStrength==0 || s.m_RDFSplitStrength==1 || s.m_RDFSplitStrength==2,__FUNCTION__": unexpected split type at ClassifierSplit()"))
return;
nclasses=s.m_NClasses;
advanceby=1;
if(n>=20)
advanceby=MathMax(2,(int)MathRound(n*0.05));
info=-1;
threshold=0;
e=CMath::m_maxrealnumber;
//--- Random split
if(s.m_RDFSplitStrength==0)
{
//--- Evaluate minimum, maximum and randomly selected values
vmin=x[0];
vmax=x[0];
for(i=1; i<n; i++)
{
v=x[i];
if(v<vmin)
vmin=v;
if(v>vmax)
vmax=v;
}
if(vmin==vmax)
return;
v=x[CHighQualityRand::HQRndUniformI(rs,n)];
if(v==vmin)
v=vmax;
//--- Calculate RMS error associated with the split
workbuf.m_classtotals0.Fill(0,0,nclasses);
n0=0;
for(i=0; i<n; i++)
if(x[i]<v)
{
k=c[i];
workbuf.m_classtotals0.Add(k,1);
n0++;
}
if(!CAp::Assert(n0>0 && n0<n,__FUNCTION__": critical integrity check failed at ClassifierSplit()"))
return;
v0=1.0/(double)n0;
v1=1.0/(double)(n-n0);
rms=0;
for(j=0; j<nclasses; j++)
{
k0=workbuf.m_classtotals0[j];
k1=workbuf.m_classpriors[j]-k0;
rms+=k0*(1-v0*k0)+k1*(1-v1*k1);
}
threshold=v;
info=1;
e=rms;
return;
}
//--- Stronger splits which require us to sort the data
//--- Quick check for degeneracy
CTSort::TagSortFastI(x,c,sortrbuf,sortibuf,n);
v=0.5*(x[0]+x[n-1]);
if(!(x[0]<v && v<x[n-1]))
return;
switch(s.m_RDFSplitStrength)
{
//--- Split at the middle
case 1:
//--- Select split position
vmin=x[0];
vmax=x[n-1];
v=x[n/2];
if(v==vmin)
v=vmin+0.001*(vmax-vmin);
if(v==vmin)
v=vmax;
//--- Calculate RMS error associated with the split
workbuf.m_classtotals0.Fill(0,0,nclasses);
n0=0;
for(i=0; i<n; i++)
if(x[i]<v)
{
k=c[i];
workbuf.m_classtotals0.Add(k,1);
n0++;
}
if(!CAp::Assert(n0>0 && n0<n,__FUNCTION__": critical integrity check failed at ClassifierSplit()"))
return;
v0=1.0/(double)n0;
v1=1.0/(double)(n-n0);
rms=0;
for(j=0; j<nclasses; j++)
{
k0=workbuf.m_classtotals0[j];
k1=workbuf.m_classpriors[j]-k0;
rms+=k0*(1-v0*k0)+k1*(1-v1*k1);
}
threshold=v;
info=1;
e=rms;
break;
//--- Strong split
case 2:
//--- Prepare initial split.
//--- Evaluate current split, prepare next one, repeat.
workbuf.m_classtotals0.Fill(0,0,nclasses);
n0=1;
while(n0<n && x[n0]==x[n0-1])
n0++;
if(!CAp::Assert(n0<n,__FUNCTION__": critical integrity check failed in ClassifierSplit()"))
return;
for(i=0; i<n0; i++)
{
k=c[i];
workbuf.m_classtotals0.Add(k,1);
}
info=-1;
threshold=x[n-1];
e=CMath::m_maxrealnumber;
while(n0<n)
{
//--- RMS error associated with current split
v0=1.0/(double)n0;
v1=1.0/(double)(n-n0);
rms=0;
for(j=0; j<nclasses; j++)
{
k0=workbuf.m_classtotals0[j];
k1=workbuf.m_classpriors[j]-k0;
rms+=k0*(1-v0*k0)+k1*(1-v1*k1);
}
if(info<0 || rms<e)
{
info=1;
e=rms;
threshold=0.5*(x[n0-1]+x[n0]);
if(threshold<=x[n0-1])
threshold=x[n0];
}
//--- Advance
n0prev=n0;
while(n0<n && (n0-n0prev)<advanceby)
{
v=x[n0];
while(n0<n && x[n0]==v)
{
k=c[n0];
workbuf.m_classtotals0.Add(k,1);
n0++;
}
}
}
if(info>0)
e=MathSqrt(e/(nclasses*n));
break;
default:
CAp::Assert(false,__FUNCTION__": ClassifierSplit(),critical error");
break;
}
}
//+------------------------------------------------------------------+
//| Regression model split |
//+------------------------------------------------------------------+
void CDForest::RegressionSplit(CDecisionForestBuilder &s,
CDFWorkBuf &workbuf,
CRowDouble &x,
CRowDouble &y,
int n,
int &info,
double &threshold,
double &e,
CRowDouble &sortrbuf,
CRowDouble &sortrbuf2)
{
//--- create variables
int i=0;
double vmin=0;
double vmax=0;
double bnd01=0;
double bnd12=0;
double bnd23=0;
int total0=0;
int total1=0;
int total2=0;
int total3=0;
int cnt0=0;
int cnt1=0;
int cnt2=0;
int cnt3=0;
int n0=0;
int advanceby=0;
double v=0;
double v0=0;
double v1=0;
double rms=0;
int n0prev=0;
int k0=0;
int k1=0;
info=0;
threshold=0;
e=0;
advanceby=1;
if(n>=20)
advanceby=MathMax(2,(int)MathRound(n*0.05));
//--- Sort data
//--- Quick check for degeneracy
CTSort::TagSortFastR(x,y,sortrbuf,sortrbuf2,n);
v=0.5*(x[0]+x[n-1]);
if(!(x[0]<v && v<x[n-1]))
{
info=-1;
threshold=x[n-1];
e=CMath::m_maxrealnumber;
return;
}
//--- Prepare initial split.
//--- Evaluate current split, prepare next one, repeat.
vmin=y[0];
vmax=y[0];
for(i=1; i<n; i++)
{
v=y[i];
if(v<vmin)
vmin=v;
if(v>vmax)
vmax=v;
}
bnd12=0.5*(vmin+vmax);
bnd01=0.5*(vmin+bnd12);
bnd23=0.5*(vmax+bnd12);
total0=0;
total1=0;
total2=0;
total3=0;
for(i=0; i<n; i++)
{
v=y[i];
if(v<bnd12)
{
if(v<bnd01)
total0++;
else
total1++;
}
else
{
if(v<bnd23)
total2++;
else
total3++;
}
}
n0=1;
while(n0<n && x[n0]==x[n0-1])
n0++;
if(!CAp::Assert(n0<n,__FUNCTION__": critical integrity check failed in ClassifierSplit()"))
return;
cnt0=0;
cnt1=0;
cnt2=0;
cnt3=0;
for(i=0; i<n0; i++)
{
v=y[i];
if(v<bnd12)
{
if(v<bnd01)
cnt0++;
else
cnt1++;
}
else
{
if(v<bnd23)
cnt2++;
else
cnt3++;
}
}
info=-1;
threshold=x[n-1];
e=CMath::m_maxrealnumber;
while(n0<n)
{
//--- RMS error associated with current split
v0=1.0/(double)n0;
v1=1.0/(double)(n-n0);
rms=0;
k0=cnt0;
k1=total0-cnt0;
rms+=k0*(1-v0*k0)+k1*(1-v1*k1);
k0=cnt1;
k1=total1-cnt1;
rms+=k0*(1-v0*k0)+k1*(1-v1*k1);
k0=cnt2;
k1=total2-cnt2;
rms+=k0*(1-v0*k0)+k1*(1-v1*k1);
k0=cnt3;
k1=total3-cnt3;
rms=rms+k0*(1-v0*k0)+k1*(1-v1*k1);
if(info<0 || rms<e)
{
info=1;
e=rms;
threshold=0.5*(x[n0-1]+x[n0]);
if(threshold<=x[n0-1])
threshold=x[n0];
}
//--- Advance
n0prev=n0;
while(n0<n && (n0-n0prev)<advanceby)
{
v0=x[n0];
while(n0<n && x[n0]==v0)
{
v=y[n0];
if(v<bnd12)
{
if(v<bnd01)
cnt0++;
else
cnt1++;
}
else
{
if(v<bnd23)
cnt2++;
else
cnt3++;
}
n0++;
}
}
}
if(info>0)
e=MathSqrt(e/(4*n));
}
//+------------------------------------------------------------------+
//| Returns split: either deterministic split at the middle |
//| of [A, B], or randomly chosen split. |
//| It is guaranteed that A < Split <= B. |
//+------------------------------------------------------------------+
double CDForest::GetSplit(CDecisionForestBuilder &s,double a,
double b,CHighQualityRandState &rs)
{
double result=0.5*(a+b);
//--- check
if(result<=a)
result=b;
//--- return result
return(result);
}
//+------------------------------------------------------------------+
//| Outputs leaf to the tree |
//| Following items of TRN and OOB sets are updated in the voting |
//| buffer: |
//| * items [Idx0, Idx1) of WorkBuf.TrnSet |
//| * items [OOBIdx0, OOBIdx1) of WorkBuf.OOBSet |
//+------------------------------------------------------------------+
void CDForest::OutputLeaf(CDecisionForestBuilder &s,
CDFWorkBuf &workbuf,
CRowDouble &treebuf,
CDFVoteBuf &votebuf,
int idx0,
int idx1,
int oobidx0,
int oobidx1,
int &treesize,
double leafval)
{
//--- create variables
int nclasses=s.m_NClasses;
int leafvali=0;
int i=0;
int j=0;
if(nclasses==1)
{
//--- Store split to the tree
treebuf.Set(treesize,-1);
treebuf.Set(treesize+1,leafval);
//--- Update training and OOB voting stats
for(i=idx0; i<idx1; i++)
{
j=workbuf.m_trnset[i];
votebuf.m_trntotals.Add(j,leafval);
votebuf.m_trncounts.Add(j,1);
}
for(i=oobidx0; i<oobidx1; i++)
{
j=workbuf.m_oobset[i];
votebuf.m_oobtotals.Add(j,leafval);
votebuf.m_oobcounts.Add(j,1);
}
}
else
{
//--- Store split to the tree
treebuf.Set(treesize,-1);
treebuf.Set(treesize+1,leafval);
//--- Update training and OOB voting stats
leafvali=(int)MathRound(leafval);
for(i=idx0; i<idx1; i++)
{
j=workbuf.m_trnset[i];
votebuf.m_trntotals.Add(j*nclasses+leafvali,1);
votebuf.m_trncounts.Add(j,1);
}
for(i=oobidx0; i<=oobidx1-1; i++)
{
j=workbuf.m_oobset[i];
votebuf.m_oobtotals.Add(j*nclasses+leafvali,1);
votebuf.m_oobcounts.Add(j,1);
}
}
treesize+=m_LeafNodeWdth;
}
//+------------------------------------------------------------------+
//| This function performs generic and algorithm - specific |
//| preprocessing of the dataset |
//+------------------------------------------------------------------+
void CDForest::AnalyzeAndPreprocessDataset(CDecisionForestBuilder &s)
{
//--- check
if(!CAp::Assert(s.m_DSType==0,__FUNCTION__": no sparsity"))
return;
//--- create variables
int i=0;
int j=0;
bool isbinary;
double v=0;
double v0=0;
double v1=0;
CHighQualityRandState rs;
int npoints=s.m_NPoints;
int nvars=s.m_NVars;
int nclasses=s.m_NClasses;
//--- seed local RNG
if(s.m_RDFGlobalSeed>0)
CHighQualityRand::HQRndSeed(s.m_RDFGlobalSeed,3532,rs);
else
CHighQualityRand::HQRndSeed(CMath::RandomInteger(30000),3532,rs);
//--- Generic processing
if(!CAp::Assert(npoints>=1,__FUNCTION__": integrity check failed"))
return;
s.m_DSMin.Resize(nvars);
s.m_DSMax.Resize(nvars);
CApServ::BVectorSetLengthAtLeast(s.m_DSBinary,nvars);
for(i=0; i<nvars; i++)
{
v0=s.m_DSData[i*npoints];
v1=s.m_DSData[i*npoints];
for(j=1; j<npoints; j++)
{
v=s.m_DSData[i*npoints+j];
if(v<v0)
v0=v;
if(v>v1)
v1=v;
}
s.m_DSMin.Set(i,v0);
s.m_DSMax.Set(i,v1);
if(!CAp::Assert(v0<=v1,__FUNCTION__": strange integrity check failure"))
return;
isbinary=true;
for(j=0; j<npoints; j++)
{
v=s.m_DSData[i*npoints+j];
isbinary=isbinary && (v==v0 || v==v1);
}
s.m_DSBinary[i]=isbinary;
}
if(nclasses==1)
{
s.m_DSRAvg=0;
for(i=0; i<npoints; i++)
s.m_DSRAvg+=s.m_DSRVal[i];
s.m_DSRAvg=s.m_DSRAvg/npoints;
}
else
{
s.m_DSCTotals.Resize(nclasses);
s.m_DSCTotals.Fill(0);
for(i=0; i<npoints; i++)
s.m_DSCTotals.Add(s.m_DSIVal[i],1);
}
}
//+------------------------------------------------------------------+
//| This function merges together trees generated during training and|
//| outputs it to the decision forest. |
//| INPUT PARAMETERS: |
//| S - decision forest builder object |
//| NTrees - NTrees >= 1, number of trees to train |
//| OUTPUT PARAMETERS: |
//| DF - decision forest |
//| Rep - report |
//+------------------------------------------------------------------+
void CDForest::MergeTrees(CDecisionForestBuilder &s,
CDecisionForest &df)
{
//--- create variables
int i=0;
int cursize=0;
int offs=0;
CRowInt treesizes;
CRowInt treeoffsets;
//--- initialize
df.m_ForestFormat=m_DFUncompressedV0;
df.m_NVars=s.m_NVars;
df.m_NClasses=s.m_NClasses;
df.m_BufSize=0;
//--- Determine trees count
df.m_NTrees=ArraySize(s.m_TreeBuf);
if(!CAp::Assert(df.m_NTrees>0,__FUNCTION__": integrity check failed,zero trees count"))
return;
//--- Determine individual tree sizes and total buffer size
treesizes.Resize(df.m_NTrees);
treesizes.Fill(-1);
for(int idx=0; idx<df.m_NTrees; idx++)
{
if(!CAp::Assert(s.m_TreeBuf[idx].m_treeidx>=0 && s.m_TreeBuf[idx].m_treeidx<df.m_NTrees,__FUNCTION__": integrity check failed (wrong TreeIdx)"))
return;
if(!CAp::Assert(treesizes[s.m_TreeBuf[idx].m_treeidx]<0,__FUNCTION__": integrity check failed (duplicate TreeIdx)"))
return;
int size=(int)MathRound(s.m_TreeBuf[idx].m_treebuf[0]);
if(!CAp::Assert(size>0,__FUNCTION__": integrity check failed (wrong TreeSize)"))
return;
df.m_BufSize+=size;
treesizes.Set(s.m_TreeBuf[idx].m_treeidx,size);
}
//--- Determine offsets for individual trees in output buffer
treeoffsets.Resize(df.m_NTrees);
treeoffsets.Set(0,0);
for(i=1; i<df.m_NTrees; i++)
treeoffsets.Set(i,treeoffsets[i-1]+treesizes[i-1]);
//--- Output trees
//--- NOTE: since ALGLIB 3.16.0 trees are sorted by tree index prior to
//--- output (necessary for variable importance estimation), that's
//--- why we need array of tree offsets
df.m_Trees.Resize(df.m_BufSize);
for(int idx=0; idx<df.m_NTrees; idx++)
{
cursize=(int)MathRound(s.m_TreeBuf[idx].m_treebuf[0]);
offs=treeoffsets[s.m_TreeBuf[idx].m_treeidx];
for(i=0; i<cursize; i++)
df.m_Trees.Set(offs+i,s.m_TreeBuf[idx].m_treebuf[i]);
}
}
//+------------------------------------------------------------------+
//| This function post - processes voting array and calculates TRN |
//| and OOB errors. |
//| INPUT PARAMETERS: |
//| S - decision forest builder object |
//| NTrees - number of trees in the forest |
//| Buf - possibly preallocated vote buffer, its contents is |
//| overwritten by this function |
//| OUTPUT PARAMETERS: |
//| Rep - report fields corresponding to errors are updated |
//+------------------------------------------------------------------+
void CDForest::ProcessVotingResults(CDecisionForestBuilder &s,
int ntrees,
CDFVoteBuf &buf,
CDFReport &rep)
{
//--- create variables
int i=0;
int j=0;
int k=0;
int k1=0;
double v=0;
int avgrelcnt=0;
int oobavgrelcnt=0;
int npoints=s.m_NPoints;
int nvars=s.m_NVars;
int nclasses=s.m_NClasses;
//--- check
if(!CAp::Assert(npoints>0,__FUNCTION__": integrity check failed"))
return;
if(!CAp::Assert(nvars>0,__FUNCTION__": integrity check failed"))
return;
if(!CAp::Assert(nclasses>0,__FUNCTION__": integrity check failed"))
return;
//--- Prepare vote buffer
buf.m_trntotals=vector<double>::Zeros(npoints*nclasses);
buf.m_oobtotals=vector<double>::Zeros(npoints*nclasses);
buf.m_trncounts.Resize(npoints);
buf.m_oobcounts.Resize(npoints);
buf.m_trncounts.Fill(0);
buf.m_oobcounts.Fill(0);
//--- Merge voting arrays
for(int idx=0; idx<ntrees; idx++)
{
buf.m_trntotals+=s.m_VoteBuf[idx].m_trntotals+s.m_VoteBuf[idx].m_oobtotals+0;
buf.m_oobtotals+=s.m_VoteBuf[idx].m_oobtotals;
for(i=0; i<=npoints-1; i++)
{
buf.m_trncounts.Add(i,s.m_VoteBuf[idx].m_trncounts[i]+s.m_VoteBuf[idx].m_oobcounts[i]);
buf.m_oobcounts.Add(i,s.m_VoteBuf[idx].m_oobcounts[i]);
}
}
for(i=0; i<npoints; i++)
{
v=1.0/CApServ::Coalesce(buf.m_trncounts[i],1);
for(j=0; j<nclasses; j++)
buf.m_trntotals.Mul(i*nclasses+j,v);
v=1.0/CApServ::Coalesce(buf.m_oobcounts[i],1);
for(j=0; j<=nclasses-1; j++)
buf.m_oobtotals.Mul(i*nclasses+j,v);
}
//--- Use aggregated voting data to output error metrics
avgrelcnt=0;
oobavgrelcnt=0;
rep.m_RMSError=0;
rep.m_AvgError=0;
rep.m_AvgRelError=0;
rep.m_RelCLSError=0;
rep.m_AvgCE=0;
rep.m_oobrmserror=0;
rep.m_oobavgerror=0;
rep.m_oobavgrelerror=0;
rep.m_oobrelclserror=0;
rep.m_oobavgce=0;
for(i=0; i<npoints; i++)
{
if(nclasses>1)
{
//--- classification-specific code
k=s.m_DSIVal[i];
for(j=0; j<nclasses; j++)
{
v=buf.m_trntotals[i*nclasses+j];
if(j==k)
{
rep.m_AvgCE-=MathLog(CApServ::Coalesce(v,CMath::m_minrealnumber));
rep.m_RMSError+=CMath::Sqr(v-1);
rep.m_AvgError+=MathAbs(v-1);
rep.m_AvgRelError+=MathAbs(v-1);
avgrelcnt++;
}
else
{
rep.m_RMSError+=CMath::Sqr(v);
rep.m_AvgError+=MathAbs(v);
}
v=buf.m_oobtotals[i*nclasses+j];
if(j==k)
{
rep.m_oobavgce-=MathLog(CApServ::Coalesce(v,CMath::m_minrealnumber));
rep.m_oobrmserror+=CMath::Sqr(v-1);
rep.m_oobavgerror+=MathAbs(v-1);
rep.m_oobavgrelerror+=MathAbs(v-1);
oobavgrelcnt++;
}
else
{
rep.m_oobrmserror+=CMath::Sqr(v);
rep.m_oobavgerror+=MathAbs(v);
}
}
//--- Classification errors are handled separately
k1=0;
for(j=1; j<nclasses; j++)
{
if(buf.m_trntotals[i*nclasses+j]>buf.m_trntotals[i*nclasses+k1])
k1=j;
}
if(k1!=k)
rep.m_RelCLSError++;
k1=0;
for(j=1; j<nclasses; j++)
{
if(buf.m_oobtotals[i*nclasses+j]>buf.m_oobtotals[i*nclasses+k1])
k1=j;
}
if(k1!=k)
rep.m_oobrelclserror++;
}
else
{
//--- regression-specific code
v=buf.m_trntotals[i]-s.m_DSRVal[i];
rep.m_RMSError+=CMath::Sqr(v);
rep.m_AvgError+=MathAbs(v);
if(s.m_DSRVal[i]!=0.0)
{
rep.m_AvgRelError+=MathAbs(v/s.m_DSRVal[i]);
avgrelcnt++;
}
v=buf.m_oobtotals[i]-s.m_DSRVal[i];
rep.m_oobrmserror+=CMath::Sqr(v);
rep.m_oobavgerror+=MathAbs(v);
if(s.m_DSRVal[i]!=0.0)
{
rep.m_oobavgrelerror+=MathAbs(v/s.m_DSRVal[i]);
oobavgrelcnt++;
}
}
}
rep.m_RelCLSError/=npoints;
rep.m_RMSError=MathSqrt(rep.m_RMSError/(npoints*nclasses));
rep.m_AvgError/=(npoints*nclasses);
rep.m_AvgRelError/=CApServ::Coalesce(avgrelcnt,1);
rep.m_oobrelclserror/=npoints;
rep.m_oobrmserror=MathSqrt(rep.m_oobrmserror/(npoints*nclasses));
rep.m_oobavgerror/=(npoints*nclasses);
rep.m_oobavgrelerror/=CApServ::Coalesce(oobavgrelcnt,1);
}
//+------------------------------------------------------------------+
//| This function performs binary compression of decision forest, |
//| using either 8-bit mantissa (a bit more compact representation) |
//| or 16-bit mantissa for splits and regression outputs. |
//| Forest is compressed in - place. |
//| Return value is a compression factor. |
//+------------------------------------------------------------------+
double CDForest::BinaryCompression(CDecisionForest &df,
bool usemantissa8)
{
//--- create variables
double result=0;
int size8=0;
int size8i=0;
int offssrc=0;
int offsdst=0;
int maxrawtreesize=0;
CRowInt dummyi;
CRowInt compressedsizes;
//--- Quick exit if already compressed
if(df.m_ForestFormat==m_DFCompressedV0)
return(1.0);
//--- Check that source format is supported
if(!CAp::Assert(df.m_ForestFormat==m_DFUncompressedV0,__FUNCTION__": unexpected forest format"))
return(result);
//--- Compute sizes of uncompressed and compressed trees.
for(int i=0; i<df.m_NTrees; i++)
{
size8i=ComputeCompressedSizeRec(df,usemantissa8,offssrc,offssrc+1,dummyi,false);
size8+=ComputeCompressedUintSize(size8i)+size8i;
maxrawtreesize=MathMax(maxrawtreesize,(int)MathRound(df.m_Trees[offssrc]));
offssrc+=(int)MathRound(df.m_Trees[offssrc]);
}
result=(double)(8.0*df.m_Trees.Size())/(double)(size8+1.0);
//--- Allocate memory and perform compression
df.m_Trees8.Resize(size8);
compressedsizes.Resize(maxrawtreesize);
offssrc=0;
offsdst=0;
for(int i=0; i<df.m_NTrees; i++)
{
//--- Call compressed size evaluator one more time, now saving subtree sizes into temporary array
size8i=ComputeCompressedSizeRec(df,usemantissa8,offssrc,offssrc+1,compressedsizes,true);
//--- Output tree header (length in bytes)
StreamUint(df.m_Trees8,offsdst,size8i);
//--- Compress recursively
CompressRec(df,usemantissa8,offssrc,offssrc+1,compressedsizes,df.m_Trees8,offsdst);
//--- Next tree
offssrc+=(int)MathRound(df.m_Trees[offssrc]);
}
if(!CAp::Assert(offsdst==size8,__FUNCTION__": integrity check failed (stream length)"))
return(result);
//--- Finalize forest conversion, clear previously allocated memory
df.m_ForestFormat=m_DFCompressedV0;
df.m_UseMantissa8=usemantissa8;
df.m_Trees.Resize(0);
//--- return result
return(result);
}
//+------------------------------------------------------------------+
//| This function returns exact number of bytes required to store |
//| compressed version of the tree starting at location TreeBase. |
//| PARAMETERS: |
//| DF - decision forest |
//| UseMantissa8- whether 8-bit or 16-bit mantissas are used to |
//| store floating point numbers |
//| TreeRoot - root of the specific tree being stored(offset |
//| in DF.Trees) |
//| TreePos - position within tree(first location in the tree |
//| is TreeRoot + 1) |
//| CompressedSizes - not referenced if SaveCompressedSizes is |
//| False; otherwise, values computed by this |
//| function for specific values of TreePos are |
//| stored to CompressedSizes[TreePos - TreeRoot] |
//| (other elements of the array are not referenced)|
//| This array must be preallocated by caller. |
//+------------------------------------------------------------------+
int CDForest::ComputeCompressedSizeRec(CDecisionForest &df,
bool usemantissa8,
int treeroot,
int treepos,
CRowInt &compressedsizes,
bool savecompressedsizes)
{
//--- create variables
int result=0;
int jmponbranch=0;
int child0size=0;
int child1size=0;
int fpwidth=0;
//--- check
if(usemantissa8)
fpwidth=2;
else
fpwidth=3;
//--- Leaf or split?
if(df.m_Trees[treepos]==-1.0)
{
//--- Leaf
result=ComputeCompressedUintSize(2*df.m_NVars);
if(df.m_NClasses==1)
result+=fpwidth;
else
result+=ComputeCompressedUintSize((int)MathRound(df.m_Trees[treepos+1]));
}
else
{
//--- Split
jmponbranch=(int)MathRound(df.m_Trees[treepos+2]);
child0size=ComputeCompressedSizeRec(df,usemantissa8,treeroot,treepos+m_InnerNodeWidth,compressedsizes,savecompressedsizes);
child1size=ComputeCompressedSizeRec(df,usemantissa8,treeroot,treeroot+jmponbranch,compressedsizes,savecompressedsizes);
if(child0size<=child1size)
{
//--- Child #0 comes first because it is shorter
result=ComputeCompressedUintSize((int)MathRound(df.m_Trees[treepos]));
result+=fpwidth;
result+=ComputeCompressedUintSize(child0size);
}
else
{
//--- Child #1 comes first because it is shorter
result=ComputeCompressedUintSize((int)MathRound(df.m_Trees[treepos])+df.m_NVars);
result+=fpwidth;
result+=ComputeCompressedUintSize(child1size);
}
result+=child0size+child1size;
}
//--- Do we have to save compressed sizes?
if(savecompressedsizes)
{
if(!CAp::Assert(treepos-treeroot<compressedsizes.Size(),__FUNCTION__": integrity check failed"))
return(result);
compressedsizes.Set(treepos-treeroot,result);
}
//--- return result
return(result);
}
//+------------------------------------------------------------------+
//| This function returns exact number of bytes required to store |
//| compressed version of the tree starting at location TreeBase. |
//| PARAMETERS: |
//| DF - decision forest |
//| UseMantissa8 - whether 8-bit or 16-bit mantissas are used |
//| to store floating point numbers |
//| TreeRoot - root of the specific tree being stored(offset |
//| in DF.Trees) |
//| TreePos - position within tree (first location in the tree|
//| is TreeRoot + 1) |
//| CompressedSizes - not referenced if SaveCompressedSizes is |
//| False; otherwise, values computed by this |
//| function for specific values of TreePos are |
//| stored to CompressedSizes[TreePos - TreeRoot] |
//| (other elements of the array are not referenced)|
//| This array must be preallocated by caller. |
//+------------------------------------------------------------------+
void CDForest::CompressRec(CDecisionForest &df,
bool usemantissa8,
int treeroot,
int treepos,
CRowInt &compressedsizes,
CRowInt &buf,
int &dstoffs)
{
//--- create variables
int jmponbranch=0;
int child0size=0;
int child1size=0;
int varidx=0;
double leafval=0;
double splitval=0;
int dstoffsold=dstoffs;
//--- Leaf or split?
varidx=(int)MathRound(df.m_Trees[treepos]);
if(varidx==-1)
{
//--- Leaf node:
//--- * stream special value which denotes leaf (2*NVars)
//--- * then, stream scalar value (floating point) or class number (unsigned integer)
leafval=df.m_Trees[treepos+1];
StreamUint(buf,dstoffs,2*df.m_NVars);
if(df.m_NClasses==1)
StreamFloat(buf,usemantissa8,dstoffs,leafval);
else
StreamUint(buf,dstoffs,(int)MathRound(leafval));
}
else
{
//--- Split node:
//--- * fetch compressed sizes of child nodes, decide which child goes first
jmponbranch=(int)MathRound(df.m_Trees[treepos+2]);
splitval=df.m_Trees[treepos+1];
child0size=compressedsizes[treepos+m_InnerNodeWidth-treeroot];
child1size=compressedsizes[treeroot+jmponbranch-treeroot];
if(child0size<=child1size)
{
//--- Child #0 comes first because it is shorter:
//--- * stream variable index used for splitting;
//--- value in [0,NVars) range indicates that split is
//--- "if VAR<VAL then BRANCH0 else BRANCH1"
//--- * stream value used for splitting
//--- * stream children #0 and #1
StreamUint(buf,dstoffs,varidx);
StreamFloat(buf,usemantissa8,dstoffs,splitval);
StreamUint(buf,dstoffs,child0size);
CompressRec(df,usemantissa8,treeroot,treepos+m_InnerNodeWidth,compressedsizes,buf,dstoffs);
CompressRec(df,usemantissa8,treeroot,treeroot+jmponbranch,compressedsizes,buf,dstoffs);
}
else
{
//--- Child #1 comes first because it is shorter:
//--- * stream variable index used for splitting + NVars;
//--- value in [NVars,2*NVars) range indicates that split is
//--- "if VAR>=VAL then BRANCH0 else BRANCH1"
//--- * stream value used for splitting
//--- * stream children #0 and #1
StreamUint(buf,dstoffs,varidx+df.m_NVars);
StreamFloat(buf,usemantissa8,dstoffs,splitval);
StreamUint(buf,dstoffs,child1size);
CompressRec(df,usemantissa8,treeroot,treeroot+jmponbranch,compressedsizes,buf,dstoffs);
CompressRec(df,usemantissa8,treeroot,treepos+m_InnerNodeWidth,compressedsizes,buf,dstoffs);
}
}
//--- Integrity check at the end
if(!CAp::Assert(dstoffs-dstoffsold==compressedsizes[treepos-treeroot],__FUNCTION__": integrity check failed (compressed size at leaf)"))
return;
}
//+------------------------------------------------------------------+
//| This function returns exact number of bytes required to store |
//| compressed unsigned integer number(negative arguments result in |
//| assertion being generated). |
//+------------------------------------------------------------------+
int CDForest::ComputeCompressedUintSize(int v)
{
//--- check
if(!CAp::Assert(v>=0))
return(0);
//--- create variables
int result=1;
//---
while(v>=128)
{
v=v/128;
result=result+1;
}
//--- return result
return(result);
}
//+------------------------------------------------------------------+
//| This function stores compressed unsigned integer number (negative|
//| arguments result in assertion being generated) to byte array at |
//| location Offs and increments Offs by number of bytes being stored|
//+------------------------------------------------------------------+
void CDForest::StreamUint(CRowInt &buf,
int &offs,
int v)
{
int v0=0;
//--- check
if(!CAp::Assert(v>=0))
return;
//---
while(true)
{
//--- Save 7 least significant bits of V, use 8th bit as a flag which
//--- tells us whether subsequent 7-bit packages will be sent.
v0=v%128;
if(v>=128)
v0=v0+128;
buf.Set(offs,(uint)v0);
offs++;
v/=128;
if(v==0)
break;
}
}
//+------------------------------------------------------------------+
//| This function reads compressed unsigned integer number from byte |
//| array starting at location Offs and increments Offs by number of |
//| bytes being read. |
//+------------------------------------------------------------------+
int CDForest::UnstreamUint(CRowInt &buf,int &offs)
{
//--- create variables
int result=0;
int v0=0;
int p=1;
//---
while(true)
{
//--- Rad 7 bits of V, use 8th bit as a flag which tells us whether
//--- subsequent 7-bit packages will be received.
v0=buf[offs];
offs=offs+1;
result+=v0%128*p;
if(v0<128)
break;
p*=128;
}
//--- return result
return(result);
}
//+------------------------------------------------------------------+
//| This function stores compressed floating point number to byte |
//| array at location Offs and increments Offs by number of bytes |
//| being stored. Either 8-bit mantissa or 16-bit mantissa is used. |
//| The exponent is always 7 bits of exponent + sign. Values which |
//| do not fit into exponent range are truncated to fit. |
//+------------------------------------------------------------------+
void CDForest::StreamFloat(CRowInt &buf,
bool usemantissa8,
int &offs,
double v)
{
//--- create variables
int signbit=0;
int e=0;
int m=0;
double twopow30=0;
double twopowm30=0;
double twopow10=0;
double twopowm10=0;
//--- check
if(!CAp::Assert(MathIsValidNumber(v),__FUNCTION__": V is not finite number"))
return;
//--- Special case: zero
if(v==0.0)
{
if(usemantissa8)
{
buf.Set(offs,0);
buf.Set(offs+1,0);
offs+=2;
}
else
{
buf.Set(offs,0);
buf.Set(offs+1,0);
buf.Set(offs+2,0);
offs+=3;
}
return;
}
//--- Handle sign
signbit=0;
if(v<0.0)
{
v=-v;
signbit=128;
}
//--- Compute exponent
twopow30=1073741824;
twopow10=1024;
twopowm30=1.0/twopow30;
twopowm10=1.0/twopow10;
e=0;
while(v>=twopow30)
{
v*=twopowm30;
e+=30;
}
while(v>=twopow10)
{
v*=twopowm10;
e+=10;
}
while(v>=1.0)
{
v*=0.5;
e++;
}
while(v<twopowm30)
{
v*=twopow30;
e-=30;
}
while(v<twopowm10)
{
v*=twopow10;
e-=10;
}
while(v<0.5)
{
v*=2;
e-=1;
}
if(!CAp::Assert(v>=0.5 && v<1.0,__FUNCTION__": integrity check failed"))
return;
//--- Handle exponent underflow/overflow
if(e<-63)
{
signbit=0;
e=0;
v=0;
}
if(e>63)
{
e=63;
v=1.0;
}
//--- Save to stream
if(usemantissa8)
{
m=(int)MathRound(v*256);
if(m==256)
{
m=m/2;
e=MathMin(e+1,63);
}
buf.Set(offs,e+64+signbit);
buf.Set(offs+1,m);
offs+=2;
}
else
{
m=(int)MathRound(v*65536);
if(m==65536)
{
m=m/2;
e=MathMin(e+1,63);
}
buf.Set(offs,e+64+signbit);
buf.Set(offs+1,m%256);
buf.Set(offs+2,m/256);
offs+=3;
}
}
//+------------------------------------------------------------------+
//| This function reads compressed floating point number from the |
//| byte array starting from location Offs and increments Offs by |
//| number of bytes being read. |
//| Either 8-bit mantissa or 16-bit mantissa is used. The exponent |
//| is always 7 bits of exponent + sign. Values which do not fit |
//| into exponent range are truncated to fit. |
//+------------------------------------------------------------------+
double CDForest::UnstreamFloat(CRowInt &buf,
bool usemantissa8,
int &offs)
{
//--- create variables
double result=0;
int e=0;
double v=0;
double inv256=1.0/256.0;
//--- Read from stream
if(usemantissa8)
{
e=buf[offs];
v=buf[offs+1]*inv256;
offs+=2;
}
else
{
e=buf[offs];
v=(buf[offs+1]*inv256+buf[offs+2])*inv256;
offs+=3 ;
}
//--- Decode
if(e>128)
{
v=-v;
e=e-128;
}
e=e-64;
result=MathPow(2.0,e)*v;
//--- return result
return(result);
}
//+------------------------------------------------------------------+
//| Classification error |
//+------------------------------------------------------------------+
int CDForest::DFClsError(CDecisionForest &df,CMatrixDouble &xy,
const int npoints)
{
//--- create variables
int result=0;
int k=0;
int tmpi=0;
int i_=0;
//--- creating arrays
CRowDouble x;
CRowDouble y;
//--- check
if(df.m_NClasses<=1)
return(0);
//--- initialization
result=0;
for(int i=0; i<npoints; i++)
{
//--- copy
x=xy[i]+0;
//--- function call
DFProcess(df,x,y);
//--- change values
k=(int)MathRound(xy.Get(i,df.m_NVars));
tmpi=0;
for(int j=1; j<df.m_NClasses; j++)
{
//--- check
if(y[j]>y[tmpi])
tmpi=j;
}
//--- check
if(tmpi!=k)
result++;
}
//--- return result
return(result);
}
//+------------------------------------------------------------------+
//| Internal subroutine for processing one decision tree stored in |
//| uncompressed format starting at SubtreeRoot (this index points |
//| to the header of the tree, not its first node). First node being |
//| processed is located at NodeOffs. |
//+------------------------------------------------------------------+
void CDForest::DFProcessInternalUncompressed(CDecisionForest &df,
int subtreeroot,
int nodeoffs,
CRowDouble &x,
CRowDouble &y)
{
int idx=0;
//--- check
if(!CAp::Assert(df.m_ForestFormat==m_DFUncompressedV0,__FUNCTION__+": unexpected forest format"))
return;
//--- Navigate through the tree
while(true)
{
if(df.m_Trees[nodeoffs]==(-1.0))
{
if(df.m_NClasses==1)
y.Add(0,df.m_Trees[nodeoffs+1]);
else
{
idx=(int)MathRound(df.m_Trees[nodeoffs+1]);
y.Add(idx,1);
}
break;
}
if(x[(int)MathRound(df.m_Trees[nodeoffs])]<df.m_Trees[nodeoffs+1])
nodeoffs+=m_InnerNodeWidth;
else
nodeoffs=subtreeroot+(int)MathRound(df.m_Trees[nodeoffs+2]);
}
}
//+------------------------------------------------------------------+
//| Internal subroutine for processing one decision tree stored in |
//| compressed format starting at Offs (this index points to the |
//| first node of the tree, right past the header field). |
//+------------------------------------------------------------------+
void CDForest::DFProcessInternalCompressed(CDecisionForest &df,
int offs,
CRowDouble &x,
CRowDouble &y)
{
//--- create variables
int leafindicator=0;
int varidx=0;
double splitval=0;
int jmplen=0;
double leafval=0;
int leafcls=0;
//--- check
if(!CAp::Assert(df.m_ForestFormat==m_DFCompressedV0,__FUNCTION__+": unexpected forest format"))
return;
//--- Navigate through the tree
leafindicator=2*df.m_NVars;
while(true)
{
//--- Read variable idx
varidx=UnstreamUint(df.m_Trees8,offs);
//--- Is it leaf?
if(varidx==leafindicator)
{
if(df.m_NClasses==1)
{
//--- Regression forest
leafval=UnstreamFloat(df.m_Trees8,df.m_UseMantissa8,offs);
y.Add(0,leafval);
}
else
{
//--- Classification forest
leafcls=UnstreamUint(df.m_Trees8,offs);
y.Add(leafcls,1);
}
break;
}
//--- Process node
splitval=UnstreamFloat(df.m_Trees8,df.m_UseMantissa8,offs);
jmplen=UnstreamUint(df.m_Trees8,offs);
if(varidx<df.m_NVars)
{
//--- The split rule is "if VAR<VAL then BRANCH0 else BRANCH1"
if(x[varidx]>=splitval)
offs+=jmplen;
}
else
{
//--- The split rule is "if VAR>=VAL then BRANCH0 else BRANCH1"
varidx-=df.m_NVars;
if(x[varidx]<splitval)
offs+=jmplen;
}
}
}
//+------------------------------------------------------------------+
//| Middle and clusterization |
//+------------------------------------------------------------------+
class CKMeans
{
public:
static void KMeansGenerate(CMatrixDouble &xy,const int npoints,const int nvars,const int k,const int restarts,int &info,CMatrixDouble &c,int &xyc[]);
private:
static bool SelectCenterPP(CMatrixDouble &xy,const int npoints,const int nvars,CMatrixDouble &centers,bool &cbusycenters[],const int ccnt,double &d2[],double &p[],double &tmp[]);
};
//+------------------------------------------------------------------+
//| k-means++ clusterization |
//| INPUT PARAMETERS: |
//| XY - dataset, array [0..NPoints-1,0..NVars-1]. |
//| NPoints - dataset size, NPoints>=K |
//| NVars - number of variables, NVars>=1 |
//| K - desired number of clusters, K>=1 |
//| Restarts - number of restarts, Restarts>=1 |
//| OUTPUT PARAMETERS: |
//| Info - return code: |
//| * -3, if task is degenerate (number of |
//| distinct points is less than K) |
//| * -1, if incorrect |
//| NPoints/NFeatures/K/Restarts was passed|
//| * 1, if subroutine finished successfully |
//| C - array[0..NVars-1,0..K-1].matrix whose columns|
//| store cluster's centers |
//| XYC - array[NPoints], which contains cluster |
//| indexes |
//+------------------------------------------------------------------+
void CKMeans::KMeansGenerate(CMatrixDouble &xy,const int npoints,
const int nvars,const int k,
const int restarts,int &info,
CMatrixDouble &c,int &xyc[])
{
//--- create variables
int i=0;
int j=0;
double e=0;
double ebest=0;
double v=0;
int cclosest=0;
bool waschanges;
bool zerosizeclusters;
int pass=0;
int i_=0;
double dclosest=0;
//--- creating arrays
int xycbest[];
double x[];
double tmp[];
double d2[];
double p[];
int csizes[];
bool cbusy[];
double work[];
//--- create matrix
CMatrixDouble ct;
CMatrixDouble ctbest;
//--- initialization
info=0;
//--- Test parameters
if(npoints<k || nvars<1 || k<1 || restarts<1)
{
info=-1;
return;
}
//--- TODO: special case K=1
//--- TODO: special case K=NPoints
info=1;
//--- Multiple passes of k-means++ algorithm
ct.Resize(k,nvars);
ctbest.Resize(k,nvars);
ArrayResizeAL(xyc,npoints);
ArrayResizeAL(xycbest,npoints);
ArrayResize(d2,npoints);
ArrayResize(p,npoints);
ArrayResize(tmp,nvars);
ArrayResizeAL(csizes,k);
ArrayResizeAL(cbusy,k);
//--- change value
ebest=CMath::m_maxrealnumber;
//--- calculation
for(pass=1; pass<=restarts; pass++)
{
//--- Select initial centers using k-means++ algorithm
//--- 1. Choose first center at random
//--- 2. Choose next centers using their distance from centers already chosen
//--- Note that for performance reasons centers are stored in ROWS of CT,not
//--- in columns. We'll transpose CT in the end and store it in the C.
i=CMath::RandomInteger(npoints);
for(i_=0; i_<=nvars-1; i_++)
ct.Set(0,i_,xy[i][i_]);
cbusy[0]=true;
for(i=1; i<=k-1; i++)
cbusy[i]=false;
//--- check
if(!SelectCenterPP(xy,npoints,nvars,ct,cbusy,k,d2,p,tmp))
{
info=-3;
return;
}
//--- Update centers:
//--- 2. update center positions
for(i=0; i<npoints; i++)
xyc[i]=-1;
//--- cycle
while(true)
{
//--- fill XYC with center numbers
waschanges=false;
for(i=0; i<npoints; i++)
{
//--- change values
cclosest=-1;
dclosest=CMath::m_maxrealnumber;
for(j=0; j<=k-1; j++)
{
//--- calculation
for(i_=0; i_<=nvars-1; i_++)
tmp[i_]=xy[i][i_];
for(i_=0; i_<=nvars-1; i_++)
tmp[i_]=tmp[i_]-ct[j][i_];
v=0.0;
for(i_=0; i_<=nvars-1; i_++)
v+=tmp[i_]*tmp[i_];
//--- check
if(v<dclosest)
{
cclosest=j;
dclosest=v;
}
}
//--- check
if(xyc[i]!=cclosest)
waschanges=true;
//--- change value
xyc[i]=cclosest;
}
//--- Update centers
for(j=0; j<=k-1; j++)
csizes[j]=0;
for(i=0; i<=k-1; i++)
{
for(j=0; j<=nvars-1; j++)
ct.Set(i,j,0);
}
//--- change values
for(i=0; i<npoints; i++)
{
csizes[xyc[i]]=csizes[xyc[i]]+1;
for(i_=0; i_<=nvars-1; i_++)
ct.Set(xyc[i],i_,ct[xyc[i]][i_]+xy[i][i_]);
}
zerosizeclusters=false;
for(i=0; i<=k-1; i++)
{
cbusy[i]=csizes[i]!=0;
zerosizeclusters=zerosizeclusters || csizes[i]==0;
}
//--- check
if(zerosizeclusters)
{
//--- Some clusters have zero size - rare,but possible.
//--- We'll choose new centers for such clusters using k-means++ rule
//--- and restart algorithm
if(!SelectCenterPP(xy,npoints,nvars,ct,cbusy,k,d2,p,tmp))
{
info=-3;
return;
}
continue;
}
//--- copy
for(j=0; j<=k-1; j++)
{
v=1.0/(double)csizes[j];
for(i_=0; i_<=nvars-1; i_++)
ct.Set(j,i_,v*ct[j][i_]);
}
//--- if nothing has changed during iteration
if(!waschanges)
break;
}
//--- 3. Calculate E,compare with best centers found so far
e=0;
for(i=0; i<npoints; i++)
{
for(i_=0; i_<=nvars-1; i_++)
tmp[i_]=xy[i][i_];
for(i_=0; i_<=nvars-1; i_++)
tmp[i_]=tmp[i_]-ct[xyc[i]][i_];
//--- calculation
v=0.0;
for(i_=0; i_<=nvars-1; i_++)
v+=tmp[i_]*tmp[i_];
e=e+v;
}
//--- check
if(e<ebest)
{
//--- store partition.
ebest=e;
//--- function call
CBlas::CopyMatrix(ct,0,k-1,0,nvars-1,ctbest,0,k-1,0,nvars-1);
//--- copy
for(i=0; i<npoints; i++)
xycbest[i]=xyc[i];
}
}
//--- Copy and transpose
c.Resize(nvars,k);
//--- function call
CBlas::CopyAndTranspose(ctbest,0,k-1,0,nvars-1,c,0,nvars-1,0,k-1);
//--- copy
for(i=0; i<npoints; i++)
xyc[i]=xycbest[i];
}
//+------------------------------------------------------------------+
//| Select center for a new cluster using k-means++ rule |
//+------------------------------------------------------------------+
bool CKMeans::SelectCenterPP(CMatrixDouble &xy,const int npoints,
const int nvars,CMatrixDouble &centers,
bool &cbusycenters[],const int ccnt,
double &d2[],double &p[],double &tmp[])
{
//--- create variables
bool result;
int i=0;
int j=0;
int cc=0;
double v=0;
double s=0;
int i_=0;
//--- create array
double busycenters[];
//--- copy
ArrayCopy(busycenters,cbusycenters);
//--- initialization
result=true;
//--- calculation
for(cc=0; cc<=ccnt-1; cc++)
{
//--- check
if(!busycenters[cc])
{
//--- fill D2
for(i=0; i<npoints; i++)
{
d2[i]=CMath::m_maxrealnumber;
for(j=0; j<=ccnt-1; j++)
{
//--- check
if(busycenters[j])
{
for(i_=0; i_<=nvars-1; i_++)
tmp[i_]=xy[i][i_];
for(i_=0; i_<=nvars-1; i_++)
tmp[i_]=tmp[i_]-centers[j][i_];
//--- calculation
v=0.0;
for(i_=0; i_<=nvars-1; i_++)
v+=tmp[i_]*tmp[i_];
//--- check
if(v<d2[i])
d2[i]=v;
}
}
}
//--- calculate P (non-cumulative)
s=0;
for(i=0; i<npoints; i++)
s=s+d2[i];
//--- check
if(s==0.0)
return(false);
//--- change value
s=1/s;
for(i_=0; i_<npoints; i_++)
p[i_]=s*d2[i_];
//--- choose one of points with probability P
//--- random number within (0,1) is generated and
//--- inverse empirical CDF is used to randomly choose a point.
s=0;
v=CMath::RandomReal();
//--- calculation
for(i=0; i<npoints; i++)
{
s=s+p[i];
//--- check
if(v<=s || i==npoints-1)
{
for(i_=0; i_<=nvars-1; i_++)
centers.Set(cc,i_,xy[i][i_]);
busycenters[cc]=true;
//--- break the cycle
break;
}
}
}
}
//--- return result
return(result);
}
//+------------------------------------------------------------------+
//| Multiclass Fisher LDA |
//+------------------------------------------------------------------+
class CLDA
{
public:
static void FisherLDA(CMatrixDouble &xy,const int npoints,const int nvars,const int nclasses,int &info,double &w[]);
static void FisherLDA(CMatrixDouble &xy,const int npoints,const int nvars,const int nclasses,int &info,CRowDouble &w);
static void FisherLDAN(CMatrixDouble &xy,const int npoints,const int nvars,const int nclasses,int &info,CMatrixDouble &w);
};
//+------------------------------------------------------------------+
//| Multiclass Fisher LDA |
//| Subroutine finds coefficients of linear combination which |
//| optimally separates training set on classes. |
//| INPUT PARAMETERS: |
//| XY - training set, array[0..NPoints-1,0..NVars]. |
//| First NVars columns store values of |
//| independent variables, next column stores |
//| number of class (from 0 to NClasses-1) which |
//| dataset element belongs to. Fractional values|
//| are rounded to nearest integer. |
//| NPoints - training set size, NPoints>=0 |
//| NVars - number of independent variables, NVars>=1 |
//| NClasses - number of classes, NClasses>=2 |
//| OUTPUT PARAMETERS: |
//| Info - return code: |
//| * -4, if internal EVD subroutine hasn't |
//| converged |
//| * -2, if there is a point with class number |
//| outside of [0..NClasses-1]. |
//| * -1, if incorrect parameters was passed |
//| (NPoints<0, NVars<1, NClasses<2) |
//| * 1, if task has been solved |
//| * 2, if there was a multicollinearity in |
//| training set, but task has been solved.|
//| W - linear combination coefficients, |
//| array[0..NVars-1] |
//+------------------------------------------------------------------+
void CLDA::FisherLDA(CMatrixDouble &xy,const int npoints,
const int nvars,const int nclasses,
int &info,double &w[])
{
CRowDouble W;
FisherLDA(xy,npoints,nvars,nclasses,info,W);
W.ToArray(w);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CLDA::FisherLDA(CMatrixDouble &xy,const int npoints,
const int nvars,const int nclasses,
int &info,CRowDouble &w)
{
//--- create matrix
CMatrixDouble w2;
//--- initialization
info=0;
//--- function call
FisherLDAN(xy,npoints,nvars,nclasses,info,w2);
//--- check and copy
if(info>0)
w=w2.Col(0)+0;
}
//+------------------------------------------------------------------+
//| N-dimensional multiclass Fisher LDA |
//| Subroutine finds coefficients of linear combinations which |
//| optimally separates |
//| training set on classes. It returns N-dimensional basis whose |
//| vector are sorted |
//| by quality of training set separation (in descending order). |
//| INPUT PARAMETERS: |
//| XY - training set, array[0..NPoints-1,0..NVars]. |
//| First NVars columns store values of |
//| independent variables, next column stores |
//| number of class (from 0 to NClasses-1) which |
//| dataset element belongs to. Fractional values|
//| are rounded to nearest integer. |
//| NPoints - training set size, NPoints>=0 |
//| NVars - number of independent variables, NVars>=1 |
//| NClasses - number of classes, NClasses>=2 |
//| OUTPUT PARAMETERS: |
//| Info - return code: |
//| * -4, if internal EVD subroutine hasn't |
//| converged |
//| * -2, if there is a point with class number |
//| outside of [0..NClasses-1]. |
//| * -1, if incorrect parameters was passed |
//| (NPoints<0, NVars<1, NClasses<2) |
//| * 1, if task has been solved |
//| * 2, if there was a multicollinearity in |
//| training set, but task has been solved.|
//| W - basis, array[0..NVars-1,0..NVars-1] |
//| columns of matrix stores basis vectors, |
//| sorted by quality of training set separation |
//| (in descending order) |
//+------------------------------------------------------------------+
void CLDA::FisherLDAN(CMatrixDouble &xy,const int npoints,const int nvars,
const int nclasses,int &info,CMatrixDouble &w)
{
//--- create variables
int i=0;
int j=0;
int k=0;
int m=0;
double v=0;
int i_=0;
//--- creating arrays
int c[];
vector<double> mu;
int nc[];
vector<double> tf;
CRowDouble d;
CRowDouble d2;
CRowDouble work;
//--- create matrix
CMatrixDouble muc;
CMatrixDouble sw;
CMatrixDouble st;
CMatrixDouble z;
CMatrixDouble z2;
CMatrixDouble tm;
CMatrixDouble sbroot;
CMatrixDouble a;
CMatrixDouble xyc;
CMatrixDouble xyproj;
CMatrixDouble wproj;
//--- initialization
info=0;
//--- Test data
if((npoints<0 || nvars<1) || nclasses<2)
{
info=-1;
return;
}
mu=xy.Col(nvars);
mu.Resize(npoints);
//--- check
if((int)MathRound(mu.Min())<0 || (int)MathRound(mu.Max())>=nclasses)
{
info=-2;
return;
}
//--- change value
info=1;
//--- Special case: NPoints<=1
//--- Degenerate task.
if(npoints<=1)
{
info=2;
//--- initialization
w=matrix<double>::Identity(nvars,nvars);
//--- exit the function
return;
}
//--- Prepare temporaries
tf.Resize(nvars);
work.Resize(MathMax(nvars,npoints)+1);
//--- Convert class labels from reals to integers (just for convenience)
ArrayResizeAL(c,npoints);
for(i=0; i<npoints; i++)
c[i]=(int)MathRound(xy.Get(i,nvars));
//--- Calculate class sizes and means
mu=xy.Sum(0);
mu.Resize(nvars);
muc=matrix<double>::Zeros(nclasses,nvars+1);
ArrayResize(nc,nclasses);
ArrayInitialize(nc,0);
//--- calculation
for(i=0; i<npoints; i++)
{
for(i_=0; i_<nvars; i_++)
muc.Add(c[i],i_,xy.Get(i,i_));
nc[c[i]]++;
}
muc.Resize(nclasses,nvars);
for(i=0; i<nclasses; i++)
muc.Row(i,muc[i]/(double)nc[i]);
//--- change values
mu=mu/(double)npoints;
//--- Create ST matrix
xyc=xy.Transpose()+0;
xyc.Resize(nvars,npoints);
for(k=0; k<nvars; k++)
xyc.Row(k,xyc[k]-mu[k]);
st=xyc.MatMul(xyc,true)+0;
//--- Create SW matrix
xyc=xy.Transpose()+0;
xyc.Resize(nvars,npoints);
for(k=0; k<npoints; k++)
xyc.Col(k,xyc.Col(k)-muc[c[k]]);
sw=xyc.MatMul(xyc,true)+0;
//--- Maximize ratio J=(w'*ST*w)/(w'*SW*w).
//--- First,make transition from w to v such that w'*ST*w becomes v'*v:
//--- v=root(ST)*w=R*w
//--- R=root(D)*Z'
//--- w=(root(ST)^-1)*v=RI*v
//--- RI=Z*inv(root(D))
//--- J=(v'*v)/(v'*(RI'*SW*RI)*v)
//--- ST=Z*D*Z'
//--- so we have
//--- J=(v'*v) / (v'*(inv(root(D))*Z'*SW*Z*inv(root(D)))*v)=
//=(v'*v) / (v'*A*v)
if(!CEigenVDetect::SMatrixEVD(st,nvars,1,true,d,z))
{
info=-4;
return;
}
//--- allocation
w.Resize(nvars,nvars);
//--- check
if(d[nvars-1]<=0.0 || d[0]<=1000*CMath::m_machineepsilon*d[nvars-1])
{
//--- Special case: D[NVars-1]<=0
//--- Degenerate task (all variables takes the same value).
if(d[nvars-1]<=0.0)
{
info=2;
w=matrix<double>::Identity(nvars,nvars);
//--- exit the function
return;
}
//--- Special case: degenerate ST matrix,multicollinearity found.
//--- Since we know ST eigenvalues/vectors we can translate task to
//--- non-degenerate form.
//--- Let WG is orthogonal basis of the non zero variance subspace
//--- of the ST and let WZ is orthogonal basis of the zero variance
//--- subspace.
//--- Projection on WG allows us to use LDA on reduced M-dimensional
//--- subspace,N-M vectors of WZ allows us to update reduced LDA
//--- factors to full N-dimensional subspace.
m=0;
for(k=0; k<nvars; k++)
{
//--- check
if(d[k]<=1000*CMath::m_machineepsilon*d[nvars-1])
m=k+1;
}
//--- check
if(!CAp::Assert(m!=0,__FUNCTION__+": internal error #1"))
return;
//--- allocation
xyproj.Resize(npoints,nvars-m+1);
//--- function call
CBlas::MatrixMatrixMultiply(xy,0,npoints-1,0,nvars-1,false,z,0,nvars-1,m,nvars-1,false,1.0,xyproj,0,npoints-1,0,nvars-m-1,0.0,work);
for(i=0; i<npoints; i++)
xyproj.Set(i,nvars-m,xy.Get(i,nvars));
//--- function call
FisherLDAN(xyproj,npoints,nvars-m,nclasses,info,wproj);
//--- check
if(info<0)
return;
//--- function call
CBlas::MatrixMatrixMultiply(z,0,nvars-1,m,nvars-1,false,wproj,0,nvars-m-1,0,nvars-m-1,false,1.0,w,0,nvars-1,0,nvars-m-1,0.0,work);
//--- change values
for(k=nvars-m; k<nvars; k++)
w.Col(k,z.Col(k-nvars+m)+0);
info=2;
}
else
{
//--- General case: no multicollinearity
tm.Resize(nvars,nvars);
a.Resize(nvars,nvars);
//--- function call
tm=sw.MatMul(z,false)+0;
a=z.Transpose()+0;
a=a.MatMul(tm,false)+0;
//--- change values
for(i=0; i<nvars; i++)
for(j=0; j<nvars; j++)
a.Mul(i,j,1.0/MathSqrt(d[i]*d[j]));
//--- check
if(!CEigenVDetect::SMatrixEVD(a,nvars,1,true,d2,z2))
{
info=-4;
return;
}
//--- calculation
for(k=0; k<nvars; k++)
z2.Row(k,z2[k]/MathSqrt(d[k]));
w=z.MatMul(z2,false)+0;
}
//--- Post-processing:
//--- * normalization
//--- * converting to non-negative form,if possible
for(k=0; k<nvars; k++)
{
//--- calculation
v=MathPow(w.Col(k)+0,2.0).Sum();
w.Col(k,w.Col(k)/MathSqrt(v));
v=(w.Col(k)+0).Sum();
//--- check
if(v<0.0)
w.Col(k,w.Col(k)*(-1.0));
}
}
//+------------------------------------------------------------------+
//| Auxiliary class for CLinReg |
//+------------------------------------------------------------------+
class CLinearModel
{
public:
CRowDouble m_w;
//--- constructor, destructor
CLinearModel(void) {}
~CLinearModel(void) {}
//--- copy
void Copy(const CLinearModel &obj) { m_w=obj.m_w; }
//--- overloading
void operator=(const CLinearModel &obj) { Copy(obj); }
};
//+------------------------------------------------------------------+
//| This class is a shell for class CLinearModel |
//+------------------------------------------------------------------+
class CLinearModelShell
{
private:
CLinearModel m_innerobj;
public:
//--- constructors, destructor
CLinearModelShell(void) {}
CLinearModelShell(CLinearModel &obj) { m_innerobj.Copy(obj); }
~CLinearModelShell(void) {}
//--- method
CLinearModel *GetInnerObj(void) { return(GetPointer(m_innerobj)); }
};
//+------------------------------------------------------------------+
//| LRReport structure contains additional information about linear |
//| model: |
//| * C - covariation matrix, array[0..NVars,0..NVars].|
//| C[i,j] = Cov(A[i],A[j]) |
//| * RMSError - root mean square error on a training set |
//| * AvgError - average error on a training set |
//| * AvgRelError - average relative error on a training set |
//| (excluding observations with zero function |
//| value). |
//| * CVRMSError - leave-one-out cross-validation estimate of |
//| generalization error. Calculated using fast |
//| algorithm with O(NVars*NPoints) complexity. |
//| * CVAvgError - cross-validation estimate of average error |
//| * CVAvgRelError - cross-validation estimate of average relative|
//| error |
//| All other fields of the structure are intended for internal use |
//| and should not be used outside ALGLIB. |
//+------------------------------------------------------------------+
class CLRReport
{
public:
//--- variables
double m_RMSError;
double m_AvgError;
double m_AvgRelError;
double m_cvrmserror;
double m_cvavgerror;
double m_cvavgrelerror;
int m_ncvdefects;
//--- array
CRowInt m_cvdefects;
//--- matrix
CMatrixDouble m_c;
//--- constructor, destructor
CLRReport(void);
~CLRReport(void) {}
//--- copy
void Copy(const CLRReport &obj);
//--- overloading
void operator=(const CLRReport &obj) { Copy(obj); }
};
//+------------------------------------------------------------------+
//| Constructor |
//+------------------------------------------------------------------+
CLRReport::CLRReport(void)
{
//--- copy variables
m_RMSError=0;
m_AvgError=0;
m_AvgRelError=0;
m_cvrmserror=0;
m_cvavgerror=0;
m_cvavgrelerror=0;
m_ncvdefects=0;
}
//+------------------------------------------------------------------+
//| Copy |
//+------------------------------------------------------------------+
void CLRReport::Copy(const CLRReport &obj)
{
//--- copy variables
m_RMSError=obj.m_RMSError;
m_AvgError=obj.m_AvgError;
m_AvgRelError=obj.m_AvgRelError;
m_cvrmserror=obj.m_cvrmserror;
m_cvavgerror=obj.m_cvavgerror;
m_cvavgrelerror=obj.m_cvavgrelerror;
m_ncvdefects=obj.m_ncvdefects;
//--- copy array
m_cvdefects=obj.m_cvdefects;
//--- copy matrix
m_c=obj.m_c;
}
//+------------------------------------------------------------------+
//| LRReport structure contains additional information about linear |
//| model: |
//| * C - covariation matrix, array[0..NVars,0..NVars].|
//| C[i,j]=Cov(A[i],A[j]) |
//| * RMSError - root mean square error on a training set |
//| * AvgError - average error on a training set |
//| * AvgRelError - average relative error on a training set |
//| (excluding observations with zero function |
//| value). |
//| * CVRMSError - leave-one-out cross-validation estimate of |
//| generalization error. Calculated using fast |
//| algorithm with O(NVars*NPoints) complexity. |
//| * CVAvgError - cross-validation estimate of average error |
//| * CVAvgRelError - cross-validation estimate of average relative|
//| error |
//| All other fields of the structure are intended for internal use |
//| and should not be used outside ALGLIB. |
//+------------------------------------------------------------------+
class CLRReportShell
{
private:
CLRReport m_innerobj;
public:
//--- constructors, destructor
CLRReportShell(void) {}
CLRReportShell(CLRReport &obj) { m_innerobj.Copy(obj); }
~CLRReportShell(void) {}
//--- methods
double GetRMSError(void);
void SetRMSError(const double d);
double GetAvgError(void);
void SetAvgError(const double d);
double GetAvgRelError(void);
void SetAvgRelError(const double d);
double GetCVRMSError(void);
void SetCVRMSError(const double d);
double GetCVAvgError(void);
void SetCVAvgError(const double d);
double GetCVAvgRelError(void);
void SetCVAvgRelError(const double d);
int GetNCVDEfects(void);
void SetNCVDEfects(const int i);
CLRReport *GetInnerObj(void);
};
//+------------------------------------------------------------------+
//| Returns the value of the variable rmserror |
//+------------------------------------------------------------------+
double CLRReportShell::GetRMSError(void)
{
return(m_innerobj.m_RMSError);
}
//+------------------------------------------------------------------+
//| Changing the value of the variable rmserror |
//+------------------------------------------------------------------+
void CLRReportShell::SetRMSError(const double d)
{
m_innerobj.m_RMSError=d;
}
//+------------------------------------------------------------------+
//| Returns the value of the variable avgerror |
//+------------------------------------------------------------------+
double CLRReportShell::GetAvgError(void)
{
return(m_innerobj.m_AvgError);
}
//+------------------------------------------------------------------+
//| Changing the value of the variable avgerror |
//+------------------------------------------------------------------+
void CLRReportShell::SetAvgError(const double d)
{
m_innerobj.m_AvgError=d;
}
//+------------------------------------------------------------------+
//| Returns the value of the variable avgrelerror |
//+------------------------------------------------------------------+
double CLRReportShell::GetAvgRelError(void)
{
return(m_innerobj.m_AvgRelError);
}
//+------------------------------------------------------------------+
//| Changing the value of the variable avgrelerror |
//+------------------------------------------------------------------+
void CLRReportShell::SetAvgRelError(const double d)
{
m_innerobj.m_AvgRelError=d;
}
//+------------------------------------------------------------------+
//| Returns the value of the variable cvrmserror |
//+------------------------------------------------------------------+
double CLRReportShell::GetCVRMSError(void)
{
return(m_innerobj.m_cvrmserror);
}
//+------------------------------------------------------------------+
//| Changing the value of the variable cvrmserror |
//+------------------------------------------------------------------+
void CLRReportShell::SetCVRMSError(const double d)
{
m_innerobj.m_cvrmserror=d;
}
//+------------------------------------------------------------------+
//| Returns the value of the variable cvavgerror |
//+------------------------------------------------------------------+
double CLRReportShell::GetCVAvgError(void)
{
return(m_innerobj.m_cvavgerror);
}
//+------------------------------------------------------------------+
//| Changing the value of the variable cvavgerror |
//+------------------------------------------------------------------+
void CLRReportShell::SetCVAvgError(const double d)
{
m_innerobj.m_cvavgerror=d;
}
//+------------------------------------------------------------------+
//| Returns the value of the variable cvavgrelerror |
//+------------------------------------------------------------------+
double CLRReportShell::GetCVAvgRelError(void)
{
return(m_innerobj.m_cvavgrelerror);
}
//+------------------------------------------------------------------+
//| Changing the value of the variable cvavgrelerror |
//+------------------------------------------------------------------+
void CLRReportShell::SetCVAvgRelError(const double d)
{
m_innerobj.m_cvavgrelerror=d;
}
//+------------------------------------------------------------------+
//| Returns the value of the variable ncvdefects |
//+------------------------------------------------------------------+
int CLRReportShell::GetNCVDEfects(void)
{
return(m_innerobj.m_ncvdefects);
}
//+------------------------------------------------------------------+
//| Changing the value of the variable ncvdefects |
//+------------------------------------------------------------------+
void CLRReportShell::SetNCVDEfects(const int i)
{
m_innerobj.m_ncvdefects=i;
}
//+------------------------------------------------------------------+
//| Return object of class |
//+------------------------------------------------------------------+
CLRReport *CLRReportShell::GetInnerObj(void)
{
return(GetPointer(m_innerobj));
}
//+------------------------------------------------------------------+
//| Linear regression class |
//+------------------------------------------------------------------+
class CLinReg
{
public:
//--- constant
static const int m_lrvnum;
//--- public methods
static void LRBuild(CMatrixDouble &xy,const int npoints,const int nvars,int &info,CLinearModel &lm,CLRReport &ar);
static void LRBuildS(CMatrixDouble &xy,double &s[],const int npoints,const int nvars,int &info,CLinearModel &lm,CLRReport &ar);
static void LRBuildS(CMatrixDouble &xy,CRowDouble &s,const int npoints,const int nvars,int &info,CLinearModel &lm,CLRReport &ar);
static void LRBuildZS(CMatrixDouble &xy,double &s[],const int npoints,const int nvars,int &info,CLinearModel &lm,CLRReport &ar);
static void LRBuildZS(CMatrixDouble &xy,CRowDouble &s,const int npoints,const int nvars,int &info,CLinearModel &lm,CLRReport &ar);
static void LRBuildZ(CMatrixDouble &xy,const int npoints,const int nvars,int &info,CLinearModel &lm,CLRReport &ar);
static void LRUnpack(CLinearModel &lm,double &v[],int &nvars);
static void LRUnpack(CLinearModel &lm,CRowDouble &v,int &nvars);
static void LRPack(double &v[],const int nvars,CLinearModel &lm);
static void LRPack(CRowDouble &v,const int nvars,CLinearModel &lm);
static double LRProcess(CLinearModel &lm,double &x[]);
static double LRProcess(CLinearModel &lm,CRowDouble &x);
static double LRRMSError(CLinearModel &lm,CMatrixDouble &xy,const int npoints);
static double LRAvgError(CLinearModel &lm,CMatrixDouble &xy,const int npoints);
static double LRAvgRelError(CLinearModel &lm,CMatrixDouble &xy,const int npoints);
static void LRCopy(CLinearModel &lm1,CLinearModel &lm2);
static void LRLines(CMatrixDouble &xy,double &s[],const int n,int &info,double &a,double &b,double &vara,double &varb,double &covab,double &corrab,double &p);
static void LRLines(CMatrixDouble &xy,CRowDouble &s,const int n,int &info,double &a,double &b,double &vara,double &varb,double &covab,double &corrab,double &p);
static void LRLine(CMatrixDouble &xy,const int n,int &info,double &a,double &b);
private:
static void LRInternal(CMatrixDouble &xy,CRowDouble &s,const int npoints,const int nvars,int &info,CLinearModel &lm,CLRReport &ar);
};
//+------------------------------------------------------------------+
//| Initialize constant |
//+------------------------------------------------------------------+
const int CLinReg::m_lrvnum=5;
//+------------------------------------------------------------------+
//| Linear regression |
//| Subroutine builds model: |
//| Y = A(0)*X[0] + ... + A(N-1)*X[N-1] + A(N) |
//| and model found in ALGLIB format, covariation matrix, training |
//| set errors (rms, average, average relative) and leave-one-out |
//| cross-validation estimate of the generalization error. CV |
//| estimate calculated using fast algorithm with O(NPoints*NVars) |
//| complexity. |
//| When covariation matrix is calculated standard deviations of|
//| function values are assumed to be equal to RMS error on the |
//| training set. |
//| INPUT PARAMETERS: |
//| XY - training set, array [0..NPoints-1,0..NVars]: |
//| * NVars columns - independent variables |
//| * last column - dependent variable |
//| NPoints - training set size, NPoints>NVars+1 |
//| NVars - number of independent variables |
//| OUTPUT PARAMETERS: |
//| Info - return code: |
//| * -255, in case of unknown internal error |
//| * -4, if internal SVD subroutine haven't |
//| converged |
//| * -1, if incorrect parameters was passed |
//| (NPoints<NVars+2, NVars<1). |
//| * 1, if subroutine successfully finished |
//| LM - linear model in the ALGLIB format. Use |
//| subroutines of this unit to work with the |
//| model. |
//| AR - additional results |
//+------------------------------------------------------------------+
void CLinReg::LRBuild(CMatrixDouble &xy,const int npoints,const int nvars,
int &info,CLinearModel &lm,CLRReport &ar)
{
//--- initialization
info=0;
//--- check
if(npoints<=nvars+1 || nvars<1)
{
info=-1;
return;
}
//--- create variables
int i=0;
double sigma2=0;
int i_=0;
//--- create array
CRowDouble s;
//--- allocation
s=vector<double>::Ones(npoints);
//--- function call
LRBuildS(xy,s,npoints,nvars,info,lm,ar);
//--- check
if(info<0)
return;
//--- calculation
sigma2=CMath::Sqr(ar.m_RMSError)*npoints/(npoints-nvars-1);
ar.m_c*=sigma2;
}
//+------------------------------------------------------------------+
//| Linear regression |
//| Variant of LRBuild which uses vector of standatd deviations |
//| (errors in function values). |
//| INPUT PARAMETERS: |
//| XY - training set, array [0..NPoints-1,0..NVars]: |
//| * NVars columns - independent variables |
//| * last column - dependent variable |
//| S - standard deviations (errors in function |
//| values) array[0..NPoints-1], S[i]>0. |
//| NPoints - training set size, NPoints>NVars+1 |
//| NVars - number of independent variables |
//| OUTPUT PARAMETERS: |
//| Info - return code: |
//| * -255, in case of unknown internal error |
//| * -4, if internal SVD subroutine haven't |
//| converged |
//| * -1, if incorrect parameters was passed |
//| (NPoints<NVars+2, NVars<1). |
//| * -2, if S[I]<=0 |
//| * 1, if subroutine successfully finished |
//| LM - linear model in the ALGLIB format. Use |
//| subroutines of this unit to work with the |
//| model. |
//| AR - additional results |
//+------------------------------------------------------------------+
void CLinReg::LRBuildS(CMatrixDouble &xy,double &s[],const int npoints,
const int nvars,int &info,CLinearModel &lm,
CLRReport &ar)
{
CRowDouble S=s;
LRBuildS(xy,S,npoints,nvars,info,lm,ar);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CLinReg::LRBuildS(CMatrixDouble &xy,CRowDouble &s,const int npoints,
const int nvars,int &info,CLinearModel &lm,
CLRReport &ar)
{
//--- Test parameters
if(npoints<=nvars+1 || nvars<1)
{
info=-1;
return;
}
//--- create variables
double v=0;
int offs=0;
double mean=0;
double variance=0;
double skewness=0;
double kurtosis=0;
int i_=0;
//--- initialization
info=0;
//--- creating arrays
vector<double> means;
vector<double> sigmas=vector<double>::Zeros(nvars);
CRowDouble x;
//--- create array
CMatrixDouble xyi;
//--- Copy data,add one more column (constant term)
xyi=xy;
xyi.Resize(npoints,nvars+2);
xyi.Col(nvars+1,vector<double>::Ones(npoints));
xyi.SwapCols(nvars,nvars+1);
//--- Standartization
means=xyi.Mean(0);
means.Resize(nvars);
for(int i=0; i<nvars; i++)
{
x=xyi.Col(i)+0;
//--- function call
sigmas[i]=MathSqrt(CBaseStat::SampleVariance(x,npoints));
if(sigmas[i]==0.0)
sigmas[i]=1.0;
xyi.Col(i,(xyi.Col(i)-means[i])/sigmas[i]);
}
//--- Internal processing
LRInternal(xyi,s,npoints,nvars+1,info,lm,ar);
//--- check
if(info<0)
return;
//--- Un-standartization
offs=(int)MathRound(lm.m_w[3]);
for(int j=0; j<nvars; j++)
{
//--- Constant term is updated (and its covariance too,
//--- since it gets some variance from J-th component)
v=means[j]/sigmas[j];
lm.m_w.Add(offs+nvars,-lm.m_w[offs+j]*v);
ar.m_c.Row(nvars,ar.m_c.Row(nvars)-ar.m_c.Row(j)*v);
ar.m_c.Col(nvars,ar.m_c.Col(nvars)-ar.m_c.Col(j)*v);
//--- J-th term is updated
v=1/sigmas[j];
lm.m_w.Mul(offs+j,v);
ar.m_c.Row(j,ar.m_c[j]*v);
ar.m_c.Col(j,ar.m_c.Col(j)*v);
}
}
//+------------------------------------------------------------------+
//| Like LRBuildS, but builds model |
//| Y=A(0)*X[0] + ... + A(N-1)*X[N-1] |
//| i.m_e. with zero constant term. |
//+------------------------------------------------------------------+
void CLinReg::LRBuildZS(CMatrixDouble &xy,double &s[],const int npoints,
const int nvars,int &info,CLinearModel &lm,
CLRReport &ar)
{
CRowDouble S=s;
LRBuildZS(xy,S,npoints,nvars,info,lm,ar);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CLinReg::LRBuildZS(CMatrixDouble &xy,CRowDouble &s,const int npoints,
const int nvars,int &info,CLinearModel &lm,
CLRReport &ar)
{
//--- Test parameters
if(npoints<=nvars+1 || nvars<1)
{
info=-1;
return;
}
//--- create variables
double v=0;
int offs=0;
double mean=0;
double variance=0;
double skewness=0;
double kurtosis=0;
//--- initialization
info=0;
//--- creating arrays
CRowDouble x;
CRowDouble c;
//--- create matrix
CMatrixDouble xyi;
//--- Copy data,add one more column (constant term)
xyi=xy;
xyi.Resize(npoints,nvars+2);
xyi.Col(nvars+1,vector<double>::Zeros(npoints));
xyi.SwapCols(nvars,nvars+1);
//--- Standartization: unusual scaling
c.Resize(nvars);
for(int j=0; j<nvars; j++)
{
x=xy.Col(j)+0;
//--- function call
CBaseStat::SampleMoments(x,npoints,mean,variance,skewness,kurtosis);
//--- check
if(MathAbs(mean)>MathSqrt(variance))
{
//--- variation is relatively small,it is better to
//--- bring mean value to 1
c.Set(j,mean);
}
else
{
//--- variation is large,it is better to bring variance to 1
if(variance==0.0)
c.Set(j,1.0);
else
c.Set(j,MathSqrt(variance));
}
xyi.Col(j,x/c[j]+0);
}
//--- Internal processing
LRInternal(xyi,s,npoints,nvars+1,info,lm,ar);
//--- check
if(info<0)
return;
//--- Un-standartization
offs=(int)MathRound(lm.m_w[3]);
for(int j=0; j<nvars; j++)
{
//--- J-th term is updated
v=1/c[j];
lm.m_w.Mul(offs+j,v);
ar.m_c.Row(j,ar.m_c[j]*v);
ar.m_c.Col(j,ar.m_c.Col(j)*v);
}
}
//+------------------------------------------------------------------+
//| Like LRBuild but builds model |
//| Y=A(0)*X[0] + ... + A(N-1)*X[N-1] |
//| i.m_e. with zero constant term. |
//+------------------------------------------------------------------+
void CLinReg::LRBuildZ(CMatrixDouble &xy,const int npoints,const int nvars,
int &info,CLinearModel &lm,CLRReport &ar)
{
//--- check
if(npoints<=nvars+1 || nvars<1)
{
info=-1;
return;
}
//--- create array
CRowDouble s=vector<double>::Ones(npoints);
//--- initialization
info=0;
//--- function call
LRBuildZS(xy,s,npoints,nvars,info,lm,ar);
//--- check
if(info<0)
return;
//--- calculation
double sigma2=CMath::Sqr(ar.m_RMSError)*npoints/(npoints-nvars-1);
ar.m_c*=sigma2;
}
//+------------------------------------------------------------------+
//| Unpacks coefficients of linear model. |
//| INPUT PARAMETERS: |
//| LM - linear model in ALGLIB format |
//| OUTPUT PARAMETERS: |
//| V - coefficients,array[0..NVars] |
//| constant term (intercept) is stored in the |
//| V[NVars]. |
//| NVars - number of independent variables (one less |
//| than number of coefficients) |
//+------------------------------------------------------------------+
void CLinReg::LRUnpack(CLinearModel &lm,double &v[],int &nvars)
{
CRowDouble V;
LRUnpack(lm,V,nvars);
V.ToArray(v);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CLinReg::LRUnpack(CLinearModel &lm,CRowDouble &v,int &nvars)
{
//--- create variables
int offs=0;
int i1_=0;
//--- initialization
nvars=0;
//--- check
if(!CAp::Assert((int)MathRound(lm.m_w[1])==m_lrvnum,__FUNCTION__+": Incorrect LINREG version!"))
{
v.Resize(0);
return;
}
//--- change values
nvars=(int)MathRound(lm.m_w[2]);
offs=(int)MathRound(lm.m_w[3]);
//--- allocation
v.Resize(nvars+1);
//--- calculation
i1_=offs;
for(int i_=0; i_<=nvars; i_++)
v.Set(i_,lm.m_w[i_+i1_]);
}
//+------------------------------------------------------------------+
//| "Packs" coefficients and creates linear model in ALGLIB format |
//| (LRUnpack reversed). |
//| INPUT PARAMETERS: |
//| V - coefficients, array[0..NVars] |
//| NVars - number of independent variables |
//| OUTPUT PAREMETERS: |
//| LM - linear model. |
//+------------------------------------------------------------------+
void CLinReg::LRPack(double &v[],const int nvars,CLinearModel &lm)
{
CRowDouble V=v;
LRPack(V,nvars,lm);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CLinReg::LRPack(CRowDouble &v,const int nvars,CLinearModel &lm)
{
//--- create variables
int offs=0;
int i1_=0;
//--- allocation
lm.m_w.Resize(5+nvars);
//--- change values
offs=4;
lm.m_w.Set(0,4+nvars+1);
lm.m_w.Set(1,m_lrvnum);
lm.m_w.Set(2,nvars);
lm.m_w.Set(3,offs);
//--- calculation
i1_=-offs;
for(int i_=offs; i_<=offs+nvars; i_++)
lm.m_w.Set(i_,v[i_+i1_]);
}
//+------------------------------------------------------------------+
//| Procesing |
//| INPUT PARAMETERS: |
//| LM - linear model |
//| X - input vector, array[0..NVars-1]. |
//| Result: |
//| value of linear model regression estimate |
//+------------------------------------------------------------------+
double CLinReg::LRProcess(CLinearModel &lm,double &x[])
{
CRowDouble X=x;
return(LRProcess(lm,X));
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
double CLinReg::LRProcess(CLinearModel &lm,CRowDouble &x)
{
//--- create variables
double v=0;
int offs=0;
int nvars=0;
int i1_=0;
//--- check
if(!CAp::Assert((int)MathRound(lm.m_w[1])==m_lrvnum,__FUNCTION__+": Incorrect LINREG version!"))
return(EMPTY_VALUE);
//--- change values
nvars=(int)MathRound(lm.m_w[2]);
offs=(int)MathRound(lm.m_w[3]);
i1_=offs;
v=0.0;
//--- calculation
for(int i_=0; i_<nvars; i_++)
v+=x[i_]*lm.m_w[i_+i1_];
//--- return result
return(v+lm.m_w[offs+nvars]);
}
//+------------------------------------------------------------------+
//| RMS error on the test set |
//| INPUT PARAMETERS: |
//| LM - linear model |
//| XY - test set |
//| NPoints - test set size |
//| RESULT: |
//| root mean square error. |
//+------------------------------------------------------------------+
double CLinReg::LRRMSError(CLinearModel &lm,CMatrixDouble &xy,
const int npoints)
{
//--- check
if(!CAp::Assert((int)MathRound(lm.m_w[1])==m_lrvnum,__FUNCTION__+": Incorrect LINREG version!"))
return(EMPTY_VALUE);
//--- create variables
double result=0;
double v=0;
int nvars=(int)MathRound(lm.m_w[2]);
CRowDouble x;
//--- calculation
for(int i=0; i<npoints; i++)
{
x=xy[i]+0;
v=LRProcess(lm,x);
result+=CMath::Sqr(v-xy.Get(i,nvars));
}
//--- return result
return(MathSqrt(result/npoints));
}
//+------------------------------------------------------------------+
//| Average error on the test set |
//| INPUT PARAMETERS: |
//| LM - linear model |
//| XY - test set |
//| NPoints - test set size |
//| RESULT: |
//| average error. |
//+------------------------------------------------------------------+
double CLinReg::LRAvgError(CLinearModel &lm,CMatrixDouble &xy,
const int npoints)
{
//--- check
if(!CAp::Assert((int)MathRound(lm.m_w[1])==m_lrvnum,__FUNCTION__+": Incorrect LINREG version!"))
return(EMPTY_VALUE);
//--- create variables
double result=0;
double v=0;
int nvars=(int)MathRound(lm.m_w[2]);
CRowDouble x;
//--- calculation
for(int i=0; i<npoints; i++)
{
x=xy[i]+0;
v=LRProcess(lm,x);
result+=MathAbs(v-xy.Get(i,nvars));
}
//--- return result
return(result/npoints);
}
//+------------------------------------------------------------------+
//| RMS error on the test set |
//| INPUT PARAMETERS: |
//| LM - linear model |
//| XY - test set |
//| NPoints - test set size |
//| RESULT: |
//| average relative error. |
//+------------------------------------------------------------------+
double CLinReg::LRAvgRelError(CLinearModel &lm,CMatrixDouble &xy,
const int npoints)
{
//--- check
if(!CAp::Assert((int)MathRound(lm.m_w[1])==m_lrvnum,__FUNCTION__+": Incorrect LINREG version!"))
return(EMPTY_VALUE);
//--- create variables
double result=0;
int k=0;
double v=0;
double y=0;
int nvars=(int)MathRound(lm.m_w[2]);
CRowDouble x;
//--- calculation
for(int i=0; i<npoints; i++)
{
y=xy.Get(i,nvars);
//--- check
if(y==0.0)
continue;
//--- calculation
x=xy[i]+0;
v=LRProcess(lm,x);
//--- get result
result+=MathAbs((v-y)/y);
k++;
}
//--- check
if(k!=0)
result/=k;
//--- return result
return(result);
}
//+------------------------------------------------------------------+
//| Copying of LinearModel strucure |
//| INPUT PARAMETERS: |
//| LM1 - original |
//| OUTPUT PARAMETERS: |
//| LM2 - copy |
//+------------------------------------------------------------------+
void CLinReg::LRCopy(CLinearModel &lm1,CLinearModel &lm2)
{
lm2.m_w=lm1.m_w;
}
//+------------------------------------------------------------------+
//| Class method |
//+------------------------------------------------------------------+
void CLinReg::LRLines(CMatrixDouble &xy,double &s[],const int n,
int &info,double &a,double &b,double &vara,
double &varb,double &covab,double &corrab,
double &p)
{
CRowDouble S=s;
LRLines(xy,S,n,info,a,b,vara,varb,covab,corrab,p);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CLinReg::LRLines(CMatrixDouble &xy,CRowDouble &s,const int n,
int &info,double &a,double &b,double &vara,
double &varb,double &covab,double &corrab,
double &p)
{
//--- initialization
info=0;
a=0;
b=0;
vara=0;
varb=0;
covab=0;
corrab=0;
p=0;
//--- check
if(n<2)
{
info=-1;
return;
}
if(s.Min()<=0.0)
{
info=-2;
return;
}
//--- create variables
double ss=0;
double sx=0;
double sxx=0;
double sy=0;
double stt=0;
double e1=0;
double e2=0;
double t=0;
double chi2=0;
//--- change value
info=1;
//--- Calculate S,SX,SY,SXX
vector<double> X=xy.Col(0);
vector<double> Y=xy.Col(1);
vector<double> S=s.ToVector();
S.Resize(n);
X.Resize(n);
Y.Resize(n);
vector<double> SS=MathPow(S*S,-1);
ss=SS.Sum();
sx=(X*SS).Sum();
sy=(Y*SS).Sum();
sxx=(X*X*SS).Sum();
//--- Test for condition number
t=MathSqrt(4*CMath::Sqr(sx)+CMath::Sqr(ss-sxx));
e1=0.5*(ss+sxx+t);
e2=0.5*(ss+sxx-t);
//--- check
if(MathMin(e1,e2)<=1000*CMath::m_machineepsilon*MathMax(e1,e2))
{
info=-3;
return;
}
//--- Calculate A,B
vector<double> T=(X-sx/ss)/S;
b=((T*Y)/S).Sum();
stt=(T*T).Sum();
b=b/stt;
a=(sy-sx*b)/ss;
//--- Calculate goodness-of-fit
if(n>2)
{
chi2=(MathPow((Y-X*b-a)/S,2)).Sum();
//--- function call
p=CIncGammaF::IncompleteGammaC((double)(n-2)/2.0,chi2/2.0);
}
else
p=1;
//--- Calculate other parameters
vara=(1+CMath::Sqr(sx)/(ss*stt))/ss;
varb=1/stt;
covab=-(sx/(ss*stt));
corrab=covab/MathSqrt(vara*varb);
}
//+------------------------------------------------------------------+
//| Class method |
//+------------------------------------------------------------------+
void CLinReg::LRLine(CMatrixDouble &xy,const int n,int &info,
double &a,double &b)
{
//--- initialization
info=0;
a=0;
b=0;
//--- check
if(n<2)
{
info=-1;
return;
}
//--- create variables
double vara=0;
double varb=0;
double covab=0;
double corrab=0;
double p=0;
//--- create array
CRowDouble s=vector<double>::Ones(n);
//--- function call
LRLines(xy,s,n,info,a,b,vara,varb,covab,corrab,p);
}
//+------------------------------------------------------------------+
//| Internal linear regression subroutine |
//+------------------------------------------------------------------+
void CLinReg::LRInternal(CMatrixDouble &xy,CRowDouble &s,const int npoints,
const int nvars,int &info,CLinearModel &lm,
CLRReport &ar)
{
//--- Check for errors in data
if(npoints<nvars || nvars<1)
{
info=-1;
return;
}
if(s.Min()<=0.0)
{
info=-2;
return;
}
//--- create variables
CMatrixDouble a;
CMatrixDouble u;
CMatrixDouble vt;
CMatrixDouble vm;
CMatrixDouble xym;
CRowDouble b;
CRowDouble sv;
CRowDouble t;
CRowDouble svi;
CRowDouble work;
int i=0;
int j=0;
int k=0;
int ncv=0;
int na=0;
int nacv=0;
double r=0;
double p=0;
double epstol=1000;
CLRReport ar2;
int offs=0;
CLinearModel tlm;
int i_=0;
int i1_=0;
//--- Check for errors in data
info=1;
//--- Create design matrix
a.Resize(npoints,nvars);
b.Resize(npoints);
for(i=0; i<npoints; i++)
{
r=1/s[i];
for(i_=0; i_<=nvars-1; i_++)
a.Set(i,i_,r*xy.Get(i,i_));
b.Set(i,xy.Get(i,nvars)/s[i]);
}
//--- Allocate W:
//--- W[0] array size
//--- W[1] version number, 0
//--- W[2] NVars (minus 1, to be compatible with external representation)
//--- W[3] coefficients offset
lm.m_w.Resize(4+nvars);
offs=4;
lm.m_w.Set(0,4+nvars);
lm.m_w.Set(1,m_lrvnum);
lm.m_w.Set(2,nvars-1);
lm.m_w.Set(3,offs);
//--- Solve problem using SVD:
//--- 0. check for degeneracy (different types)
//--- 1. A = U*diag(sv)*V'
//--- 2. T = b'*U
//--- 3. w = SUM((T[i]/sv[i])*V[..,i])
//--- 4. cov(wi,wj) = SUM(Vji*Vjk/sv[i]^2,K=1..M)
//--- see $15.4 of "Numerical Recipes in C" for more information
t.Resize(nvars);
svi.Resize(nvars);
ar.m_c.Resize(nvars,nvars);
vm.Resize(nvars,nvars);
if(!CSingValueDecompose::RMatrixSVD(a,npoints,nvars,1,1,2,sv,u,vt))
{
info=-4;
return;
}
if(sv[0]<=0.0)
{
//--- Degenerate case: zero design matrix.
for(i=offs; i<offs+nvars; i++)
lm.m_w.Set(i,0);
ar.m_RMSError=LRRMSError(lm,xy,npoints);
ar.m_AvgError=LRAvgError(lm,xy,npoints);
ar.m_AvgRelError=LRAvgRelError(lm,xy,npoints);
ar.m_cvrmserror=ar.m_RMSError;
ar.m_cvavgerror=ar.m_AvgError;
ar.m_cvavgrelerror=ar.m_AvgRelError;
ar.m_ncvdefects=0;
ar.m_cvdefects.Resize(nvars);
ar.m_cvdefects.Fill(-1);
ar.m_c=matrix<double>::Zeros(nvars,nvars);
return;
}
if(sv[nvars-1]<=(epstol*CMath::m_machineepsilon*sv[0]))
{
//--- Degenerate case, non-zero design matrix.
//--- We can leave it and solve task in SVD least squares fashion.
//--- Solution and covariance matrix will be obtained correctly,
//--- but CV error estimates - will not. It is better to reduce
//--- it to non-degenerate task and to obtain correct CV estimates.
for(k=nvars; k>=1; k--)
{
if(sv[k-1]>(epstol*CMath::m_machineepsilon*sv[0]))
{
//--- Reduce
xym.Resize(npoints,k+1);
for(i=0; i<npoints; i++)
{
for(j=0; j<k; j++)
{
r=CAblasF::RDotRR(nvars,xy,i,vt,j);
xym.Set(i,j,r);
}
xym.Set(i,k,xy.Get(i,nvars));
}
//--- Solve
LRInternal(xym,s,npoints,k,info,tlm,ar2);
if(info!=1)
return;
//--- Convert back to un-reduced format
for(j=0; j<nvars; j++)
lm.m_w.Set(offs+j,0);
for(j=0; j<k; j++)
{
r=tlm.m_w[offs+j];
i1_=-offs;
for(i_=offs; i_<offs+nvars; i_++)
lm.m_w.Add(i_,r*vt.Get(j,i_+i1_));
}
ar.m_RMSError=ar2.m_RMSError;
ar.m_AvgError=ar2.m_AvgError;
ar.m_AvgRelError=ar2.m_AvgRelError;
ar.m_cvrmserror=ar2.m_cvrmserror;
ar.m_cvavgerror=ar2.m_cvavgerror;
ar.m_cvavgrelerror=ar2.m_cvavgrelerror;
ar.m_ncvdefects=ar2.m_ncvdefects;
ar.m_cvdefects.Resize(nvars);
for(j=0; j<ar.m_ncvdefects; j++)
ar.m_cvdefects.Set(j,ar2.m_cvdefects[j]);
for(j=ar.m_ncvdefects; j<nvars; j++)
ar.m_cvdefects.Set(j,-1);
ar.m_c.Resize(nvars,nvars);
work.Resize(nvars+1);
CBlas::MatrixMatrixMultiply(ar2.m_c,0,k-1,0,k-1,false,vt,0,k-1,0,nvars-1,false,1.0,vm,0,k-1,0,nvars-1,0.0,work);
CBlas::MatrixMatrixMultiply(vt,0,k-1,0,nvars-1,true,vm,0,k-1,0,nvars-1,false,1.0,ar.m_c,0,nvars-1,0,nvars-1,0.0,work);
return;
}
}
info=-255;
return;
}
for(i=0; i<nvars; i++)
{
if(sv[i]>(epstol*CMath::m_machineepsilon*sv[0]))
svi.Set(i,1/sv[i]);
else
svi.Set(i,0);
}
t=vector<double>::Zeros(nvars);
for(i=0; i<npoints; i++)
{
r=b[i];
for(i_=0; i_<nvars; i_++)
t.Add(i_,r*u.Get(i,i_));
}
for(i=0; i<nvars; i++)
lm.m_w.Set(offs+i,0);
for(i=0; i<=nvars-1; i++)
{
r=t[i]*svi[i];
i1_=-offs;
for(i_=offs; i_<offs+nvars; i_++)
lm.m_w.Add(i_,r*vt.Get(i,i_+i1_));
}
for(j=0; j<=nvars-1; j++)
{
r=svi[j];
vm.Col(j,vt[j]*r);
}
for(i=0; i<nvars; i++)
{
for(j=i; j<nvars; j++)
{
r=CAblasF::RDotRR(nvars,vm,i,vm,j);
ar.m_c.Set(i,j,r);
ar.m_c.Set(j,i,r);
}
}
//--- Leave-1-out cross-validation error.
//--- NOTATIONS:
//--- A design matrix
//--- A*x = b original linear least squares task
//--- U*S*V' SVD of A
//--- ai i-th row of the A
//--- bi i-th element of the b
//--- xf solution of the original LLS task
//--- Cross-validation error of i-th element from a sample is
//--- calculated using following formula:
//--- ERRi = ai*xf - (ai*xf-bi*(ui*ui'))/(1-ui*ui') (1)
//--- This formula can be derived from normal equations of the
//--- original task
//--- (A'*A)x = A'*b (2)
//--- by applying modification (zeroing out i-th row of A) to (2):
//--- (A-ai)'*(A-ai) = (A-ai)'*b
//--- and using Sherman-Morrison formula for updating matrix inverse
//--- NOTE 1: b is not zeroed out since it is much simpler and
//--- does not influence final result.
//--- NOTE 2: some design matrices A have such ui that 1-ui*ui'=0.
//--- Formula (1) can't be applied for such cases and they are skipped
//--- from CV calculation (which distorts resulting CV estimate).
//--- But from the properties of U we can conclude that there can
//--- be no more than NVars such vectors. Usually
//--- NVars << NPoints, so in a normal case it only slightly
//--- influences result.
ncv=0;
na=0;
nacv=0;
ar.m_RMSError=0;
ar.m_AvgError=0;
ar.m_AvgRelError=0;
ar.m_cvrmserror=0;
ar.m_cvavgerror=0;
ar.m_cvavgrelerror=0;
ar.m_ncvdefects=0;
ar.m_cvdefects.Resize(nvars);
ar.m_cvdefects.Fill(-1,0,nvars);
for(i=0; i<npoints; i++)
{
//--- Error on a training set
i1_=offs;
r=0.0;
for(i_=0; i_<nvars; i_++)
r+=xy.Get(i,i_)*lm.m_w[i_+i1_];
ar.m_RMSError+=CMath::Sqr(r-xy.Get(i,nvars));
ar.m_AvgError+=MathAbs(r-xy.Get(i,nvars));
if(xy.Get(i,nvars)!=0.0)
{
ar.m_AvgRelError+=MathAbs((r-xy.Get(i,nvars))/xy.Get(i,nvars));
na++;
}
//--- Error using fast leave-one-out cross-validation
p=CAblasF::RDotRR(nvars,u,i,u,i);
if(p>(1-epstol*CMath::m_machineepsilon))
{
ar.m_cvdefects.Set(ar.m_ncvdefects,i);
ar.m_ncvdefects++;
continue;
}
r=s[i]*(r/s[i]-b[i]*p)/(1-p);
ar.m_cvrmserror+=CMath::Sqr(r-xy.Get(i,nvars));
ar.m_cvavgerror+=MathAbs(r-xy.Get(i,nvars));
if(xy.Get(i,nvars)!=0.0)
{
ar.m_cvavgrelerror+=MathAbs((r-xy.Get(i,nvars))/xy.Get(i,nvars));
nacv++;
}
ncv++;
}
if(ncv==0)
{
//--- Something strange: ALL ui are degenerate.
//--- Unexpected...
info=-255;
return;
}
ar.m_RMSError=MathSqrt(ar.m_RMSError/npoints);
ar.m_AvgError=ar.m_AvgError/npoints;
if(na!=0)
ar.m_AvgRelError=ar.m_AvgRelError/na;
ar.m_cvrmserror=MathSqrt(ar.m_cvrmserror/ncv);
ar.m_cvavgerror=ar.m_cvavgerror/ncv;
if(nacv!=0)
ar.m_cvavgrelerror=ar.m_cvavgrelerror/nacv;
}
//+------------------------------------------------------------------+
//| Model's errors: |
//| * RelCLSError - fraction of misclassified cases. |
//| * AvgCE - acerage cross-entropy |
//| * RMSError - root-mean-square error |
//| * AvgError - average error |
//| * AvgRelError - average relative error |
//| NOTE 1: RelCLSError/AvgCE are zero on regression problems. |
//| NOTE 2: on classification problems RMSError/AvgError/AvgRelError |
//| contain errors in prediction of posterior probabilities |
//+------------------------------------------------------------------+
struct CModelErrors
{
double m_RelCLSError;
double m_AvgCE;
double m_RMSError;
double m_AvgError;
double m_AvgRelError;
//---
CModelErrors(void) { ZeroMemory(this); }
~CModelErrors(void) {}
//---
void Copy(const CModelErrors &obj);
//--- overloading
void operator=(const CModelErrors &obj) { Copy(obj); }
};
//+------------------------------------------------------------------+
//| Copy |
//+------------------------------------------------------------------+
void CModelErrors::Copy(const CModelErrors &obj)
{
m_RelCLSError=obj.m_RelCLSError;
m_AvgCE=obj.m_AvgCE;
m_RMSError=obj.m_RMSError;
m_AvgError=obj.m_AvgError;
m_AvgRelError=obj.m_AvgRelError;
}
//+------------------------------------------------------------------+
//|This structure is used to store MLP error and gradient. |
//+------------------------------------------------------------------+
struct CSMLPGrad
{
double m_F;
CRowDouble m_G;
//---
CSMLPGrad(void) { m_F=0; }
~CSMLPGrad(void) {}
//---
void Copy(const CSMLPGrad &obj) { m_F=obj.m_F; m_G=obj.m_G; }
//--- overloading
void operator=(const CSMLPGrad &obj) { Copy(obj); }
};
//+------------------------------------------------------------------+
//| Auxiliary class for CMLPBase |
//+------------------------------------------------------------------+
class CMultilayerPerceptron
{
public:
//--- variables
int m_hlnetworktype;
int m_hlnormtype;
CModelErrors m_err;
CSMLPGrad m_grad;
//--- arrays
CRowInt m_dummyidx;
CRowInt m_hllayersizes;
CRowInt m_hlconnections;
CRowInt m_hlneurons;
CRowInt m_integerbuf;
CRowInt m_structinfo;
CRowDouble m_weights;
CRowDouble m_columnmeans;
CRowDouble m_columnsigmas;
CRowDouble m_neurons;
CRowDouble m_dfdnet;
CRowDouble m_derror;
CRowDouble m_x;
CRowDouble m_y;
CRowDouble m_xyrow;
CRowDouble m_nwbuf;
CRowDouble m_rndbuf;
//--- matrix
CMatrixDouble m_xy;
CMatrixDouble m_dummydxy;
CSparseMatrix m_dummysxy;
//--- constructor, destructor
CMultilayerPerceptron(void) { m_hlnetworktype=0; m_hlnormtype=0; }
~CMultilayerPerceptron(void) {}
//--- copy
void Copy(const CMultilayerPerceptron &obj);
//--- overloading
void operator=(const CMultilayerPerceptron &obj) { Copy(obj); }
};
//+------------------------------------------------------------------+
//| Copy |
//+------------------------------------------------------------------+
void CMultilayerPerceptron::Copy(const CMultilayerPerceptron &obj)
{
//--- copy variables
m_hlnetworktype=obj.m_hlnetworktype;
m_hlnormtype=obj.m_hlnormtype;
m_err=obj.m_err;
m_grad=obj.m_grad;
//--- copy arrays
m_dummyidx=obj.m_dummyidx;
m_hllayersizes=obj.m_hllayersizes;
m_hlconnections=obj.m_hlconnections;
m_hlneurons=obj.m_hlneurons;
m_integerbuf=obj.m_integerbuf;
m_structinfo=obj.m_structinfo;
m_weights=obj.m_weights;
m_columnmeans=obj.m_columnmeans;
m_columnsigmas=obj.m_columnsigmas;
m_neurons=obj.m_neurons;
m_dfdnet=obj.m_dfdnet;
m_derror=obj.m_derror;
m_x=obj.m_x;
m_y=obj.m_y;
m_xyrow=obj.m_xyrow;
m_nwbuf=obj.m_nwbuf;
m_rndbuf=obj.m_rndbuf;
//--- copy matrix
m_xy=obj.m_xy;
m_dummydxy=obj.m_dummydxy;
m_dummysxy=obj.m_dummysxy;
}
//+------------------------------------------------------------------+
//| This class is a shell for class CMultilayerPerceptron |
//+------------------------------------------------------------------+
class CMultilayerPerceptronShell
{
private:
CMultilayerPerceptron m_innerobj;
public:
//--- constructors, destructor
CMultilayerPerceptronShell(void) {}
CMultilayerPerceptronShell(CMultilayerPerceptron &obj) { m_innerobj.Copy(obj); }
~CMultilayerPerceptronShell(void) {}
//--- method
CMultilayerPerceptron *GetInnerObj(void) { return(GetPointer(m_innerobj)); }
};
//+------------------------------------------------------------------+
//| Multilayer perceptron class |
//+------------------------------------------------------------------+
class CMLPBase
{
public:
//--- variables
static const int m_mlpvnum;
static const int m_mlpfirstversion;
static const int m_nfieldwidth;
static const int m_hlconm_nfieldwidth;
static const int m_hlm_nfieldwidth;
static const int m_gradbasecasecost;
static const int m_microbatchsize;
//--- public methods
static int MLPGradSplitCost(void);
static int MLPGradSplitSize(void);
static void MLPCreate0(const int nin,const int nout,CMultilayerPerceptron &network);
static void MLPCreate1(const int nin,const int nhid,const int nout,CMultilayerPerceptron &network);
static void MLPCreate2(const int nin,const int nhid1,const int nhid2,const int nout,CMultilayerPerceptron &network);
static void MLPCreateB0(const int nin,const int nout,const double b,double d,CMultilayerPerceptron &network);
static void MLPCreateB1(const int nin,const int nhid,const int nout,const double b,double d,CMultilayerPerceptron &network);
static void MLPCreateB2(const int nin,const int nhid1,const int nhid2,const int nout,const double b,double d,CMultilayerPerceptron &network);
static void MLPCreateR0(const int nin,const int nout,const double a,const double b,CMultilayerPerceptron &network);
static void MLPCreateR1(const int nin,const int nhid,const int nout,const double a,const double b,CMultilayerPerceptron &network);
static void MLPCreateR2(const int nin,const int nhid1,const int nhid2,const int nout,const double a,const double b,CMultilayerPerceptron &network);
static void MLPCreateC0(const int nin,const int nout,CMultilayerPerceptron &network);
static void MLPCreateC1(const int nin,const int nhid,const int nout,CMultilayerPerceptron &network);
static void MLPCreateC2(const int nin,const int nhid1,const int nhid2,const int nout,CMultilayerPerceptron &network);
static void MLPCopy(const CMultilayerPerceptron &network1,CMultilayerPerceptron &network2);
static bool MLPSameArchitecture(CMultilayerPerceptron &network1,CMultilayerPerceptron &network2);
static void MLPCopyTunableParameters(CMultilayerPerceptron &network1,CMultilayerPerceptron &network2);
static void MLPExportTunableParameters(CMultilayerPerceptron &network,CRowDouble &p,int &pcount);
static void MLPImportTunableParameters(CMultilayerPerceptron &network,CRowDouble &p);
static void MLPSerializeOld(CMultilayerPerceptron &network,double &ra[],int &rlen);
static void MLPSerializeOld(CMultilayerPerceptron &network,CRowDouble &ra,int &rlen);
static void MLPUnserializeOld(double &ra[],CMultilayerPerceptron &network);
static void MLPUnserializeOld(CRowDouble &ra,CMultilayerPerceptron &network);
static void MLPRandomize(CMultilayerPerceptron &network);
static void MLPRandomizeFull(CMultilayerPerceptron &network);
static void MLPInitPreprocessor(CMultilayerPerceptron &network,CMatrixDouble &xy,const int ssize);
static void MLPInitPreprocessorSparse(CMultilayerPerceptron &network,CSparseMatrix &xy,int ssize);
static void MLPInitPreprocessorSubset(CMultilayerPerceptron &network,CMatrixDouble &xy,int setsize,CRowInt &idx,int subsetsize);
static void MLPInitPreprocessorSparseSubset(CMultilayerPerceptron &network,CSparseMatrix &xy,int setsize,CRowInt &idx,int subsetsize);
static void MLPProperties(CMultilayerPerceptron &network,int &nin,int &nout,int &wcount);
static int MLPNTotal(CMultilayerPerceptron &network);
static int MLPGetInputsCount(CMultilayerPerceptron &network);
static int MLPGetOutputsCount(CMultilayerPerceptron &network);
static int MLPGetWeightsCount(CMultilayerPerceptron &network);
static bool MLPIsSoftMax(CMultilayerPerceptron &network);
static int MLPGetLayersCount(CMultilayerPerceptron &network);
static int MLPGetLayerSize(CMultilayerPerceptron &network,const int k);
static void MLPGetInputScaling(CMultilayerPerceptron &network,const int i,double &mean,double &sigma);
static void MLPGetOutputScaling(CMultilayerPerceptron &network,const int i,double &mean,double &sigma);
static void MLPGetNeuronInfo(CMultilayerPerceptron &network,const int k,const int i,int &fkind,double &threshold);
static double MLPGetWeight(CMultilayerPerceptron &network,const int k0,const int i0,const int k1,const int i1);
static void MLPSetInputScaling(CMultilayerPerceptron &network,const int i,const double mean,double sigma);
static void MLPSetOutputScaling(CMultilayerPerceptron &network,const int i,const double mean,double sigma);
static void MLPSetNeuronInfo(CMultilayerPerceptron &network,const int k,const int i,const int fkind,const double threshold);
static void MLPSetWeight(CMultilayerPerceptron &network,const int k0,const int i0,const int k1,const int i1,const double w);
static void MLPActivationFunction(double net,const int k,double &f,double &df,double &d2f);
static void MLPProcess(CMultilayerPerceptron &network,double &x[],double &y[]);
static void MLPProcess(CMultilayerPerceptron &network,CRowDouble &x,CRowDouble &y);
static void MLPProcessI(CMultilayerPerceptron &network,double &x[],double &y[]);
static void MLPProcessI(CMultilayerPerceptron &network,CRowDouble &x,CRowDouble &y);
static double MLPError(CMultilayerPerceptron &network,CMatrixDouble &xy,const int npoints);
static double MLPErrorSparse(CMultilayerPerceptron &network,CSparseMatrix &xy,int npoints);
static double MLPErrorN(CMultilayerPerceptron &network,CMatrixDouble &xy,const int ssize);
static int MLPClsError(CMultilayerPerceptron &network,CMatrixDouble &xy,const int ssize);
static double MLPRelClsError(CMultilayerPerceptron &network,CMatrixDouble &xy,const int npoints);
static double MLPRelClsErrorSparse(CMultilayerPerceptron &network,CSparseMatrix &xy,int npoints);
static double MLPAvgCE(CMultilayerPerceptron &network,CMatrixDouble &xy,const int npoints);
static double MLPAvgCESparse(CMultilayerPerceptron &network,CSparseMatrix &xy,int npoints);
static double MLPRMSError(CMultilayerPerceptron &network,CMatrixDouble &xy,const int npoints);
static double MLPRMSErrorSparse(CMultilayerPerceptron &network,CSparseMatrix &xy,int npoints);
static double MLPAvgError(CMultilayerPerceptron &network,CMatrixDouble &xy,const int npoints);
static double MLPAvgErrorSparse(CMultilayerPerceptron &network,CSparseMatrix &xy,int npoints);
static double MLPAvgRelError(CMultilayerPerceptron &network,CMatrixDouble &xy,const int npoints);
static double MLPAvgRelErrorSparse(CMultilayerPerceptron &network,CSparseMatrix &xy,int npoints);
static void MLPGrad(CMultilayerPerceptron &network,double &x[],double &desiredy[],double &e,double &grad[]);
static void MLPGrad(CMultilayerPerceptron &network,CRowDouble &x,CRowDouble &desiredy,double &e,CRowDouble &grad);
static void MLPGradN(CMultilayerPerceptron &network,double &x[],double &desiredy[],double &e,double &grad[]);
static void MLPGradN(CMultilayerPerceptron &network,CRowDouble &x,CRowDouble &desiredy,double &e,CRowDouble &grad);
static void MLPGradBatch(CMultilayerPerceptron &network,CMatrixDouble &xy,const int ssize,double &e,double &grad[]);
static void MLPGradBatch(CMultilayerPerceptron &network,CMatrixDouble &xy,const int ssize,double &e,CRowDouble &grad);
static void MLPGradBatchSparse(CMultilayerPerceptron &network,CSparseMatrix &xy,int ssize,double &e,CRowDouble &grad);
static void MLPGradBatchSubset(CMultilayerPerceptron &network,CMatrixDouble &xy,int setsize,CRowInt &idx,int subsetsize,double &e,CRowDouble &grad);
static void MLPGradBatchSparseSubset(CMultilayerPerceptron &network,CSparseMatrix &xy,int setsize,CRowInt &idx,int subsetsize,double &e,CRowDouble &grad);
static void MLPGradBatchX(CMultilayerPerceptron &network,CMatrixDouble &densexy,CSparseMatrix &sparsexy,int datasetsize,int datasettype,CRowInt &idx,int subset0,int subset1,int subsettype,double &e,CRowDouble &grad);
static void MLPGradNBatch(CMultilayerPerceptron &network,CMatrixDouble &xy,const int ssize,double &e,double &grad[]);
static void MLPGradNBatch(CMultilayerPerceptron &network,CMatrixDouble &xy,const int ssize,double &e,CRowDouble &grad);
static void MLPHessianNBatch(CMultilayerPerceptron &network,CMatrixDouble &xy,const int ssize,double &e,double &grad[],CMatrixDouble &h);
static void MLPHessianNBatch(CMultilayerPerceptron &network,CMatrixDouble &xy,const int ssize,double &e,CRowDouble &grad,CMatrixDouble &h);
static void MLPHessianBatch(CMultilayerPerceptron &network,CMatrixDouble &xy,const int ssize,double &e,double &grad[],CMatrixDouble &h);
static void MLPHessianBatch(CMultilayerPerceptron &network,CMatrixDouble &xy,const int ssize,double &e,CRowDouble &grad,CMatrixDouble &h);
static void MLPInternalProcessVector(int &structinfo[],double &weights[],double &columnmeans[],double &columnsigmas[],double &neurons[],double &dfdnet[],double &x[],double &y[]);
static void MLPInternalProcessVector(CRowInt &structinfo,CRowDouble &weights,CRowDouble &columnmeans,CRowDouble &columnsigmas,CRowDouble &neurons,CRowDouble &dfdnet,CRowDouble &x,CRowDouble &y);
static void MLPAlloc(CSerializer &s,CMultilayerPerceptron &network);
static void MLPSerialize(CSerializer &s,CMultilayerPerceptron &network);
static void MLPUnserialize(CSerializer &s,CMultilayerPerceptron &network);
static void MLPAllErrorsSubset(CMultilayerPerceptron &network,CMatrixDouble &xy,int setsize,CRowInt &subset,int subsetsize,CModelErrors &rep);
static void MLPAllErrorsSparseSubset(CMultilayerPerceptron &network,CSparseMatrix &xy,int setsize,CRowInt &subset,int subsetsize,CModelErrors &rep);
static double MLPErrorSubset(CMultilayerPerceptron &network,CMatrixDouble &xy,int setsize,CRowInt &subset,int subsetsize);
static double MLPErrorSparseSubset(CMultilayerPerceptron &network,CSparseMatrix &xy,int setsize,CRowInt &subset,int subsetsize);
static void MLPAllErrorsX(CMultilayerPerceptron &network,CMatrixDouble &densexy,CSparseMatrix &sparsexy,int datasetsize,int datasettype,CRowInt &idx,int subset0,int subset1,int subsettype,CModelErrors &rep);
private:
static void AddInputLayer(const int ncount,CRowInt &lsizes,CRowInt &ltypes,CRowInt &lconnfirst,CRowInt &lconnlast,int &lastproc);
static void AddBiasedSummatorLayer(const int ncount,CRowInt &lsizes,CRowInt &ltypes,CRowInt &lconnfirst,CRowInt &lconnlast,int &lastproc);
static void AddActivationLayer(const int functype,CRowInt &lsizes,CRowInt &ltypes,CRowInt &lconnfirst,CRowInt &lconnlast,int &lastproc);
static void AddZeroLayer(CRowInt &lsizes,CRowInt &ltypes,CRowInt &lconnfirst,CRowInt &lconnlast,int &lastproc);
static void HLAddInputLayer(CMultilayerPerceptron &network,int &connidx,int &neuroidx,int &structinfoidx,int nin);
static void HLAddOutputLayer(CMultilayerPerceptron &network,int &connidx,int &neuroidx,int &structinfoidx,int &weightsidx,const int k,const int nprev,const int nout,const bool iscls,const bool islinearout);
static void HLAddHiddenLayer(CMultilayerPerceptron &network,int &connidx,int &neuroidx,int &structinfoidx,int &weightsidx,const int k,const int nprev,const int ncur);
static void FillHighLevelInformation(CMultilayerPerceptron &network,const int nin,const int nhid1,const int nhid2,const int nout,const bool iscls,const bool islinearout);
static void MLPCreate(const int nin,const int nout,CRowInt &lsizes,CRowInt &ltypes,CRowInt &lconnfirst,CRowInt &lconnlast,const int layerscount,const bool isclsnet,CMultilayerPerceptron &network);
static void MLPHessianBatchInternal(CMultilayerPerceptron &network,CMatrixDouble &xy,const int ssize,const bool naturalerr,double &e,CRowDouble &grad,CMatrixDouble &h);
static void MLPInternalCalculateGradient(CMultilayerPerceptron &network,CRowDouble &neurons,CRowDouble &weights,CRowDouble &derror,CRowDouble &grad,const bool naturalerrorfunc);
static void MLPChunkedGradient(CMultilayerPerceptron &network,CMatrixDouble &xy,const int cstart,const int csize,CRowDouble &batch4buf,CRowDouble &hpcbuf,double &e,const bool naturalerrorfunc);
static void MLPChunkedProcess(CMultilayerPerceptron &network,CMatrixDouble &xy,int cstart,int csize,CRowDouble &batch4buf,CRowDouble &hpcbuf);
static double SafeCrossEntropy(const double t,const double z);
static void RandomizeBackwardPass(CMultilayerPerceptron &network,int neuronidx,double v);
};
//+------------------------------------------------------------------+
//| Initialize constants |
//+------------------------------------------------------------------+
const int CMLPBase::m_mlpvnum=7;
const int CMLPBase::m_mlpfirstversion=0;
const int CMLPBase::m_nfieldwidth=4;
const int CMLPBase::m_hlconm_nfieldwidth=5;
const int CMLPBase::m_hlm_nfieldwidth=4;
const int CMLPBase::m_gradbasecasecost=50000;
const int CMLPBase::m_microbatchsize=64;
//+------------------------------------------------------------------+
//| This function returns number of weights updates which is required|
//| for gradient calculation problem to be splitted. |
//+------------------------------------------------------------------+
int CMLPBase::MLPGradSplitCost(void)
{
return(m_gradbasecasecost);
}
//+------------------------------------------------------------------+
//| This function returns number of elements in subset of dataset |
//| which is required for gradient calculation problem to be splitted|
//+------------------------------------------------------------------+
int CMLPBase::MLPGradSplitSize(void)
{
return(m_microbatchsize);
}
//+------------------------------------------------------------------+
//| Creates neural network with NIn inputs, NOut outputs, |
//| without hidden layers, with linear output layer. Network weights |
//| are filled with small random values. |
//+------------------------------------------------------------------+
void CMLPBase::MLPCreate0(const int nin,const int nout,
CMultilayerPerceptron &network)
{
//--- create variables
int layerscount=0;
int lastproc=0;
//--- creating arrays
CRowInt lsizes;
CRowInt ltypes;
CRowInt lconnfirst;
CRowInt lconnlast;
//--- initialization
layerscount=4;
//--- Allocate arrays
lsizes.Resize(layerscount);
ltypes.Resize(layerscount);
lconnfirst.Resize(layerscount);
lconnlast.Resize(layerscount);
//--- Layers
AddInputLayer(nin,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddBiasedSummatorLayer(nout,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddActivationLayer(-5,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- Create
MLPCreate(nin,nout,lsizes,ltypes,lconnfirst,lconnlast,layerscount,false,network);
//--- function call
FillHighLevelInformation(network,nin,0,0,nout,false,true);
}
//+------------------------------------------------------------------+
//| Same as MLPCreate0, but with one hidden layer (NHid neurons) with|
//| non-linear activation function. Output layer is linear. |
//+------------------------------------------------------------------+
void CMLPBase::MLPCreate1(const int nin,const int nhid,const int nout,
CMultilayerPerceptron &network)
{
//--- create variables
int layerscount=0;
int lastproc=0;
//--- creating arrays
CRowInt lsizes;
CRowInt ltypes;
CRowInt lconnfirst;
CRowInt lconnlast;
//--- create variables
layerscount=7;
//--- Allocate arrays
lsizes.Resize(layerscount);
ltypes.Resize(layerscount);
lconnfirst.Resize(layerscount);
lconnlast.Resize(layerscount);
//--- Layers
AddInputLayer(nin,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddBiasedSummatorLayer(nhid,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddActivationLayer(1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddBiasedSummatorLayer(nout,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddActivationLayer(-5,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- Create
MLPCreate(nin,nout,lsizes,ltypes,lconnfirst,lconnlast,layerscount,false,network);
//--- function call
FillHighLevelInformation(network,nin,nhid,0,nout,false,true);
}
//+------------------------------------------------------------------+
//| Same as MLPCreate0,but with two hidden layers (NHid1 and NHid2 |
//| neurons) with non-linear activation function. Output layer is |
//| linear. |
//| $ALL |
//+------------------------------------------------------------------+
void CMLPBase::MLPCreate2(const int nin,const int nhid1,const int nhid2,
const int nout,CMultilayerPerceptron &network)
{
//--- create variables
int layerscount=0;
int lastproc=0;
//--- creating arrays
CRowInt lsizes;
CRowInt ltypes;
CRowInt lconnfirst;
CRowInt lconnlast;
//--- initialization
layerscount=10;
//--- Allocate arrays
lsizes.Resize(layerscount);
ltypes.Resize(layerscount);
lconnfirst.Resize(layerscount);
lconnlast.Resize(layerscount);
//--- Layers
AddInputLayer(nin,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddBiasedSummatorLayer(nhid1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddActivationLayer(1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddBiasedSummatorLayer(nhid2,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddActivationLayer(1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddBiasedSummatorLayer(nout,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddActivationLayer(-5,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- Create
MLPCreate(nin,nout,lsizes,ltypes,lconnfirst,lconnlast,layerscount,false,network);
//--- function call
FillHighLevelInformation(network,nin,nhid1,nhid2,nout,false,true);
}
//+------------------------------------------------------------------+
//| Creates neural network with NIn inputs, NOut outputs, without |
//| hidden layers with non-linear output layer. Network weights are |
//| filled with small random values. |
//| Activation function of the output layer takes values: |
//| (B, +INF), if D>=0 |
//| or |
//| (-INF, B), if D<0. |
//+------------------------------------------------------------------+
void CMLPBase::MLPCreateB0(const int nin,const int nout,const double b,
double d,CMultilayerPerceptron &network)
{
//--- create variables
int layerscount=0;
int lastproc=0;
//--- creating arrays
CRowInt lsizes;
CRowInt ltypes;
CRowInt lconnfirst;
CRowInt lconnlast;
//--- initialization
layerscount=4;
//--- check
if(d>=0.0)
d=1;
else
d=-1;
//--- Allocate arrays
lsizes.Resize(layerscount);
ltypes.Resize(layerscount);
lconnfirst.Resize(layerscount);
lconnlast.Resize(layerscount);
//--- Layers
AddInputLayer(nin,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddBiasedSummatorLayer(nout,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddActivationLayer(3,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- Create
MLPCreate(nin,nout,lsizes,ltypes,lconnfirst,lconnlast,layerscount,false,network);
//--- function call
FillHighLevelInformation(network,nin,0,0,nout,false,false);
//--- Turn on ouputs shift/scaling.
for(int i=nin; i<nin+nout; i++)
{
network.m_columnmeans.Set(i,b);
network.m_columnsigmas.Set(i,d);
}
}
//+------------------------------------------------------------------+
//| Same as MLPCreateB0 but with non-linear hidden layer. |
//+------------------------------------------------------------------+
void CMLPBase::MLPCreateB1(const int nin,const int nhid,const int nout,
const double b,double d,CMultilayerPerceptron &network)
{
//--- create variables
int layerscount=0;
int lastproc=0;
//--- creating arrays
CRowInt lsizes;
CRowInt ltypes;
CRowInt lconnfirst;
CRowInt lconnlast;
layerscount=7;
//--- check
if(d>=0.0)
d=1;
else
d=-1;
//--- Allocate arrays
lsizes.Resize(layerscount);
ltypes.Resize(layerscount);
lconnfirst.Resize(layerscount);
lconnlast.Resize(layerscount);
//--- Layers
AddInputLayer(nin,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddBiasedSummatorLayer(nhid,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddActivationLayer(1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddBiasedSummatorLayer(nout,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddActivationLayer(3,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- Create
MLPCreate(nin,nout,lsizes,ltypes,lconnfirst,lconnlast,layerscount,false,network);
//--- function call
FillHighLevelInformation(network,nin,nhid,0,nout,false,false);
//--- Turn on ouputs shift/scaling.
for(int i=nin; i<nin+nout; i++)
{
network.m_columnmeans.Set(i,b);
network.m_columnsigmas.Set(i,d);
}
}
//+------------------------------------------------------------------+
//| Same as MLPCreateB0 but with two non-linear hidden layers. |
//+------------------------------------------------------------------+
void CMLPBase::MLPCreateB2(const int nin,const int nhid1,const int nhid2,
const int nout,const double b,double d,
CMultilayerPerceptron &network)
{
//--- create variables
int layerscount=0;
int lastproc=0;
//--- creating arrays
CRowInt lsizes;
CRowInt ltypes;
CRowInt lconnfirst;
CRowInt lconnlast;
//--- initialization
layerscount=10;
//--- check
if(d>=0.0)
d=1;
else
d=-1;
//--- Allocate arrays
lsizes.Resize(layerscount);
ltypes.Resize(layerscount);
lconnfirst.Resize(layerscount);
lconnlast.Resize(layerscount);
//--- Layers
AddInputLayer(nin,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddBiasedSummatorLayer(nhid1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddActivationLayer(1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddBiasedSummatorLayer(nhid2,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddActivationLayer(1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddBiasedSummatorLayer(nout,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddActivationLayer(3,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- Create
MLPCreate(nin,nout,lsizes,ltypes,lconnfirst,lconnlast,layerscount,false,network);
//--- function call
FillHighLevelInformation(network,nin,nhid1,nhid2,nout,false,false);
//--- Turn on ouputs shift/scaling.
for(int i=nin; i<nin+nout; i++)
{
network.m_columnmeans.Set(i,b);
network.m_columnsigmas.Set(i,d);
}
}
//+------------------------------------------------------------------+
//| Creates neural network with NIn inputs, NOut outputs, |
//| without hidden layers with non-linear output layer. Network |
//| weights are filled with small random values. Activation function |
//| of the output layer takes values [A,B]. |
//+------------------------------------------------------------------+
void CMLPBase::MLPCreateR0(const int nin,const int nout,const double a,
const double b,CMultilayerPerceptron &network)
{
//--- create variables
int layerscount=0;
int lastproc=0;
//--- creating arrays
CRowInt lsizes;
CRowInt ltypes;
CRowInt lconnfirst;
CRowInt lconnlast;
//--- initialization
layerscount=4;
//--- Allocate arrays
lsizes.Resize(layerscount);
ltypes.Resize(layerscount);
lconnfirst.Resize(layerscount);
lconnlast.Resize(layerscount);
//--- Layers
AddInputLayer(nin,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddBiasedSummatorLayer(nout,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddActivationLayer(1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- Create
MLPCreate(nin,nout,lsizes,ltypes,lconnfirst,lconnlast,layerscount,false,network);
//--- function call
FillHighLevelInformation(network,nin,0,0,nout,false,false);
//--- Turn on outputs shift/scaling.
for(int i=nin; i<nin+nout; i++)
{
network.m_columnmeans.Set(i,0.5*(a+b));
network.m_columnsigmas.Set(i,0.5*(a-b));
}
}
//+------------------------------------------------------------------+
//| Same as MLPCreateR0,but with non-linear hidden layer. |
//+------------------------------------------------------------------+
void CMLPBase::MLPCreateR1(const int nin,const int nhid,const int nout,
const double a,const double b,
CMultilayerPerceptron &network)
{
//--- create variables
int layerscount=0;
int lastproc=0;
//--- creating arrays
CRowInt lsizes;
CRowInt ltypes;
CRowInt lconnfirst;
CRowInt lconnlast;
//--- initialization
layerscount=7;
//--- Allocate arrays
lsizes.Resize(layerscount);
ltypes.Resize(layerscount);
lconnfirst.Resize(layerscount);
lconnlast.Resize(layerscount);
//--- Layers
AddInputLayer(nin,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddBiasedSummatorLayer(nhid,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddActivationLayer(1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddBiasedSummatorLayer(nout,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddActivationLayer(1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- Create
MLPCreate(nin,nout,lsizes,ltypes,lconnfirst,lconnlast,layerscount,false,network);
//--- function call
FillHighLevelInformation(network,nin,nhid,0,nout,false,false);
//--- Turn on outputs shift/scaling.
for(int i=nin; i<nin+nout; i++)
{
network.m_columnmeans.Set(i,0.5*(a+b));
network.m_columnsigmas.Set(i,0.5*(a-b));
}
}
//+------------------------------------------------------------------+
//| Same as MLPCreateR0,but with two non-linear hidden layers. |
//+------------------------------------------------------------------+
void CMLPBase::MLPCreateR2(const int nin,const int nhid1,const int nhid2,
const int nout,const double a,const double b,
CMultilayerPerceptron &network)
{
//--- create variables
int layerscount=0;
int lastproc=0;
//--- creating arrays
CRowInt lsizes;
CRowInt ltypes;
CRowInt lconnfirst;
CRowInt lconnlast;
//--- initialization
layerscount=10;
//--- Allocate arrays
lsizes.Resize(layerscount);
ltypes.Resize(layerscount);
lconnfirst.Resize(layerscount);
lconnlast.Resize(layerscount);
//--- Layers
AddInputLayer(nin,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddBiasedSummatorLayer(nhid1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddActivationLayer(1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddBiasedSummatorLayer(nhid2,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddActivationLayer(1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddBiasedSummatorLayer(nout,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddActivationLayer(1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- Create
MLPCreate(nin,nout,lsizes,ltypes,lconnfirst,lconnlast,layerscount,false,network);
//--- function call
FillHighLevelInformation(network,nin,nhid1,nhid2,nout,false,false);
//--- Turn on outputs shift/scaling.
for(int i=nin; i<nin+nout; i++)
{
network.m_columnmeans.Set(i,0.5*(a+b));
network.m_columnsigmas.Set(i,0.5*(a-b));
}
}
//+------------------------------------------------------------------+
//| Creates classifier network with NIn inputs and NOut possible |
//| classes. |
//| Network contains no hidden layers and linear output layer with |
//| SOFTMAX-normalization (so outputs sums up to 1.0 and converge to |
//| posterior probabilities). |
//+------------------------------------------------------------------+
void CMLPBase::MLPCreateC0(const int nin,const int nout,
CMultilayerPerceptron &network)
{
//--- create variables
int layerscount=0;
int lastproc=0;
//--- creating arrays
CRowInt lsizes;
CRowInt ltypes;
CRowInt lconnfirst;
CRowInt lconnlast;
//--- check
if(!CAp::Assert(nout>=2,__FUNCTION__+": NOut<2!"))
return;
//--- initialization
layerscount=4;
//--- Allocate arrays
lsizes.Resize(layerscount);
ltypes.Resize(layerscount);
lconnfirst.Resize(layerscount);
lconnlast.Resize(layerscount);
//--- Layers
AddInputLayer(nin,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddBiasedSummatorLayer(nout-1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddZeroLayer(lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- Create
MLPCreate(nin,nout,lsizes,ltypes,lconnfirst,lconnlast,layerscount,true,network);
//--- function call
FillHighLevelInformation(network,nin,0,0,nout,true,true);
}
//+------------------------------------------------------------------+
//| Same as MLPCreateC0,but with one non-linear hidden layer. |
//+------------------------------------------------------------------+
void CMLPBase::MLPCreateC1(const int nin,const int nhid,const int nout,
CMultilayerPerceptron &network)
{
//--- create variables
int layerscount=0;
int lastproc=0;
//--- creating arrays
CRowInt lsizes;
CRowInt ltypes;
CRowInt lconnfirst;
CRowInt lconnlast;
//--- check
if(!CAp::Assert(nout>=2,__FUNCTION__+": NOut<2!"))
return;
//--- initialization
layerscount=7;
//--- Allocate arrays
lsizes.Resize(layerscount);
ltypes.Resize(layerscount);
lconnfirst.Resize(layerscount);
lconnlast.Resize(layerscount);
//--- Layers
AddInputLayer(nin,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddBiasedSummatorLayer(nhid,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddActivationLayer(1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddBiasedSummatorLayer(nout-1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddZeroLayer(lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- Create
MLPCreate(nin,nout,lsizes,ltypes,lconnfirst,lconnlast,layerscount,true,network);
//--- function call
FillHighLevelInformation(network,nin,nhid,0,nout,true,true);
}
//+------------------------------------------------------------------+
//| Same as MLPCreateC0, but with two non-linear hidden layers. |
//+------------------------------------------------------------------+
void CMLPBase::MLPCreateC2(const int nin,const int nhid1,const int nhid2,
const int nout,CMultilayerPerceptron &network)
{
//--- create variables
int layerscount=0;
int lastproc=0;
//--- creating arrays
CRowInt lsizes;
CRowInt ltypes;
CRowInt lconnfirst;
CRowInt lconnlast;
//--- check
if(!CAp::Assert(nout>=2,__FUNCTION__+": NOut<2!"))
return;
//--- initialization
layerscount=10;
//--- Allocate arrays
lsizes.Resize(layerscount);
ltypes.Resize(layerscount);
lconnfirst.Resize(layerscount);
lconnlast.Resize(layerscount);
//--- Layers
AddInputLayer(nin,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddBiasedSummatorLayer(nhid1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddActivationLayer(1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddBiasedSummatorLayer(nhid2,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddActivationLayer(1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddBiasedSummatorLayer(nout-1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
AddZeroLayer(lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- Create
MLPCreate(nin,nout,lsizes,ltypes,lconnfirst,lconnlast,layerscount,true,network);
//--- function call
FillHighLevelInformation(network,nin,nhid1,nhid2,nout,true,true);
}
//+------------------------------------------------------------------+
//| Copying of neural network |
//| INPUT PARAMETERS: |
//| Network1 - original |
//| OUTPUT PARAMETERS: |
//| Network2 - copy |
//+------------------------------------------------------------------+
void CMLPBase::MLPCopy(const CMultilayerPerceptron &network1,
CMultilayerPerceptron &network2)
{
//--- copy
network2=network1;
network2.m_grad.m_F=0;
network2.m_grad.m_G.Fill(0);
}
//+------------------------------------------------------------------+
//| This function compares architectures of neural networks. Only |
//| geometries are compared, weights and other parameters are |
//| not tested. |
//+------------------------------------------------------------------+
bool CMLPBase::MLPSameArchitecture(CMultilayerPerceptron &network1,
CMultilayerPerceptron &network2)
{
//--- check
if(!CAp::Assert(network1.m_structinfo.Size()>0 && network1.m_structinfo.Size()>=network1.m_structinfo[0],__FUNCTION__": Network1 is uninitialized"))
return(false);
if(!CAp::Assert(network2.m_structinfo.Size()>0 && network2.m_structinfo.Size()>=network2.m_structinfo[0],__FUNCTION__": Network2 is uninitialized"))
return(false);
if(network1.m_structinfo[0]!=network2.m_structinfo[0])
return(false);
int ninfo=network1.m_structinfo[0];
for(int i=0; i<ninfo; i++)
if(network1.m_structinfo[i]!=network2.m_structinfo[i])
return(false);
return(true);
}
//+------------------------------------------------------------------+
//| This function copies tunable parameters (weights/means/sigmas) |
//| from one network to another with same architecture. It performs |
//| some rudimentary checks that architectures are same, and throws |
//| exception if check fails. |
//| It is intended for fast copying of states between two network |
//| which are known to have same geometry. |
//| INPUT PARAMETERS: |
//| Network1 - source, must be correctly initialized |
//| Network2 - target, must have same architecture |
//| OUTPUT PARAMETERS: |
//| Network2 - network state is copied from source to target |
//+------------------------------------------------------------------+
void CMLPBase::MLPCopyTunableParameters(CMultilayerPerceptron &network1,
CMultilayerPerceptron &network2)
{
//--- create variables
int nin=0;
int nout=0;
int wcount=0;
//--- check
if(!CAp::Assert(network1.m_structinfo.Size()>0 && network1.m_structinfo.Size()>=network1.m_structinfo[0],__FUNCTION__": Network1 is uninitialized"))
return;
if(!CAp::Assert(network2.m_structinfo.Size()>0 && network2.m_structinfo.Size()>=network2.m_structinfo[0],__FUNCTION__": Network2 is uninitialized"))
return;
if(!CAp::Assert(network1.m_structinfo[0]==network2.m_structinfo[0],__FUNCTION__": Network1 geometry differs from that of Network2"))
return;
int ninfo=network1.m_structinfo[0];
for(int i=0; i<ninfo; i++)
if(!CAp::Assert(network1.m_structinfo[i]==network2.m_structinfo[i],__FUNCTION__": Network1 geometry differs from that of Network2"))
return;
MLPProperties(network1,nin,nout,wcount);
network2.m_weights=network1.m_weights;
if(MLPIsSoftMax(network1))
{
network2.m_columnmeans=network1.m_columnmeans;
network2.m_columnsigmas=network1.m_columnsigmas;
}
else
{
network2.m_columnmeans=network1.m_columnmeans;
network2.m_columnsigmas=network1.m_columnsigmas;
}
}
//+------------------------------------------------------------------+
//| This function exports tunable parameters (weights/means/sigmas) |
//| from network to contiguous array. Nothing is guaranteed about |
//| array format, the only thing you can count for is that |
//| MLPImportTunableParameters() will be able to parse it. |
//| It is intended for fast copying of states between network and |
//| backup array |
//| INPUT PARAMETERS: |
//| Network - source, must be correctly initialized |
//| P - array to use. If its size is enough to store |
//| data, it is reused. |
//| OUTPUT PARAMETERS: |
//| P - array which stores network parameters, resized |
//| if needed |
//| PCount - number of parameters stored in array. |
//+------------------------------------------------------------------+
void CMLPBase::MLPExportTunableParameters(CMultilayerPerceptron &network,
CRowDouble &p,int &pcount)
{
//--- create variables
int k=0;
int nin=0;
int nout=0;
int wcount=0;
pcount=0;
//--- check
if(!CAp::Assert(network.m_structinfo.Size()>0 && network.m_structinfo.Size()>=network.m_structinfo[0],__FUNCTION__": Network is uninitialized"))
return;
MLPProperties(network,nin,nout,wcount);
if(MLPIsSoftMax(network))
{
pcount=wcount+2*nin;
CApServ::RVectorSetLengthAtLeast(p,pcount);
for(int i=0; i<wcount; i++)
p.Set(i,network.m_weights[i]);
k=wcount;
for(int i=0; i<nin; i++)
{
p.Set(k,network.m_columnmeans[i]);
p.Set(k+1,network.m_columnsigmas[i]);
k+=2;
}
}
else
{
pcount=wcount+2*(nin+nout);
CApServ::RVectorSetLengthAtLeast(p,pcount);
for(int i=0; i<wcount; i++)
p.Set(i,network.m_weights[i]);
k=wcount;
for(int i=0; i<nin+nout; i++)
{
p.Set(k,network.m_columnmeans[i]);
p.Set(k+1,network.m_columnsigmas[i]);
k+=2;
}
}
}
//+------------------------------------------------------------------+
//| This function imports tunable parameters (weights/means/sigmas) |
//| which were exported by MLPExportTunableParameters(). |
//| It is intended for fast copying of states between network and |
//| backup array |
//| INPUT PARAMETERS: |
//| Network - target: |
//| * must be correctly initialized |
//| * must have same geometry as network used |
//| to export params |
//| P - array with parameters |
//+------------------------------------------------------------------+
void CMLPBase::MLPImportTunableParameters(CMultilayerPerceptron &network,
CRowDouble &p)
{
//--- create variables
int i=0;
int k=0;
int nin=0;
int nout=0;
int wcount=0;
//--- check
if(!CAp::Assert(network.m_structinfo.Size()>0 && network.m_structinfo.Size()>=network.m_structinfo[0],__FUNCTION__": Network is uninitialized"))
return;
MLPProperties(network,nin,nout,wcount);
if(MLPIsSoftMax(network))
{
for(i=0; i<wcount; i++)
network.m_weights.Set(i,p[i]);
k=wcount;
for(i=0; i<nin; i++)
{
network.m_columnmeans.Set(i,p[k]);
network.m_columnsigmas.Set(i,p[k+1]);
k+=2;
}
}
else
{
for(i=0; i<wcount; i++)
network.m_weights.Set(i,p[i]);
k=wcount;
for(i=0; i<nin+nout; i++)
{
network.m_columnmeans.Set(i,p[k]);
network.m_columnsigmas.Set(i,p[k+1]);
k+=2;
}
}
}
//+------------------------------------------------------------------+
//| Serialization of MultiLayerPerceptron strucure |
//| INPUT PARAMETERS: |
//| Network - original |
//| OUTPUT PARAMETERS: |
//| RA - array of real numbers which stores network, |
//| array[0..RLen-1] |
//| RLen - RA lenght |
//+------------------------------------------------------------------+
void CMLPBase::MLPSerializeOld(CMultilayerPerceptron &network,
double &ra[],int &rlen)
{
CRowDouble RA;
MLPSerializeOld(network,ra,rlen);
RA.ToArray(ra);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CMLPBase::MLPSerializeOld(CMultilayerPerceptron &network,
CRowDouble &ra,int &rlen)
{
//--- create variables
int ssize=network.m_structinfo[0];
int nin=network.m_structinfo[1];
int nout=network.m_structinfo[2];
int ntotal=network.m_structinfo[3];
int wcount=network.m_structinfo[4];
int sigmalen=0;
int offs=0;
int i=0;
int i_=0;
int i1_=0;
//--- check
if(MLPIsSoftMax(network))
sigmalen=nin;
else
sigmalen=nin+nout;
//--- RA format:
//--- LEN DESRC.
//--- 1 RLen
//--- 1 version (MLPVNum)
//--- 1 StructInfo size
//--- SSize StructInfo
//--- WCount Weights
//--- SigmaLen ColumnMeans
//--- SigmaLen ColumnSigmas
rlen=3+ssize+wcount+2*sigmalen;
//--- allocation
ra.Resize(rlen);
//--- change values
ra.Set(0,rlen);
ra.Set(1,m_mlpvnum);
ra.Set(2,ssize);
//--- calculation
offs=3;
for(i=0; i<ssize; i++)
ra.Set(offs+i,network.m_structinfo[i]);
//--- calculation
offs+=ssize;
for(i=0; i<wcount; i++)
ra.Set(offs+i,network.m_weights[i]);
//--- calculation
offs+=wcount;
for(i=0; i<sigmalen; i++)
ra.Set(offs+i,network.m_columnmeans[i]);
//--- calculation
offs+=sigmalen;
for(i=0; i<sigmalen; i++)
ra.Set(offs+i,network.m_columnsigmas[i]);
}
//+------------------------------------------------------------------+
//| Unserialization of MultiLayerPerceptron strucure |
//| INPUT PARAMETERS: |
//| RA - real array which stores network |
//| OUTPUT PARAMETERS: |
//| Network - restored network |
//+------------------------------------------------------------------+
void CMLPBase::MLPUnserializeOld(double &ra[],CMultilayerPerceptron &network)
{
CRowDouble RA=ra;
MLPUnserializeOld(RA,network);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CMLPBase::MLPUnserializeOld(CRowDouble &ra,CMultilayerPerceptron &network)
{
//--- create variables
int i=0;
int ssize=0;
int ntotal=0;
int nin=0;
int nout=0;
int wcount=0;
int sigmalen=0;
int offs=0;
int i_=0;
int i1_=0;
//--- check
if(!CAp::Assert((int)MathRound(ra[1])==m_mlpvnum,__FUNCTION__+": incorrect array!"))
return;
//--- Unload StructInfo from IA
offs=3;
ssize=(int)MathRound(ra[2]);
//--- allocation
network.m_structinfo.Resize(ssize);
for(i=0; i<ssize; i++)
network.m_structinfo.Set(i,(int)MathRound(ra[offs+i]));
offs+=ssize;
//--- Unload info from StructInfo
ssize=network.m_structinfo[0];
nin=network.m_structinfo[1];
nout=network.m_structinfo[2];
ntotal=network.m_structinfo[3];
wcount=network.m_structinfo[4];
//--- check
if(network.m_structinfo[6]==0)
sigmalen=nin+nout;
else
sigmalen=nin;
//--- Allocate space for other fields
network.m_weights.Resize(wcount);
network.m_columnmeans.Resize(sigmalen);
network.m_columnsigmas.Resize(sigmalen);
network.m_neurons.Resize(ntotal);
network.m_nwbuf.Resize(MathMax(wcount,2*nout));
network.m_dfdnet.Resize(ntotal);
network.m_x.Resize(nin);
network.m_y.Resize(nout);
network.m_derror.Resize(ntotal);
//--- Copy parameters from RA
for(i=0; i<wcount; i++)
network.m_weights.Set(i_,ra[i+offs]);
//--- calculation
offs+=wcount;
for(i=0; i_<sigmalen; i++)
network.m_columnmeans.Set(i,ra[i+offs]);
//--- calculation
offs+=sigmalen;
for(i=0; i<sigmalen; i++)
network.m_columnsigmas.Set(i,ra[i+offs]);
}
//+------------------------------------------------------------------+
//| Randomization of neural network weights |
//+------------------------------------------------------------------+
void CMLPBase::MLPRandomize(CMultilayerPerceptron &network)
{
//--- create variables
int nin=0;
int nout=0;
int wcount=0;
int ntotal=0;
int istart=0;
CHighQualityRandState r;
int entrysize=0;
int entryoffs=0;
int neuronidx=0;
int neurontype=0;
double vmean=0;
double vvar=0;
int i=0;
int n1=0;
int n2=0;
double desiredsigma=0;
int montecarlocnt=0;
double ef=0;
double ef2=0;
double v=0;
double wscale=0;
CHighQualityRand::HQRndRandomize(r);
MLPProperties(network,nin,nout,wcount);
ntotal=network.m_structinfo[3];
istart=network.m_structinfo[5];
desiredsigma=0.5;
montecarlocnt=20;
//--- Stage 1:
//--- * Network.Weights is filled by standard deviation of weights
//--- * default values: sigma=1
network.m_weights.Fill(1.0);
//--- Stage 2:
//--- * assume that input neurons have zero mean and unit standard deviation
//--- * assume that constant neurons have zero standard deviation
//--- * perform forward pass along neurons
//--- * for each non-input non-constant neuron:
//--- * calculate mean and standard deviation of neuron's output
//--- assuming that we know means/deviations of neurons which feed it
//--- and assuming that weights has unit variance and zero mean.
//--- * for each nonlinear neuron additionally we perform backward pass:
//--- * scale variances of weights which feed it in such way that neuron's
//--- input has unit standard deviation
//--- NOTE: this algorithm assumes that each connection feeds at most one
//--- non-linear neuron. This assumption can be incorrect in upcoming
//--- architectures with strong neurons. However, algorithm should
//--- work smoothly even in this case.
//--- During this stage we use Network.RndBuf, which is grouped into NTotal
//--- entries, each of them having following format:
//--- Buf[Offset+0] mean value of neuron's output
//--- Buf[Offset+1] standard deviation of neuron's output
entrysize=2;
CApServ::RVectorSetLengthAtLeast(network.m_rndbuf,entrysize*ntotal);
for(neuronidx=0; neuronidx<ntotal; neuronidx++)
{
neurontype=network.m_structinfo[istart+neuronidx*m_nfieldwidth];
entryoffs=entrysize*neuronidx;
switch(neurontype)
{
case -2:
//--- Input neuron: zero mean, unit variance.
network.m_rndbuf.Set(entryoffs,0.0);
network.m_rndbuf.Set(entryoffs+1,1.0);
break;
case -3:
//--- "-1" neuron: mean=-1, zero variance.
network.m_rndbuf.Set(entryoffs,-1.0);
network.m_rndbuf.Set(entryoffs+1,0.0);
break;
case -4:
//--- "0" neuron: mean=0, zero variance.
network.m_rndbuf.Set(entryoffs,0.0);
network.m_rndbuf.Set(entryoffs+1,0.0);
break;
case 0:
//--- Adaptive summator neuron:
//--- * calculate its mean and variance.
//--- * we assume that weights of this neuron have unit variance and zero mean.
//--- * thus, neuron's output is always have zero mean
//--- * as for variance, it is a bit more interesting:
//--- * let n[i] is i-th input neuron
//--- * let w[i] is i-th weight
//--- * we assume that n[i] and w[i] are independently distributed
//--- * Var(n0*w0+n1*w1+...) = Var(n0*w0)+Var(n1*w1)+...
//--- * Var(X*Y) = mean(X)^2*Var(Y) + mean(Y)^2*Var(X) + Var(X)*Var(Y)
//--- * mean(w[i])=0, var(w[i])=1
//--- * Var(n[i]*w[i]) = mean(n[i])^2 + Var(n[i])
n1=network.m_structinfo[istart+neuronidx*m_nfieldwidth+2];
n2=n1+network.m_structinfo[istart+neuronidx*m_nfieldwidth+1];
vmean=0.0;
vvar=0.0;
for(i=n1; i<n2; i++)
vvar+=CMath::Sqr(network.m_rndbuf[entrysize*i])+CMath::Sqr(network.m_rndbuf[entrysize*i+1]);
network.m_rndbuf.Set(entryoffs,vmean);
network.m_rndbuf.Set(entryoffs+1,MathSqrt(vvar));
break;
case -5:
//--- Linear activation function
i=network.m_structinfo[istart+neuronidx*m_nfieldwidth+2];
vmean=network.m_rndbuf[entrysize*i];
vvar=CMath::Sqr(network.m_rndbuf[entrysize*i+1]);
if(vvar>0.0)
wscale=desiredsigma/MathSqrt(vvar);
else
wscale=1.0;
RandomizeBackwardPass(network,i,wscale);
network.m_rndbuf.Set(entryoffs,vmean*wscale);
network.m_rndbuf.Set(entryoffs+1,desiredsigma);
break;
default:
if(neurontype>0)
{
//--- Nonlinear activation function:
//--- * scale its inputs
//--- * estimate mean/sigma of its output using Monte-Carlo method
//--- (we simulate different inputs with unit deviation and
//--- sample activation function output on such inputs)
i=network.m_structinfo[istart+neuronidx*m_nfieldwidth+2];
vmean=network.m_rndbuf[entrysize*i];
vvar=CMath::Sqr(network.m_rndbuf[entrysize*i+1]);
if(vvar>0.0)
wscale=desiredsigma/MathSqrt(vvar);
else
wscale=1.0;
RandomizeBackwardPass(network,i,wscale);
ef=0.0;
ef2=0.0;
vmean=vmean*wscale;
for(i=0; i<montecarlocnt; i++)
{
v=vmean+desiredsigma*CHighQualityRand::HQRndNormal(r);
ef+=v;
ef2+=v*v;
}
ef=ef/montecarlocnt;
ef2=ef2/montecarlocnt;
network.m_rndbuf.Set(entryoffs,ef);
network.m_rndbuf.Set(entryoffs+1,MathMax(ef2-ef*ef,0.0));
}
else
{
CAp::Assert(false,__FUNCTION__": unexpected neuron type");
return;
}
break;
}
}
//--- Stage 3: generate weights.
for(i=0; i<wcount; i++)
network.m_weights.Mul(i,CHighQualityRand::HQRndNormal(r));
}
//+------------------------------------------------------------------+
//| Randomization of neural network weights and standartisator |
//+------------------------------------------------------------------+
void CMLPBase::MLPRandomizeFull(CMultilayerPerceptron &network)
{
//--- create variables
int i=0;
int nin=0;
int nout=0;
int wcount=0;
int ntotal=0;
int istart=0;
int offs=0;
int ntype=0;
//--- function call
MLPProperties(network,nin,nout,wcount);
//--- initialization
ntotal=network.m_structinfo[3];
istart=network.m_structinfo[5];
//--- Process network
MLPRandomize(network);
for(i=0; i<nin; i++)
{
network.m_columnmeans.Set(i,CMath::RandomReal()-0.5);
network.m_columnsigmas.Set(i,CMath::RandomReal()+0.5);
}
//--- check
if(!MLPIsSoftMax(network))
{
for(i=0; i<nout; i++)
{
offs=istart+(ntotal-nout+i)*m_nfieldwidth;
ntype=network.m_structinfo[offs];
//--- check
if(ntype==0)
{
//--- Shifts are changed only for linear outputs neurons
network.m_columnmeans.Set(nin+i,2*CMath::RandomReal()-1);
}
//--- check
if(ntype==0 || ntype==3)
{
//--- Scales are changed only for linear or bounded outputs neurons.
//--- Note that scale randomization preserves sign.
network.m_columnsigmas.Set(nin+i,MathSign(network.m_columnsigmas[nin+i])*(1.5*CMath::RandomReal()+0.5));
}
}
}
}
//+------------------------------------------------------------------+
//| Internal subroutine. |
//+------------------------------------------------------------------+
void CMLPBase::MLPInitPreprocessor(CMultilayerPerceptron &network,
CMatrixDouble &xy,const int ssize)
{
//--- create variables
int i=0;
int j=0;
int nin=0;
int nout=0;
int wcount=0;
int ntotal=0;
int istart=0;
int offs=0;
int ntype=0;
int jmax=0;
double s=0;
//--- creating arrays
vector<double> means;
vector<double> sigmas;
//--- function call
MLPProperties(network,nin,nout,wcount);
//--- initialization
ntotal=network.m_structinfo[3];
istart=network.m_structinfo[5];
//--- allocation
if(ssize>0)
{
matrix<double> XY=xy.ToMatrix();
XY.Resize(ssize,nin+nout);
means=XY.Mean(0);
sigmas=XY.Std(0);
}
else
{
if(MLPIsSoftMax(network))
jmax=nin+1;
else
jmax=nin+nout;
means=vector<double>::Zeros(jmax);
sigmas=vector<double>::Ones(jmax);
}
//--- Inputs
for(i=0; i<nin; i++)
{
network.m_columnmeans.Set(i,means[i]);
network.m_columnsigmas.Set(i,(sigmas[i]!=0?sigmas[i]:1));
}
//--- Outputs
if(!MLPIsSoftMax(network))
{
for(i=0; i<nout; i++)
{
offs=istart+(ntotal-nout+i)*m_nfieldwidth;
ntype=network.m_structinfo[offs];
//--- Linear outputs
if(ntype==0)
{
network.m_columnmeans.Set(nin+i,means[nin+i]);
network.m_columnsigmas.Set(nin+i,(sigmas[nin+i]!=0?sigmas[nin+i]:1));
}
//--- Bounded outputs (half-interval)
if(ntype==3)
{
s=means[nin+i]-network.m_columnmeans[nin+i];
//--- check
if(s==0.0)
s=MathSign(network.m_columnsigmas[nin+i]);
//--- check
if(s==0.0)
s=1.0;
//--- change value
network.m_columnsigmas.Set(nin+i,MathSign(network.m_columnsigmas[nin+i])*MathAbs(s));
//--- check
if(network.m_columnsigmas[nin+i]==0.0)
network.m_columnsigmas.Set(nin+i,1);
}
}
}
}
//+------------------------------------------------------------------+
//| Internal subroutine. |
//| Initialization for preprocessor based on a sample. |
//| INPUT |
//| Network - initialized neural network; |
//| XY - sample, given by sparse matrix; |
//| SSize - sample size. |
//| OUTPUT |
//| Network - neural network with initialised preprocessor. |
//+------------------------------------------------------------------+
void CMLPBase::MLPInitPreprocessorSparse(CMultilayerPerceptron &network,
CSparseMatrix &xy,int ssize)
{
//--- create variables
int jmax=0;
int nin=0;
int nout=0;
int wcount=0;
int ntotal=0;
int istart=0;
int offs=0;
int ntype=0;
double s=0;
int i=0;
int j=0;
//--- create arrays
vector<double> means;
vector<double> sigmas;
CMatrixDouble dense_xy;
MLPProperties(network,nin,nout,wcount);
ntotal=network.m_structinfo[3];
istart=network.m_structinfo[5];
//--- Means/Sigmas
if(MLPIsSoftMax(network))
jmax=nin+1;
else
jmax=nin+nout;
dense_xy=matrix<double>::Zeros(ssize,jmax);
for(i=0; i<ssize; i++)
{
CSparse::SparseGetRow(xy,i,network.m_xyrow);
dense_xy.Row(i,network.m_xyrow);
}
if(ssize>0)
{
means=dense_xy.Mean(0);
sigmas=dense_xy.Std(0);
}
else
{
means=vector<double>::Zeros(jmax);
sigmas=vector<double>::Ones(jmax);
}
//--- Inputs
for(i=0; i<nin; i++)
{
network.m_columnmeans.Set(i,means[i]);
network.m_columnsigmas.Set(i,(sigmas[i]!=0?sigmas[i]:1));
}
//--- Outputs
if(!MLPIsSoftMax(network))
{
for(i=0; i<nout; i++)
{
offs=istart+(ntotal-nout+i)*m_nfieldwidth;
ntype=network.m_structinfo[offs];
//--- Linear outputs
if(ntype==0)
{
network.m_columnmeans.Set(nin+i,means[nin+i]);
network.m_columnsigmas.Set(nin+i,(sigmas[nin+i]!=0.0?sigmas[nin+i]:1.0));
}
//--- Bounded outputs (half-interval)
if(ntype==3)
{
s=means[nin+i]-network.m_columnmeans[nin+i];
if(s==0.0)
s=MathSign(network.m_columnsigmas[nin+i]);
if(s==0.0)
s=1.0;
s=MathSign(network.m_columnsigmas[nin+i])*MathAbs(s);
network.m_columnsigmas.Set(nin+i,(s!=0.0?s:1.0));
}
}
}
}
//+------------------------------------------------------------------+
//| Internal subroutine. |
//| Initialization for preprocessor based on a subsample. |
//| INPUT PARAMETERS: |
//| Network - network initialized with one of the network |
//| creation funcs |
//| XY - original dataset; one sample = one row; |
//| first NIn columns contain inputs, |
//| next NOut columns - desired outputs. |
//| SetSize - real size of XY, SetSize>=0; |
//| Idx - subset of SubsetSize elements, array[SubsetSize]|
//| * Idx[I] stores row index in the original dataset which is |
//| given by XY. Gradient is calculated with respect to rows |
//| whose indexes are stored in Idx[]. |
//| * Idx[] must store correct indexes; this function throws |
//| an exception in case incorrect index (less than 0 or |
//| larger than rows(XY)) is given |
//| * Idx[] may store indexes in any order and even with |
//| repetitions. |
//| SubsetSize - number of elements in Idx[] array. |
//| OUTPUT: |
//| Network - neural network with initialised preprocessor. |
//| NOTE: when SubsetSize<0 is used full dataset by call |
//| MLPInitPreprocessor function. |
//+------------------------------------------------------------------+
void CMLPBase::MLPInitPreprocessorSubset(CMultilayerPerceptron &network,
CMatrixDouble &xy,int setsize,
CRowInt &idx,int subsetsize)
{
//--- create variables
int jmax=0;
int nin=0;
int nout=0;
int wcount=0;
int ntotal=0;
int istart=0;
int offs=0;
int ntype=0;
double s=0;
int npoints=0;
int i=0;
int j=0;
//--- create arrays
vector<double> means;
vector<double> sigmas;
//--- check
if(!CAp::Assert(setsize>=0,__FUNCTION__": SetSize<0"))
return;
if(subsetsize<0)
{
MLPInitPreprocessor(network,xy,setsize);
return;
}
//--- check
if(!CAp::Assert(subsetsize<=idx.Size(),__FUNCTION__": SubsetSize>Length(Idx)"))
return;
npoints=setsize;
for(i=0; i<subsetsize; i++)
{
if(!CAp::Assert(idx[i]>=0,__FUNCTION__": incorrect index of XY row(Idx[I]<0)"))
return;
if(!CAp::Assert(idx[i]<npoints,__FUNCTION__": incorrect index of XY row(Idx[I]>=Rows(XY))"))
return;
}
//--- get Network info
MLPProperties(network,nin,nout,wcount);
ntotal=network.m_structinfo[3];
istart=network.m_structinfo[5];
//--- Means/Sigmas
if(MLPIsSoftMax(network))
jmax=nin;
else
jmax=nin+nout;
means=vector<double>::Zeros(jmax);
sigmas=means;
for(i=0; i<subsetsize; i++)
{
for(j=0; j<jmax; j++)
means[j]+=xy.Get(idx[i],j);
}
means=means/subsetsize;
for(i=0; i<subsetsize; i++)
{
for(j=0; j<jmax; j++)
sigmas[j]+=CMath::Sqr(xy.Get(idx[i],j)-means[j]);
}
sigmas=MathSqrt(sigmas/subsetsize);
//--- Inputs
for(i=0; i<nin; i++)
{
network.m_columnmeans.Set(i,means[i]);
network.m_columnsigmas.Set(i,(sigmas[i]!=0?sigmas[i]:1));
}
//--- Outputs
if(!MLPIsSoftMax(network))
{
for(i=0; i<nout; i++)
{
offs=istart+(ntotal-nout+i)*m_nfieldwidth;
ntype=network.m_structinfo[offs];
//--- Linear outputs
if(ntype==0)
{
network.m_columnmeans.Set(nin+i,means[nin+i]);
network.m_columnsigmas.Set(nin+i,(sigmas[nin+i]!=0.0?sigmas[nin+i]:1.0));
}
//--- Bounded outputs (half-interval)
if(ntype==3)
{
s=means[nin+i]-network.m_columnmeans[nin+i];
if(s==0.0)
s=MathSign(network.m_columnsigmas[nin+i]);
if(s==0.0)
s=1.0;
s=MathSign(network.m_columnsigmas[nin+i])*MathAbs(s);
network.m_columnsigmas.Set(nin+i,(s!=0.0?s:1));
}
}
}
}
//+------------------------------------------------------------------+
//| Internal subroutine. |
//| Initialization for preprocessor based on a subsample. |
//| INPUT PARAMETERS: |
//| Network - network initialized with one of the network |
//| creation funcs |
//| XY - original dataset, given by sparse matrix; |
//| one sample = one row; |
//| first NIn columns contain inputs, |
//| next NOut columns - desired outputs. |
//| SetSize - real size of XY, SetSize>=0; |
//| Idx - subset of SubsetSize elements, array[SubsetSize]: |
//| * Idx[I] stores row index in the original dataset which is |
//| given by XY. Gradient is calculated with respect to rows |
//| whose indexes are stored in Idx[]. |
//| * Idx[] must store correct indexes; this function throws |
//| an exception in case incorrect index (less than 0 or |
//| larger than rows(XY)) is given |
//| * Idx[] may store indexes in any order and even with |
//| repetitions. |
//| SubsetSize- number of elements in Idx[] array. |
//| OUTPUT: |
//| Network - neural network with initialised preprocessor. |
//| NOTE: when SubsetSize<0 is used full dataset by call |
//| MLPInitPreprocessorSparse function. |
//+------------------------------------------------------------------+
void CMLPBase::MLPInitPreprocessorSparseSubset(CMultilayerPerceptron &network,
CSparseMatrix &xy,int setsize,
CRowInt &idx,int subsetsize)
{
//--- create variables
int jmax=0;
int nin=0;
int nout=0;
int wcount=0;
int ntotal=0;
int istart=0;
int offs=0;
int ntype=0;
double s=0;
int npoints=0;
int i=0;
int j=0;
//--- create arrays
vector<double> means;
vector<double> sigmas;
//--- check
if(!CAp::Assert(setsize>=0,__FUNCTION__": SetSize<0"))
return;
if(subsetsize<0)
{
MLPInitPreprocessorSparse(network,xy,setsize);
return;
}
if(!CAp::Assert(subsetsize<=idx.Size(),__FUNCTION__": SubsetSize>Length(Idx)"))
return;
npoints=setsize;
for(i=0; i<subsetsize; i++)
{
if(!CAp::Assert(idx[i]>=0,__FUNCTION__": incorrect index of XY row(Idx[I]<0)"))
return;
if(!CAp::Assert(idx[i]<npoints,__FUNCTION__": incorrect index of XY row(Idx[I]>=Rows(XY))"))
return;
}
//--- get Network info
MLPProperties(network,nin,nout,wcount);
ntotal=network.m_structinfo[3];
istart=network.m_structinfo[5];
//
//--- Means/Sigmas
//
if(MLPIsSoftMax(network))
jmax=nin;
else
jmax=nin+nout;
means=vector<double>::Zeros(jmax);
sigmas=means;
for(i=0; i<subsetsize; i++)
{
CSparse::SparseGetRow(xy,idx[i],network.m_xyrow);
means+=network.m_xyrow.ToVector();
}
means=means/subsetsize;
for(i=0; i<=subsetsize-1; i++)
{
CSparse::SparseGetRow(xy,idx[i],network.m_xyrow);
sigmas+=MathPow(network.m_xyrow-means+0,2.0);
}
sigmas=MathSqrt(sigmas/subsetsize);
//--- Inputs
for(i=0; i<nin; i++)
{
network.m_columnmeans.Set(i,means[i]);
network.m_columnsigmas.Set(i,(sigmas[i]!=0?sigmas[i]:1));
}
//--- Outputs
if(!MLPIsSoftMax(network))
{
for(i=0; i<nout; i++)
{
offs=istart+(ntotal-nout+i)*m_nfieldwidth;
ntype=network.m_structinfo[offs];
//--- Linear outputs
if(ntype==0)
{
network.m_columnmeans.Set(nin+i,means[nin+i]);
network.m_columnsigmas.Set(nin+i,(sigmas[nin+i]!=0.0?sigmas[nin+i]:1.0));
}
//--- Bounded outputs (half-interval)
if(ntype==3)
{
s=means[nin+i]-network.m_columnmeans[nin+i];
if(s==0.0)
s=MathSign(network.m_columnsigmas[nin+i]);
if(s==0.0)
s=1.0;
s=MathSign(network.m_columnsigmas[nin+i])*MathAbs(s);
network.m_columnsigmas.Set(nin+i,(s!=0?s:1));
}
}
}
}
//+------------------------------------------------------------------+
//| Returns information about initialized network: number of inputs, |
//| outputs, weights. |
//+------------------------------------------------------------------+
void CMLPBase::MLPProperties(CMultilayerPerceptron &network,int &nin,
int &nout,int &wcount)
{
//--- change values
nin=network.m_structinfo[1];
nout=network.m_structinfo[2];
wcount=network.m_structinfo[4];
}
//+------------------------------------------------------------------+
//| Returns number of "internal", low-level neurons in the network |
//| (one which is stored in StructInfo). |
//+------------------------------------------------------------------+
int CMLPBase::MLPNTotal(CMultilayerPerceptron &network)
{
return(network.m_structinfo[3]);
}
//+------------------------------------------------------------------+
//| Returns number of inputs. |
//+------------------------------------------------------------------+
int CMLPBase::MLPGetInputsCount(CMultilayerPerceptron &network)
{
return(network.m_structinfo[1]);
}
//+------------------------------------------------------------------+
//| Returns number of outputs. |
//+------------------------------------------------------------------+
int CMLPBase::MLPGetOutputsCount(CMultilayerPerceptron &network)
{
return(network.m_structinfo[2]);
}
//+------------------------------------------------------------------+
//| Returns number of weights. |
//+------------------------------------------------------------------+
int CMLPBase::MLPGetWeightsCount(CMultilayerPerceptron &network)
{
return(network.m_structinfo[4]);
}
//+------------------------------------------------------------------+
//| Tells whether network is SOFTMAX-normalized (i.m_e. classifier) |
//| or not. |
//+------------------------------------------------------------------+
bool CMLPBase::MLPIsSoftMax(CMultilayerPerceptron &network)
{
//--- check
if(network.m_structinfo[6]==1)
return(true);
//--- return result
return(false);
}
//+------------------------------------------------------------------+
//| This function returns total number of layers (including input, |
//| hidden and output layers). |
//+------------------------------------------------------------------+
int CMLPBase::MLPGetLayersCount(CMultilayerPerceptron &network)
{
//--- return result
return(CAp::Len(network.m_hllayersizes));
}
//+------------------------------------------------------------------+
//| This function returns size of K-th layer. |
//| K=0 corresponds to input layer, K=CNT-1 corresponds to output |
//| layer. |
//| Size of the output layer is always equal to the number of |
//| outputs, although when we have softmax-normalized network, last |
//| neuron doesn't have any connections - it is just zero. |
//+------------------------------------------------------------------+
int CMLPBase::MLPGetLayerSize(CMultilayerPerceptron &network,
const int k)
{
//--- check
if(!CAp::Assert(k>=0 && k<CAp::Len(network.m_hllayersizes),__FUNCTION__+": incorrect layer index"))
return(-1);
//--- return result
return(network.m_hllayersizes[k]);
}
//+------------------------------------------------------------------+
//| This function returns offset/scaling coefficients for I-th input |
//| of the network. |
//| INPUT PARAMETERS: |
//| Network - network |
//| I - input index |
//| OUTPUT PARAMETERS: |
//| Mean - mean term |
//| Sigma - sigma term,guaranteed to be nonzero. |
//| I-th input is passed through linear transformation |
//| IN[i]=(IN[i]-Mean)/Sigma |
//| before feeding to the network |
//+------------------------------------------------------------------+
void CMLPBase::MLPGetInputScaling(CMultilayerPerceptron &network,
const int i,double &mean,
double &sigma)
{
//--- check
if(!CAp::Assert(i>=0 && i<network.m_hllayersizes[0],__FUNCTION__+": incorrect (nonexistent) I"))
{
mean=0;
sigma=0;
return;
}
//--- change values
mean=network.m_columnmeans[i];
sigma=network.m_columnsigmas[i];
//--- check
if(sigma==0.0)
sigma=1;
}
//+------------------------------------------------------------------+
//| This function returns offset/scaling coefficients for I-th output|
//| of the network. |
//| INPUT PARAMETERS: |
//| Network - network |
//| I - input index |
//| OUTPUT PARAMETERS: |
//| Mean - mean term |
//| Sigma - sigma term, guaranteed to be nonzero. |
//| I-th output is passed through linear transformation |
//| OUT[i] = OUT[i]*Sigma+Mean |
//| before returning it to user. In case we have SOFTMAX-normalized |
//| network, we return (Mean,Sigma)=(0.0,1.0). |
//+------------------------------------------------------------------+
void CMLPBase::MLPGetOutputScaling(CMultilayerPerceptron &network,
const int i,double &mean,
double &sigma)
{
//--- check
if(!CAp::Assert(i>=0 && i<network.m_hllayersizes[CAp::Len(network.m_hllayersizes)-1],__FUNCTION__+": incorrect (nonexistent) I"))
{
mean=0;
sigma=0;
return;
}
//--- check
if(network.m_structinfo[6]==1)
{
//--- change values
mean=0;
sigma=1;
}
else
{
//--- change values
mean=network.m_columnmeans[network.m_hllayersizes[0]+i];
sigma=network.m_columnsigmas[network.m_hllayersizes[0]+i];
}
}
//+------------------------------------------------------------------+
//| This function returns information about Ith neuron of Kth layer |
//| INPUT PARAMETERS: |
//| Network - network |
//| K - layer index |
//| I - neuron index (within layer) |
//| OUTPUT PARAMETERS: |
//| FKind - activation function type (used by |
//| MLPActivationFunction()) this value is zero |
//| for input or linear neurons |
//| Threshold - also called offset, bias |
//| zero for input neurons |
//| NOTE: this function throws exception if layer or neuron with |
//| given index do not exists. |
//+------------------------------------------------------------------+
void CMLPBase::MLPGetNeuronInfo(CMultilayerPerceptron &network,
const int k,const int i,int &fkind,
double &threshold)
{
//--- create variables
int ncnt=0;
int istart=0;
int highlevelidx=0;
int activationoffset=0;
//--- initialization
fkind=0;
threshold=0;
ncnt=network.m_hlneurons.Size()/m_hlm_nfieldwidth;
istart=network.m_structinfo[5];
//--- search
network.m_integerbuf.Set(0,k);
network.m_integerbuf.Set(1,i);
//--- function call
highlevelidx=CApServ::RecSearch(network.m_hlneurons,m_hlm_nfieldwidth,2,0,ncnt,network.m_integerbuf);
//--- check
if(!CAp::Assert(highlevelidx>=0,__FUNCTION__+": incorrect (nonexistent) layer or neuron index"))
return;
//--- 1. find offset of the activation function record in the
if(network.m_hlneurons[highlevelidx*m_hlm_nfieldwidth+2]>=0)
{
activationoffset=istart+network.m_hlneurons[highlevelidx*m_hlm_nfieldwidth+2]*m_nfieldwidth;
fkind=network.m_structinfo[activationoffset+0];
}
else
fkind=0;
//--- check
if(network.m_hlneurons[highlevelidx*m_hlm_nfieldwidth+3]>=0)
threshold=network.m_weights[network.m_hlneurons[highlevelidx*m_hlm_nfieldwidth+3]];
else
threshold=0;
}
//+------------------------------------------------------------------+
//| This function returns information about connection from I0-th |
//| neuron of K0-th layer to I1-th neuron of K1-th layer. |
//| INPUT PARAMETERS: |
//| Network - network |
//| K0 - layer index |
//| I0 - neuron index (within layer) |
//| K1 - layer index |
//| I1 - neuron index (within layer) |
//| RESULT: |
//| connection weight (zero for non-existent connections) |
//| This function: |
//| 1. throws exception if layer or neuron with given index do not |
//| exists. |
//| 2. returns zero if neurons exist, but there is no connection |
//| between them |
//+------------------------------------------------------------------+
double CMLPBase::MLPGetWeight(CMultilayerPerceptron &network,
const int k0,const int i0,
const int k1,const int i1)
{
//--- create variables
double result=0;
int ccnt=0;
int highlevelidx=0;
//--- initialization
ccnt=CAp::Len(network.m_hlconnections)/m_hlconm_nfieldwidth;
//--- check params
if(!CAp::Assert(k0>=0 && k0<CAp::Len(network.m_hllayersizes),__FUNCTION__+": incorrect (nonexistent) K0"))
return(EMPTY_VALUE);
//--- check
if(!CAp::Assert(i0>=0 && i0<network.m_hllayersizes[k0],__FUNCTION__+": incorrect (nonexistent) I0"))
return(EMPTY_VALUE);
//--- check
if(!CAp::Assert(k1>=0 && k1<CAp::Len(network.m_hllayersizes),__FUNCTION__+": incorrect (nonexistent) K1"))
return(EMPTY_VALUE);
//--- check
if(!CAp::Assert(i1>=0 && i1<network.m_hllayersizes[k1],__FUNCTION__+": incorrect (nonexistent) I1"))
return(EMPTY_VALUE);
//--- search
network.m_integerbuf.Set(0,k0);
network.m_integerbuf.Set(1,i0);
network.m_integerbuf.Set(2,k1);
network.m_integerbuf.Set(3,i1);
//--- function call
highlevelidx=CApServ::RecSearch(network.m_hlconnections,m_hlconm_nfieldwidth,4,0,ccnt,network.m_integerbuf);
//--- check
if(highlevelidx>=0)
result=network.m_weights[network.m_hlconnections[highlevelidx*m_hlconm_nfieldwidth+4]];
else
result=0;
//--- return result
return(result);
}
//+------------------------------------------------------------------+
//| This function sets offset/scaling coefficients for I-th input of |
//| the network. |
//| INPUT PARAMETERS: |
//| Network - network |
//| I - input index |
//| Mean - mean term |
//| Sigma - sigma term (if zero,will be replaced by 1.0) |
//| NTE: I-th input is passed through linear transformation |
//| IN[i]=(IN[i]-Mean)/Sigma |
//| before feeding to the network. This function sets Mean and Sigma.|
//+------------------------------------------------------------------+
void CMLPBase::MLPSetInputScaling(CMultilayerPerceptron &network,
const int i,const double mean,
double sigma)
{
//--- check
if(!CAp::Assert(i>=0 && i<network.m_hllayersizes[0],__FUNCTION__+": incorrect (nonexistent) I"))
return;
//--- check
if(!CAp::Assert(MathIsValidNumber(mean),__FUNCTION__+": infinite or NAN Mean"))
return;
//--- check
if(!CAp::Assert(MathIsValidNumber(sigma),__FUNCTION__+": infinite or NAN Sigma"))
return;
//--- check
if(sigma==0.0)
sigma=1;
//--- change values
network.m_columnmeans.Set(i,mean);
network.m_columnsigmas.Set(i,sigma);
}
//+------------------------------------------------------------------+
//| This function sets offset/scaling coefficients for I-th output of|
//| the network. |
//| INPUT PARAMETERS: |
//| Network - network |
//| I - input index |
//| Mean - mean term |
//| Sigma - sigma term (if zero, will be replaced by 1.0)|
//| OUTPUT PARAMETERS: |
//| NOTE: I-th output is passed through linear transformation |
//| OUT[i] = OUT[i]*Sigma+Mean |
//| before returning it to user. This function sets Sigma/Mean. In |
//| case we have SOFTMAX-normalized network, you can not set (Sigma, |
//| Mean) to anything other than(0.0,1.0) - this function will throw |
//| exception. |
//+------------------------------------------------------------------+
void CMLPBase::MLPSetOutputScaling(CMultilayerPerceptron &network,
const int i,const double mean,
double sigma)
{
//--- check
if(!CAp::Assert(i>=0 && i<network.m_hllayersizes[CAp::Len(network.m_hllayersizes)-1],__FUNCTION__+": incorrect (nonexistent) I"))
return;
//--- check
if(!CAp::Assert(MathIsValidNumber(mean),__FUNCTION__+": infinite or NAN Mean"))
return;
//--- check
if(!CAp::Assert(MathIsValidNumber(sigma),__FUNCTION__+": infinite or NAN Sigma"))
return;
//--- check
if(network.m_structinfo[6]==1)
{
//--- check
if(!CAp::Assert(mean==0.0,__FUNCTION__+": you can not set non-zero Mean term for classifier network"))
return;
//--- check
if(!CAp::Assert(sigma==1.0,__FUNCTION__+": you can not set non-unit Sigma term for classifier network"))
return;
}
else
{
//--- check
if(sigma==0.0)
sigma=1;
//--- change values
network.m_columnmeans.Set(network.m_hllayersizes[0]+i,mean);
network.m_columnsigmas.Set(network.m_hllayersizes[0]+i,sigma);
}
}
//+------------------------------------------------------------------+
//| This function modifies information about Ith neuron of Kth layer |
//| INPUT PARAMETERS: |
//| Network - network |
//| K - layer index |
//| I - neuron index (within layer) |
//| FKind - activation function type (used by |
//| MLPActivationFunction()) this value must be |
//| zero for input neurons (you can not set |
//| activation function for input neurons) |
//| Threshold - also called offset, bias |
//| this value must be zero for input neurons |
//| (you can not set threshold for input neurons)|
//| NOTES: |
//| 1. this function throws exception if layer or neuron with given |
//| index do not exists. |
//| 2. this function also throws exception when you try to set |
//| non-linear activation function for input neurons (any kind |
//| of network) or for output neurons of classifier network. |
//| 3. this function throws exception when you try to set non-zero |
//| threshold for input neurons (any kind of network). |
//+------------------------------------------------------------------+
void CMLPBase::MLPSetNeuronInfo(CMultilayerPerceptron &network,
const int k,const int i,
const int fkind,const double threshold)
{
//--- create variables
int ncnt=0;
int istart=0;
int highlevelidx=0;
int activationoffset=0;
//--- check
if(!CAp::Assert(MathIsValidNumber(threshold),__FUNCTION__+": infinite or NAN Threshold"))
return;
//--- convenience vars
ncnt=CAp::Len(network.m_hlneurons)/m_hlm_nfieldwidth;
istart=network.m_structinfo[5];
//--- search
network.m_integerbuf.Set(0,k);
network.m_integerbuf.Set(1,i);
//--- function call
highlevelidx=CApServ::RecSearch(network.m_hlneurons,m_hlm_nfieldwidth,2,0,ncnt,network.m_integerbuf);
//--- check
if(!CAp::Assert(highlevelidx>=0,__FUNCTION__+": incorrect (nonexistent) layer or neuron index"))
return;
//--- activation function
if(network.m_hlneurons[highlevelidx*m_hlm_nfieldwidth+2]>=0)
{
activationoffset=istart+network.m_hlneurons[highlevelidx*m_hlm_nfieldwidth+2]*m_nfieldwidth;
network.m_structinfo.Set(activationoffset,fkind);
}
else
{
//--- check
if(!CAp::Assert(fkind==0,__FUNCTION__+": you try to set activation function for neuron which can not have one"))
return;
}
//--- Threshold
if(network.m_hlneurons[highlevelidx*m_hlm_nfieldwidth+3]>=0)
network.m_weights.Set(network.m_hlneurons[highlevelidx*m_hlm_nfieldwidth+3],threshold);
else
{
//--- check
if(!CAp::Assert(threshold==0.0,__FUNCTION__+": you try to set non-zero threshold for neuron which can not have one"))
return;
}
}
//+------------------------------------------------------------------+
//| This function modifies information about connection from I0-th |
//| neuron of K0-th layer to I1-th neuron of K1-th layer. |
//| INPUT PARAMETERS: |
//| Network - network |
//| K0 - layer index |
//| I0 - neuron index (within layer) |
//| K1 - layer index |
//| I1 - neuron index (within layer) |
//| W - connection weight (must be zero for |
//| non-existent connections) |
//| This function: |
//| 1. throws exception if layer or neuron with given index do not |
//| exists. |
//| 2. throws exception if you try to set non-zero weight for |
//| non-existent connection |
//+------------------------------------------------------------------+
void CMLPBase::MLPSetWeight(CMultilayerPerceptron &network,const int k0,
const int i0,const int k1,
const int i1,const double w)
{
//--- create variables
int ccnt=0;
int highlevelidx=0;
//--- initialization
ccnt=CAp::Len(network.m_hlconnections)/m_hlconm_nfieldwidth;
//--- check params
if(!CAp::Assert(k0>=0 && k0<CAp::Len(network.m_hllayersizes),__FUNCTION__+": incorrect (nonexistent) K0"))
return;
//--- check
if(!CAp::Assert(i0>=0 && i0<network.m_hllayersizes[k0],__FUNCTION__+": incorrect (nonexistent) I0"))
return;
//--- check
if(!CAp::Assert(k1>=0 && k1<CAp::Len(network.m_hllayersizes),__FUNCTION__+": incorrect (nonexistent) K1"))
return;
//--- check
if(!CAp::Assert(i1>=0 && i1<network.m_hllayersizes[k1],__FUNCTION__+": incorrect (nonexistent) I1"))
return;
//--- check
if(!CAp::Assert(MathIsValidNumber(w),__FUNCTION__+": infinite or NAN weight"))
return;
//--- search
network.m_integerbuf.Set(0,k0);
network.m_integerbuf.Set(1,i0);
network.m_integerbuf.Set(2,k1);
network.m_integerbuf.Set(3,i1);
//--- function call
highlevelidx=CApServ::RecSearch(network.m_hlconnections,m_hlconm_nfieldwidth,4,0,ccnt,network.m_integerbuf);
//--- check
if(highlevelidx>=0)
network.m_weights.Set(network.m_hlconnections[highlevelidx*m_hlconm_nfieldwidth+4],w);
else
{
//--- check
if(!CAp::Assert(w==0.0,__FUNCTION__+": you try to set non-zero weight for non-existent connection"))
return;
}
}
//+------------------------------------------------------------------+
//| Neural network activation function |
//| INPUT PARAMETERS: |
//| NET - neuron input |
//| K - function index (zero for linear function) |
//| OUTPUT PARAMETERS: |
//| F - function |
//| DF - its derivative |
//| D2F - its second derivative |
//+------------------------------------------------------------------+
void CMLPBase::MLPActivationFunction(double net,const int k,double &f,
double &df,double &d2f)
{
//--- create variables
double net2=0;
double arg=0;
double root=0;
double r=0;
//--- initialization
f=0;
df=0;
d2f=0;
//--- check
if(k==0 || k==-5)
{
f=net;
df=1;
d2f=0;
//--- exit the function
return;
}
//--- check
if(k==1)
{
//--- TanH activation function
if(MathAbs(net)<100.0)
f=MathTanh(net);
else
f=MathSign(net);
//--- change values
df=1-CMath::Sqr(f);
d2f=-(2*f*df);
//--- exit the function
return;
}
//--- check
if(k==3)
{
//--- EX activation function
if(net>=0.0)
{
//--- change values
net2=net*net;
arg=net2+1;
root=MathSqrt(arg);
f=net+root;
r=net/root;
df=1+r;
d2f=(root-net*r)/arg;
}
else
{
//--- change values
f=MathExp(net);
df=f;
d2f=f;
}
//--- exit the function
return;
}
//--- check
if(k==2)
{
//--- calculation
f=MathExp(-CMath::Sqr(net));
df=-(2*net*f);
d2f=-(2*(f+df*net));
//--- exit the function
return;
}
}
//+------------------------------------------------------------------+
//| Procesing |
//| INPUT PARAMETERS: |
//| Network - neural network |
//| X - input vector, array[0..NIn-1]. |
//| OUTPUT PARAMETERS: |
//| Y - result. Regression estimate when solving |
//| regression task, vector of posterior |
//| probabilities for classification task. |
//| See also MLPProcessI |
//+------------------------------------------------------------------+
void CMLPBase::MLPProcess(CMultilayerPerceptron &network,double &x[],
double &y[])
{
//--- check
CRowDouble X=x;
CRowDouble Y=vector<double>::Zeros(network.m_structinfo[2]);
//--- function call
MLPInternalProcessVector(network.m_structinfo,network.m_weights,network.m_columnmeans,network.m_columnsigmas,network.m_neurons,network.m_dfdnet,X,Y);
//--- return result
Y.ToArray(y);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CMLPBase::MLPProcess(CMultilayerPerceptron &network,CRowDouble &x,
CRowDouble &y)
{
//--- check
if(y.Size()<network.m_structinfo[2])
y.Resize(network.m_structinfo[2]);
//--- function call
MLPInternalProcessVector(network.m_structinfo,network.m_weights,network.m_columnmeans,network.m_columnsigmas,network.m_neurons,network.m_dfdnet,x,y);
}
//+------------------------------------------------------------------+
//| 'interactive' variant of MLPProcess for languages like Python |
//| which support constructs like "Y = MLPProcess(NN,X)" and |
//| interactive mode of the interpreter |
//| This function allocates new array on each call, so it is |
//| significantly slower than its 'non-interactive' counterpart, |
//| but it is more convenient when you call it from command line. |
//+------------------------------------------------------------------+
void CMLPBase::MLPProcessI(CMultilayerPerceptron &network,double &x[],
double &y[])
{
MLPProcess(network,x,y);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CMLPBase::MLPProcessI(CMultilayerPerceptron &network,CRowDouble &x,
CRowDouble &y)
{
MLPProcess(network,x,y);
}
//+------------------------------------------------------------------+
//| Error function for neural network,internal subroutine. |
//+------------------------------------------------------------------+
double CMLPBase::MLPError(CMultilayerPerceptron &network,
CMatrixDouble &xy,const int npoints)
{
double result=0;
//--- check
if(!CAp::Assert(xy.Rows()>=npoints,__FUNCTION__": XY has less than NPoints rows"))
return(result);
if(npoints>0)
{
if(MLPIsSoftMax(network))
{
if(!CAp::Assert(xy.Cols()>MLPGetInputsCount(network),__FUNCTION__": XY has less than NIn+1 columns"))
return(result);
}
else
if(!CAp::Assert(xy.Cols()>=MLPGetInputsCount(network)+MLPGetOutputsCount(network),__FUNCTION__": XY has less than NIn+NOut columns"))
return(result);
}
//--- function call
MLPAllErrorsX(network,xy,network.m_dummysxy,npoints,0,network.m_dummyidx,0,npoints,0,network.m_err);
result=CMath::Sqr(network.m_err.m_RMSError)*npoints*MLPGetOutputsCount(network)/2;
//--- return result
return(result);
}
//+------------------------------------------------------------------+
//| Error of the neural network on dataset given by sparse matrix. |
//| INPUT PARAMETERS: |
//| Network - neural network |
//| XY - training set, see below for information on the |
//| training set format. This function checks |
//| correctness of the dataset (no NANs/INFs, class |
//| numbers are correct) and throws exception when |
//| incorrect dataset is passed. Sparse matrix must |
//| use CRS format for storage. |
//| NPoints - points count, >=0 |
//| RESULT: |
//| sum-of-squares error, SUM(sqr(y[i]-desired_y[i])/2) |
//| DATASET FORMAT: |
//| This function uses two different dataset formats - one for |
//| regression networks, another one for classification networks. |
//| For regression networks with NIn inputs and NOut outputs |
//| following dataset format is used: |
//| * dataset is given by NPoints*(NIn+NOut) matrix |
//| * each row corresponds to one example |
//| * first NIn columns are inputs, next NOut columns are outputs |
//| For classification networks with NIn inputs and NClasses clases |
//| following dataset format is used: |
//| * dataset is given by NPoints*(NIn+1) matrix |
//| * each row corresponds to one example |
//| * first NIn columns are inputs, last column stores class number|
//| (from 0 to NClasses-1). |
//+------------------------------------------------------------------+
double CMLPBase::MLPErrorSparse(CMultilayerPerceptron &network,
CSparseMatrix &xy,int npoints)
{
double result=0;
//--- check
if(!CAp::Assert(CSparse::SparseIsCRS(xy),__FUNCTION__": XY is not in CRS format."))
return(result);
if(!CAp::Assert(CSparse::SparseGetNRows(xy)>=npoints,__FUNCTION__": XY has less than NPoints rows"))
return(result);
if(npoints>0)
{
if(MLPIsSoftMax(network))
{
if(!CAp::Assert(CSparse::SparseGetNCols(xy)>MLPGetInputsCount(network),__FUNCTION__": XY has less than NIn+1 columns"))
return(result);
}
else
if(!CAp::Assert(CSparse::SparseGetNCols(xy)>=MLPGetInputsCount(network)+MLPGetOutputsCount(network),__FUNCTION__": XY has less than NIn+NOut columns"))
return(result);
}
MLPAllErrorsX(network,network.m_dummydxy,xy,npoints,1,network.m_dummyidx,0,npoints,0,network.m_err);
result=CMath::Sqr(network.m_err.m_RMSError)*npoints*MLPGetOutputsCount(network)/2;
//--- return result
return(result);
}
//+------------------------------------------------------------------+
//| Natural error function for neural network,internal subroutine. |
//+------------------------------------------------------------------+
double CMLPBase::MLPErrorN(CMultilayerPerceptron &network,
CMatrixDouble &xy,const int ssize)
{
//--- create variables
double result=0;
int i=0;
int k=0;
int nin=0;
int nout=0;
int wcount=0;
double e=0;
int i_=0;
int i1_=0;
//--- function call
MLPProperties(network,nin,nout,wcount);
matrix<double> splitted[];
ulong parts[]={nin,nout};
xy.Split(parts,1,splitted);
//--- calculation
for(i=0; i<ssize; i++)
{
//--- Process vector
network.m_x=splitted[0].Row(i);
//--- function call
MLPProcess(network,network.m_x,network.m_y);
//--- Update error function
if(network.m_structinfo[6]==0)
{
//--- Least squares error function
network.m_y-=splitted[1].Row(i);
//--- calculation
e=network.m_y.Dot(network.m_y);
result+=e/2;
}
else
{
//--- Cross-entropy error function
k=(int)MathRound(splitted[1][i,0]);
//--- check
if(k>=0 && k<nout)
result+=SafeCrossEntropy(1,network.m_y[k]);
}
}
//--- return result
return(result);
}
//+------------------------------------------------------------------+
//| Classification error |
//+------------------------------------------------------------------+
int CMLPBase::MLPClsError(CMultilayerPerceptron &network,
CMatrixDouble &xy,const int npoints)
{
int result=0;
//--- check
if(!CAp::Assert(xy.Rows()>=npoints,__FUNCTION__": XY has less than NPoints rows"))
return(result);
if(npoints>0)
{
if(MLPIsSoftMax(network))
{
if(!CAp::Assert(xy.Cols()>MLPGetInputsCount(network),__FUNCTION__": XY has less than NIn+1 columns"))
return(result);
}
else
if(!CAp::Assert(xy.Cols()>=MLPGetInputsCount(network)+MLPGetOutputsCount(network),__FUNCTION__": XY has less than NIn+NOut columns"))
return(result);
}
//--- function call
MLPAllErrorsX(network,xy,network.m_dummysxy,npoints,0,network.m_dummyidx,0,npoints,0,network.m_err);
result=(int)MathRound(npoints*network.m_err.m_RelCLSError);
//--- return result
return(result);
}
//+------------------------------------------------------------------+
//| Relative classification error on the test set |
//| INPUT PARAMETERS: |
//| Network - network |
//| XY - test set |
//| NPoints - test set size |
//| RESULT: |
//| percent of incorrectly classified cases. Works both for |
//| classifier networks and general purpose networks used as |
//| classifiers. |
//+------------------------------------------------------------------+
double CMLPBase::MLPRelClsError(CMultilayerPerceptron &network,
CMatrixDouble &xy,const int npoints)
{
return((double)MLPClsError(network,xy,npoints)/(double)npoints);
}
//+------------------------------------------------------------------+
//| Relative classification error on the test set given by sparse |
//| matrix. |
//| INPUT PARAMETERS: |
//| Network - neural network; |
//| XY - training set, see below for information on the |
//| training set format. Sparse matrix must use CRS |
//| format for storage. |
//| NPoints - points count, >=0. |
//| RESULT: |
//| Percent of incorrectly classified cases. Works both for |
//| classifier networks and general purpose networks used as |
//| classifiers. |
//| DATASET FORMAT: |
//| This function uses two different dataset formats - one for |
//| regression networks, another one for classification networks. |
//| For regression networks with NIn inputs and NOut outputs |
//| following dataset format is used: |
//| * dataset is given by NPoints*(NIn+NOut) matrix |
//| * each row corresponds to one example |
//| * first NIn columns are inputs, next NOut columns are |
//| outputs |
//| For classification networks with NIn inputs and NClasses clases|
//| following dataset format is used: |
//| * dataset is given by NPoints*(NIn+1) matrix |
//| * each row corresponds to one example |
//| * first NIn columns are inputs, last column stores class |
//| number (from 0 to NClasses-1). |
//+------------------------------------------------------------------+
double CMLPBase::MLPRelClsErrorSparse(CMultilayerPerceptron &network,
CSparseMatrix &xy,int npoints)
{
double result=0;
//--- check
if(!CAp::Assert(CSparse::SparseIsCRS(xy),__FUNCTION__": sparse matrix XY is not in CRS format."))
return(result);
if(!CAp::Assert(CSparse::SparseGetNRows(xy)>=npoints,__FUNCTION__": sparse matrix XY has less than NPoints rows"))
return(result);
if(npoints>0)
{
if(MLPIsSoftMax(network))
{
if(!CAp::Assert(CSparse::SparseGetNCols(xy)>MLPGetInputsCount(network),__FUNCTION__": sparse matrix XY has less than NIn+1 columns"))
return(result);
}
else
if(!CAp::Assert(CSparse::SparseGetNCols(xy)>=MLPGetInputsCount(network)+MLPGetOutputsCount(network),__FUNCTION__": sparse matrix XY has less than NIn+NOut columns"))
return(result);
}
//--- function call
MLPAllErrorsX(network,network.m_dummydxy,xy,npoints,1,network.m_dummyidx,0,npoints,0,network.m_err);
//--- return result
return(network.m_err.m_RelCLSError);
}
//+------------------------------------------------------------------+
//| Average cross-entropy (in bits per element) on the test set |
//| INPUT PARAMETERS: |
//| Network - neural network |
//| XY - test set |
//| NPoints - test set size |
//| RESULT: |
//| CrossEntropy/(NPoints*LN(2)). |
//| Zero if network solves regression task. |
//+------------------------------------------------------------------+
double CMLPBase::MLPAvgCE(CMultilayerPerceptron &network,CMatrixDouble &xy,
const int npoints)
{
double result=0;
//--- check
if(!CAp::Assert(xy.Rows()>=npoints,__FUNCTION__": XY has less than NPoints rows"))
return(result);
if(npoints>0)
{
if(MLPIsSoftMax(network))
{
if(!CAp::Assert(xy.Cols()>MLPGetInputsCount(network),__FUNCTION__": XY has less than NIn+1 columns"))
return(result);
}
else
if(!CAp::Assert(xy.Cols()>=MLPGetInputsCount(network)+MLPGetOutputsCount(network),__FUNCTION__": XY has less than NIn+NOut columns"))
return(result);
}
//--- function call
MLPAllErrorsX(network,xy,network.m_dummysxy,npoints,0,network.m_dummyidx,0,npoints,0,network.m_err);
//--- return result
return(network.m_err.m_AvgCE);
}
//+------------------------------------------------------------------+
//| Average cross-entropy (in bits per element) on the test set given|
//| by sparse matrix. |
//| INPUT PARAMETERS: |
//| Network - neural network; |
//| XY - training set, see below for information on the |
//| training set format. This function checks |
//| correctness of the dataset (no NANs/INFs, class |
//| numbers are correct) and throws exception when |
//| incorrect dataset is passed. Sparse matrix must |
//| use CRS format for storage. |
//| NPoints - points count, >=0. |
//| RESULT: |
//| CrossEntropy/(NPoints*LN(2)). |
//| Zero if network solves regression task. |
//| DATASET FORMAT: |
//| This function uses two different dataset formats - one for |
//| regression networks, another one for classification networks. |
//| For regression networks with NIn inputs and NOut outputs |
//| following dataset format is used: |
//| * dataset is given by NPoints*(NIn+NOut) matrix |
//| * each row corresponds to one example |
//| * first NIn columns are inputs, next NOut columns are |
//| outputs |
//| For classification networks with NIn inputs and NClasses clases|
//| following dataset format is used: |
//| * dataset is given by NPoints*(NIn+1) matrix |
//| * each row corresponds to one example |
//| * first NIn columns are inputs, last column stores class |
//| number (from 0 to NClasses-1). |
//+------------------------------------------------------------------+
double CMLPBase::MLPAvgCESparse(CMultilayerPerceptron &network,
CSparseMatrix &xy,int npoints)
{
double result=0;
//--- check
if(!CAp::Assert(CSparse::SparseIsCRS(xy),__FUNCTION__": sparse matrix XY is not in CRS format."))
return(result);
if(!CAp::Assert(CSparse::SparseGetNRows(xy)>=npoints,__FUNCTION__": sparse matrix XY has less than NPoints rows"))
return(result);
if(npoints>0)
{
if(MLPIsSoftMax(network))
{
if(!CAp::Assert(CSparse::SparseGetNCols(xy)>MLPGetInputsCount(network),__FUNCTION__": sparse matrix XY has less than NIn+1 columns"))
return(result);
}
else
if(!CAp::Assert(CSparse::SparseGetNCols(xy)>=MLPGetInputsCount(network)+MLPGetOutputsCount(network),__FUNCTION__": sparse matrix XY has less than NIn+NOut columns"))
return(result);
}
//--- function call
MLPAllErrorsX(network,network.m_dummydxy,xy,npoints,1,network.m_dummyidx,0,npoints,0,network.m_err);
//--- return result
return(network.m_err.m_AvgCE);
}
//+------------------------------------------------------------------+
//| RMS error on the test set |
//| INPUT PARAMETERS: |
//| Network - neural network |
//| XY - test set |
//| NPoints - test set size |
//| RESULT: |
//| root mean square error. |
//| Its meaning for regression task is obvious. As for |
//| classification task,RMS error means error when estimating |
//| posterior probabilities. |
//+------------------------------------------------------------------+
double CMLPBase::MLPRMSError(CMultilayerPerceptron &network,
CMatrixDouble &xy,const int npoints)
{
//--- check
if(!CAp::Assert(xy.Rows()>=npoints,__FUNCTION__": XY has less than NPoints rows"))
return(0);
if(npoints>0)
{
if(MLPIsSoftMax(network))
{
if(!CAp::Assert(xy.Cols()>MLPGetInputsCount(network),__FUNCTION__": XY has less than NIn+1 columns"))
return(0);
}
else
if(!CAp::Assert(xy.Cols()>=MLPGetInputsCount(network)+MLPGetOutputsCount(network),__FUNCTION__": XY has less than NIn+NOut columns"))
return(0);
}
//--- function call
MLPAllErrorsX(network,xy,network.m_dummysxy,npoints,0,network.m_dummyidx,0,npoints,0,network.m_err);
//--- return result
return(network.m_err.m_RMSError);
}
//+------------------------------------------------------------------+
//| RMS error on the test set given by sparse matrix. |
//| INPUT PARAMETERS: |
//| Network - neural network; |
//| XY - training set, see below for information on the |
//| training set format. This function checks |
//| correctness of the dataset (no NANs/INFs, class |
//| numbers are correct) and throws exception when |
//| incorrect dataset is passed. Sparse matrix must |
//| use CRS format for storage. |
//| NPoints - points count, >=0. |
//| RESULT: |
//| Root mean square error. Its meaning for regression task is |
//| obvious. As for classification task, RMS error means error when|
//| estimating posterior probabilities. |
//| DATASET FORMAT: |
//| This function uses two different dataset formats - one for |
//| regression networks, another one for classification networks. |
//| For regression networks with NIn inputs and NOut outputs |
//| following dataset format is used: |
//| * dataset is given by NPoints*(NIn + NOut) matrix |
//| * each row corresponds to one example |
//| * first NIn columns are inputs, next NOut columns are |
//| outputs |
//| For classification networks with NIn inputs and NClasses |
//| clases following dataset format is used: |
//| * dataset is given by NPoints*(NIn + 1) matrix |
//| * each row corresponds to one example |
//| * first NIn columns are inputs, last column stores class |
//| number(from 0 to NClasses - 1). |
//+------------------------------------------------------------------+
double CMLPBase::MLPRMSErrorSparse(CMultilayerPerceptron &network,
CSparseMatrix &xy,int npoints)
{
//--- check
if(!CAp::Assert(CSparse::SparseIsCRS(xy),__FUNCTION__": sparse matrix XY is not in CRS format."))
return(0);
if(!CAp::Assert(CSparse::SparseGetNRows(xy)>=npoints,__FUNCTION__": sparse matrix XY has less than NPoints rows"))
return(0);
if(npoints>0)
{
if(MLPIsSoftMax(network))
{
if(!CAp::Assert(CSparse::SparseGetNCols(xy)>MLPGetInputsCount(network),__FUNCTION__": sparse matrix XY has less than NIn+1 columns"))
return(0);
}
else
if(!CAp::Assert(CSparse::SparseGetNCols(xy)>=MLPGetInputsCount(network)+MLPGetOutputsCount(network),__FUNCTION__": sparse matrix XY has less than NIn+NOut columns"))
return(0);
}
//---function call
MLPAllErrorsX(network,network.m_dummydxy,xy,npoints,1,network.m_dummyidx,0,npoints,0,network.m_err);
//--- return result
return(network.m_err.m_RMSError);
}
//+------------------------------------------------------------------+
//| Average error on the test set |
//| INPUT PARAMETERS: |
//| Network - neural network |
//| XY - test set |
//| NPoints - test set size |
//| RESULT: |
//| Its meaning for regression task is obvious. As for |
//| classification task,it means average error when estimating |
//| posterior probabilities. |
//+------------------------------------------------------------------+
double CMLPBase::MLPAvgError(CMultilayerPerceptron &network,
CMatrixDouble &xy,const int npoints)
{
//--- check
if(!CAp::Assert(xy.Rows()>=npoints,__FUNCTION__": XY has less than NPoints rows"))
return(0);
if(npoints>0)
{
if(MLPIsSoftMax(network))
{
if(!CAp::Assert(xy.Cols()>MLPGetInputsCount(network),__FUNCTION__": XY has less than NIn+1 columns"))
return(0);
}
else
if(!CAp::Assert(xy.Cols()>=MLPGetInputsCount(network)+MLPGetOutputsCount(network),__FUNCTION__": XY has less than NIn+NOut columns"))
return(0);
}
//--- function call
MLPAllErrorsX(network,xy,network.m_dummysxy,npoints,0,network.m_dummyidx,0,npoints,0,network.m_err);
//--- return result
return(network.m_err.m_AvgError);
}
//+------------------------------------------------------------------+
//| Average absolute error on the test set given by sparse matrix. |
//| INPUT PARAMETERS: |
//| Network - neural network; |
//| XY - training set, see below for information on the |
//| training set format. This function checks |
//| correctness of the dataset (no NANs/INFs, class |
//| numbers are correct) and throws exception when |
//| incorrect dataset is passed. Sparse matrix must |
//| use CRS format for storage. |
//| NPoints - points count, >=0. |
//| RESULT: |
//| Its meaning for regression task is obvious. As for |
//| classification task, it means average error when estimating |
//| posterior probabilities. |
//| DATASET FORMAT: |
//| This function uses two different dataset formats - one for |
//| regression networks, another one for classification networks. |
//| For regression networks with NIn inputs and NOut outputs |
//| following dataset format is used: |
//| * dataset is given by NPoints*(NIn+NOut) matrix |
//| * each row corresponds to one example |
//| * first NIn columns are inputs, next NOut columns are |
//| outputs |
//| For classification networks with NIn inputs and NClasses clases|
//| following dataset format is used: |
//| * dataset is given by NPoints*(NIn+1) matrix |
//| * each row corresponds to one example |
//| * first NIn columns are inputs, last column stores class |
//| number (from 0 to NClasses-1). |
//+------------------------------------------------------------------+
double CMLPBase::MLPAvgErrorSparse(CMultilayerPerceptron &network,
CSparseMatrix &xy,int npoints)
{
//--- check
if(!CAp::Assert(CSparse::SparseIsCRS(xy),__FUNCTION__": sparse matrix XY is not in CRS format."))
return(0);
if(!CAp::Assert(CSparse::SparseGetNRows(xy)>=npoints,__FUNCTION__": sparse matrix XY has less than NPoints rows"))
return(0);
if(npoints>0)
{
if(MLPIsSoftMax(network))
{
if(!CAp::Assert(CSparse::SparseGetNCols(xy)>MLPGetInputsCount(network),__FUNCTION__": sparse matrix XY has less than NIn+1 columns"))
return(0);
}
else
if(!CAp::Assert(CSparse::SparseGetNCols(xy)>=MLPGetInputsCount(network)+MLPGetOutputsCount(network),__FUNCTION__": sparse matrix XY has less than NIn+NOut columns"))
return(0);
}
//---function call
MLPAllErrorsX(network,network.m_dummydxy,xy,npoints,1,network.m_dummyidx,0,npoints,0,network.m_err);
//--- return result
return(network.m_err.m_AvgError);
}
//+------------------------------------------------------------------+
//| Average relative error on the test set |
//| INPUT PARAMETERS: |
//| Network - neural network |
//| XY - test set |
//| NPoints - test set size |
//| RESULT: |
//| Its meaning for regression task is obvious. As for |
//| classification task, it means average relative error when |
//| estimating posterior probability of belonging to the correct |
//| class. |
//+------------------------------------------------------------------+
double CMLPBase::MLPAvgRelError(CMultilayerPerceptron &network,
CMatrixDouble &xy,const int npoints)
{
//--- check
if(!CAp::Assert(xy.Rows()>=npoints,__FUNCTION__": XY has less than NPoints rows"))
return(0);
if(npoints>0)
{
if(MLPIsSoftMax(network))
{
if(!CAp::Assert(xy.Cols()>MLPGetInputsCount(network),__FUNCTION__": XY has less than NIn+1 columns"))
return(0);
}
else
if(!CAp::Assert(xy.Cols()>=MLPGetInputsCount(network)+MLPGetOutputsCount(network),__FUNCTION__": XY has less than NIn+NOut columns"))
return(0);
}
//--- function call
MLPAllErrorsX(network,xy,network.m_dummysxy,npoints,0,network.m_dummyidx,0,npoints,0,network.m_err);
//--- return result
return(network.m_err.m_AvgRelError);
}
//+------------------------------------------------------------------+
//| Average relative error on the test set given by sparse matrix. |
//| INPUT PARAMETERS: |
//| Network - neural network; |
//| XY - training set, see below for information on the |
//| training set format. This function checks |
//| correctness of the dataset (no NANs/INFs, class |
//| numbers are correct) and throws exception when |
//| incorrect dataset is passed. Sparse matrix must |
//| use CRS format for storage. |
//| NPoints - points count, >=0. |
//| RESULT: |
//| Its meaning for regression task is obvious. As for |
//| classification task, it means average relative error when |
//| estimating posterior probability of belonging to the correct |
//| class. |
//| DATASET FORMAT: |
//| This function uses two different dataset formats - one for |
//| regression networks, another one for classification networks. |
//| For regression networks with NIn inputs and NOut outputs |
//| following dataset format is used: |
//| * dataset is given by NPoints*(NIn+NOut) matrix |
//| * each row corresponds to one example |
//| * first NIn columns are inputs, next NOut columns are |
//| outputs |
//| For classification networks with NIn inputs and NClasses clases|
//| following dataset format is used: |
//| * dataset is given by NPoints*(NIn+1) matrix |
//| * each row corresponds to one example |
//| * first NIn columns are inputs, last column stores class |
//| number (from 0 to NClasses-1). |
//+------------------------------------------------------------------+
double CMLPBase::MLPAvgRelErrorSparse(CMultilayerPerceptron &network,
CSparseMatrix &xy,int npoints)
{
//--- check
if(!CAp::Assert(CSparse::SparseIsCRS(xy),__FUNCTION__": sparse matrix XY is not in CRS format."))
return(0);
if(!CAp::Assert(CSparse::SparseGetNRows(xy)>=npoints,__FUNCTION__": sparse matrix XY has less than NPoints rows"))
return(0);
if(npoints>0)
{
if(MLPIsSoftMax(network))
{
if(!CAp::Assert(CSparse::SparseGetNCols(xy)>MLPGetInputsCount(network),__FUNCTION__": sparse matrix XY has less than NIn+1 columns"))
return(0);
}
else
if(!CAp::Assert(CSparse::SparseGetNCols(xy)>=MLPGetInputsCount(network)+MLPGetOutputsCount(network),__FUNCTION__": sparse matrix XY has less than NIn+NOut columns"))
return(0);
}
//---function call
MLPAllErrorsX(network,network.m_dummydxy,xy,npoints,1,network.m_dummyidx,0,npoints,0,network.m_err);
//--- return result
return(network.m_err.m_AvgRelError);
}
//+------------------------------------------------------------------+
//| Gradient calculation |
//| INPUT PARAMETERS: |
//| Network - network initialized with one of the network |
//| creation funcs |
//| X - input vector, length of array must be at least |
//| NIn |
//| DesiredY- desired outputs, length of array must be at least|
//| NOut |
//| Grad - possibly preallocated array. If size of array is |
//| smaller than WCount, it will be reallocated. It |
//| is recommended to reuse previously allocated |
//| array to reduce allocation overhead. |
//| OUTPUT PARAMETERS: |
//| E - error function, SUM(sqr(y[i]-desiredy[i])/2,i) |
//| Grad - gradient of E with respect to weights of network,|
//| array[WCount] |
//+------------------------------------------------------------------+
void CMLPBase::MLPGrad(CMultilayerPerceptron &network,double &x[],
double &desiredy[],double &e,double &grad[])
{
CRowDouble X=x;
CRowDouble Y=desiredy;
CRowDouble Grad;
MLPGrad(network,X,Y,e,Grad);
Grad.ToArray(grad);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CMLPBase::MLPGrad(CMultilayerPerceptron &network,CRowDouble &x,
CRowDouble &desiredy,double &e,CRowDouble &grad)
{
//--- create variables
int nout=0;
int ntotal=0;
//--- Alloc
if(CAp::Len(grad)<network.m_structinfo[4])
grad.Resize(network.m_structinfo[4]);
//--- Prepare dError/dOut,internal structures
MLPProcess(network,x,network.m_y);
//--- initialization
nout=network.m_structinfo[2];
ntotal=network.m_structinfo[3];
e=MathPow(network.m_y.ToVector()-desiredy.ToVector(),2.0).Sum()/2.0;
//--- change values
network.m_derror=vector<double>::Zeros(ntotal);
for(int i=0; i<nout; i++)
network.m_derror.Set(ntotal-nout+i,network.m_y[i]-desiredy[i]);
//--- gradient
MLPInternalCalculateGradient(network,network.m_neurons,network.m_weights,network.m_derror,grad,false);
}
//+------------------------------------------------------------------+
//| Gradient calculation (natural error function is used) |
//| INPUT PARAMETERS: |
//| Network - network initialized with one of the network |
//| creation funcs |
//| X - input vector, length of array must be at least |
//| NIn |
//| DesiredY- desired outputs, length of array must be at least|
//| NOut |
//| Grad - possibly preallocated array. If size of array is |
//| smaller than WCount, it will be reallocated. It |
//| is recommended to reuse previously allocated |
//| array to reduce allocation overhead. |
//| OUTPUT PARAMETERS: |
//| E - error function, sum-of-squares for regression |
//| networks, cross-entropy for classification |
//| networks. |
//| Grad - gradient of E with respect to weights of network,|
//| array[WCount] |
//+------------------------------------------------------------------+
void CMLPBase::MLPGradN(CMultilayerPerceptron &network,double &x[],
double &desiredy[],double &e,double &grad[])
{
CRowDouble X=x;
CRowDouble Y=desiredy;
CRowDouble Grad;
MLPGradN(network,X,Y,e,Grad);
Grad.ToArray(grad);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CMLPBase::MLPGradN(CMultilayerPerceptron &network,CRowDouble &x,
CRowDouble &desiredy,double &e,CRowDouble &grad)
{
//--- create variables
double s=0;
int nout=0;
int ntotal=0;
//--- initialization
e=0;
//--- Alloc
if(grad.Size()<network.m_structinfo[4])
grad.Resize(network.m_structinfo[4]);
//--- Prepare dError/dOut,internal structures
MLPProcess(network,x,network.m_y);
//--- change values
nout=network.m_structinfo[2];
ntotal=network.m_structinfo[3];
network.m_derror=vector<double>::Zeros(ntotal);
e=0;
//--- check
if(network.m_structinfo[6]==0)
{
//--- Regression network,least squares
e=MathPow(network.m_y.ToVector()-desiredy.ToVector(),2.0).Sum()/2.0;
for(int i=0; i<=nout-1; i++)
network.m_derror.Set(ntotal-nout+i,network.m_y[i]-desiredy[i]);
}
else
{
//--- Classification network,cross-entropy
s=desiredy.Sum();
for(int i=0; i<nout; i++)
{
network.m_derror.Set(ntotal-nout+i,s*network.m_y[i]-desiredy[i]);
e+=SafeCrossEntropy(desiredy[i],network.m_y[i]);
}
}
//--- gradient
MLPInternalCalculateGradient(network,network.m_neurons,network.m_weights,network.m_derror,grad,true);
}
//+------------------------------------------------------------------+
//| Batch gradient calculation for a set of inputs/outputs |
//| INPUT PARAMETERS: |
//| Network - network initialized with one of the network |
//| creation funcs |
//| XY - set of inputs/outputs; one sample = one row; |
//| first NIn columns contain inputs, |
//| next NOut columns - desired outputs. |
//| SSize - number of elements in XY |
//| Grad - possibly preallocated array. If size of array is |
//| smaller than WCount, it will be reallocated. It |
//| is recommended to reuse previously allocated |
//| array to reduce allocation overhead. |
//| OUTPUT PARAMETERS: |
//| E - error function, SUM(sqr(y[i]-desiredy[i])/2,i) |
//| Grad - gradient of E with respect to weights of network,|
//| array[WCount] |
//+------------------------------------------------------------------+
void CMLPBase::MLPGradBatch(CMultilayerPerceptron &network,
CMatrixDouble &xy,const int ssize,
double &e,double &grad[])
{
CRowDouble Grad;
MLPGradBatch(network,xy,ssize,e,Grad);
Grad.ToArray(grad);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CMLPBase::MLPGradBatch(CMultilayerPerceptron &network,
CMatrixDouble &xy,const int ssize,
double &e,CRowDouble &grad)
{
//--- create variables
int nin=0;
int nout=0;
int wcount=0;
//--- function call
MLPProperties(network,nin,nout,wcount);
//--- initialization
grad=vector<double>::Zeros(wcount);
e=0;
//--- function call
MLPGradBatchX(network,xy,network.m_dummysxy,ssize,0,network.m_dummyidx,0,ssize,0,e,grad);
}
//+------------------------------------------------------------------+
//| Batch gradient calculation for a set of inputs/outputs given by |
//| sparse matrices |
//| INPUT PARAMETERS: |
//| Network - network initialized with one of the network |
//| creation funcs |
//| XY - original dataset in sparse format; one sample = one|
//| row: |
//| * MATRIX MUST BE STORED IN CRS FORMAT |
//| * first NIn columns contain inputs. |
//| * for regression problem, next NOut columns store |
//| desired outputs. |
//| * for classification problem, next column (just |
//| one!) stores class number. |
//| SSize - number of elements in XY |
//| Grad - possibly preallocated array. If size of array is |
//| smaller than WCount, it will be reallocated. It is |
//| recommended to reuse previously allocated array to |
//| reduce allocation overhead. |
//| OUTPUT PARAMETERS: |
//| E - error function, SUM(sqr(y[i]-desiredy[i])/2,i) |
//| Grad - gradient of E with respect to weights of network, |
//| array[WCount] |
//+------------------------------------------------------------------+
void CMLPBase::MLPGradBatchSparse(CMultilayerPerceptron &network,
CSparseMatrix &xy,int ssize,
double &e,CRowDouble &grad)
{
//--- create variables
int nin=0;
int nout=0;
int wcount=0;
int subset0=0;
int subset1=ssize;
int subsettype=0;
e=0;
//--- check
if(!CAp::Assert(ssize>=0,__FUNCTION__": SSize<0"))
return;
if(!CAp::Assert(CSparse::SparseIsCRS(xy),__FUNCTION__": sparse matrix XY must be in CRS format."))
return;
MLPProperties(network,nin,nout,wcount);
grad=vector<double>::Zeros(wcount);
MLPGradBatchX(network,network.m_dummydxy,xy,ssize,1,network.m_dummyidx,subset0,subset1,subsettype,e,grad);
}
//+------------------------------------------------------------------+
//| Batch gradient calculation for a subset of dataset |
//| INPUT PARAMETERS: |
//| Network - network initialized with one of the network |
//| creation funcs |
//| XY - original dataset in dense format; one sample = one |
//| row: |
//| * first NIn columns contain inputs, |
//| * for regression problem, next NOut columns store |
//| desired outputs. |
//| * for classification problem, next column (just |
//| one!) stores class number. |
//| SetSize - real size of XY, SetSize>=0; |
//| Idx - subset of SubsetSize elements, array[SubsetSize]: |
//| * Idx[I] stores row index in the original dataset |
//| which is given by XY. Gradient is calculated with|
//| respect to rows whose indexes are stored in Idx[]|
//| * Idx[] must store correct indexes; this function |
//| throws an exception in case incorrect index (less|
//| than 0 or larger than rows(XY)) is given |
//| * Idx[] may store indexes in any order and even |
//| with repetitions. |
//| SubsetSize- number of elements in Idx[] array: |
//| * positive value means that subset given by Idx[] |
//| is processed |
//| * zero value results in zero gradient |
//| * negative value means that full dataset is |
//| processed |
//| Grad - possibly preallocated array. If size of array is |
//| smaller than WCount, it will be reallocated. It is |
//| recommended to reuse previously allocated array to |
//| reduce allocation overhead. |
//| OUTPUT PARAMETERS: |
//| E - error function, SUM(sqr(y[i]-desiredy[i])/2,i) |
//| Grad - gradient of E with respect to weights of network, |
//| array[WCount] |
//+------------------------------------------------------------------+
void CMLPBase::MLPGradBatchSubset(CMultilayerPerceptron &network,
CMatrixDouble &xy,int setsize,
CRowInt &idx,int subsetsize,
double &e,CRowDouble &grad)
{
//--- create variables
int nin=0;
int nout=0;
int wcount=0;
int npoints=0;
int subset0=0;
int subset1=0;
int subsettype=0;
e=0;
//--- check
if(!CAp::Assert(setsize>=0,__FUNCTION__": SetSize<0"))
return;
if(!CAp::Assert(subsetsize<=idx.Size(),__FUNCTION__": SubsetSize>Length(Idx)"))
return;
npoints=setsize;
if(subsetsize<0)
{
subset0=0;
subset1=setsize;
subsettype=0;
}
else
{
subset0=0;
subset1=subsetsize;
subsettype=1;
for(int i=0; i<subsetsize; i++)
{
if(!CAp::Assert(idx[i]>=0,__FUNCTION__": incorrect index of XY row(Idx[I]<0)"))
return;
if(!CAp::Assert(idx[i]<npoints,__FUNCTION__": incorrect index of XY row(Idx[I]>Rows(XY)-1)"))
return;
}
}
MLPProperties(network,nin,nout,wcount);
grad=vector<double>::Zeros(wcount);
MLPGradBatchX(network,xy,network.m_dummysxy,setsize,0,idx,subset0,subset1,subsettype,e,grad);
}
//+------------------------------------------------------------------+
//| Batch gradient calculation for a set of inputs/outputs for a |
//| subset of dataset given by set of indexes. |
//| INPUT PARAMETERS: |
//| Network - network initialized with one of the network |
//| creation funcs |
//| XY - original dataset in sparse format; one sample = one|
//| row: |
//| * MATRIX MUST BE STORED IN CRS FORMAT |
//| * first NIn columns contain inputs, |
//| * for regression problem, next NOut columns store |
//| desired outputs. |
//| * for classification problem, next column (just |
//| one!) stores class number. |
//| SetSize - real size of XY, SetSize>=0; |
//| Idx - subset of SubsetSize elements, array[SubsetSize]: |
//| * Idx[I] stores row index in the original dataset |
//| which is given by XY. Gradient is calculated with|
//| respect to rows whose indexes are stored in Idx[]|
//| * Idx[] must store correct indexes; this function |
//| throws an exception in case incorrect index (less|
//| than 0 or larger than rows(XY)) is given |
//| * Idx[] may store indexes in any order and even |
//| with repetitions. |
//| SubsetSize- number of elements in Idx[] array: |
//| * positive value means that subset given by Idx[] |
//| is processed |
//| * zero value results in zero gradient |
//| * negative value means that full dataset is |
//| processed |
//| Grad - possibly preallocated array. If size of array is |
//| smaller than WCount, it will be reallocated. It is |
//| recommended to reuse previously allocated array to |
//| reduce allocation overhead. |
//| OUTPUT PARAMETERS: |
//| E - error function, SUM(sqr(y[i]-desiredy[i])/2,i) |
//| Grad - gradient of E with respect to weights of network, |
//| array[WCount] |
//| NOTE: when SubsetSize<0 is used full dataset by call |
//| MLPGradBatchSparse function. |
//+------------------------------------------------------------------+
void CMLPBase::MLPGradBatchSparseSubset(CMultilayerPerceptron &network,
CSparseMatrix &xy,int setsize,
CRowInt &idx,int subsetsize,
double &e,CRowDouble &grad)
{
//--- create variables
int nin=0;
int nout=0;
int wcount=0;
int npoints=0;
int subset0=0;
int subset1=0;
int subsettype=0;
e=0;
//--- check
if(!CAp::Assert(setsize>=0,__FUNCTION__": SetSize<0"))
return;
if(!CAp::Assert(subsetsize<=idx.Size(),__FUNCTION__": SubsetSize>Length(Idx)"))
return;
if(!CAp::Assert(CSparse::SparseIsCRS(xy),__FUNCTION__": sparse matrix XY must be in CRS format."))
return;
npoints=setsize;
if(subsetsize<0)
{
subset0=0;
subset1=setsize;
subsettype=0;
}
else
{
subset0=0;
subset1=subsetsize;
subsettype=1;
for(int i=0; i<subsetsize; i++)
{
if(!CAp::Assert(idx[i]>=0,__FUNCTION__": incorrect index of XY row(Idx[I]<0)"))
return;
if(!CAp::Assert(idx[i]<npoints,__FUNCTION__": incorrect index of XY row(Idx[I]>Rows(XY)-1)"))
return;
}
}
MLPProperties(network,nin,nout,wcount);
grad=vector<double>::Zeros(wcount);
MLPGradBatchX(network,network.m_dummydxy,xy,setsize,1,idx,subset0,subset1,subsettype,e,grad);
}
//+------------------------------------------------------------------+
//| Internal function which actually calculates batch gradient for a |
//| subset or full dataset, which can be represented in different |
//| formats. |
//| THIS FUNCTION IS NOT INTENDED TO BE USED BY ALGLIB USERS! |
//+------------------------------------------------------------------+
void CMLPBase::MLPGradBatchX(CMultilayerPerceptron &network,
CMatrixDouble &densexy,
CSparseMatrix &sparsexy,
int datasetsize,
int datasettype,
CRowInt &idx,
int subset0,
int subset1,
int subsettype,
double &e,
CRowDouble &grad)
{
//--- create variables
int nin=0;
int nout=0;
int wcount=0;
int rowsize=0;
int srcidx=0;
int cstart=0;
int csize=0;
double problemcost=0;
CMLPBuffers buf2;
int len0=0;
int len1=0;
CMLPBuffers pbuf;
CSMLPGrad sgrad;
//--- check
if(!CAp::Assert(datasetsize>=0,__FUNCTION__": SetSize<0"))
return;
if(!CAp::Assert(datasettype==0 || datasettype==1,__FUNCTION__": DatasetType is incorrect"))
return;
if(!CAp::Assert(subsettype==0 || subsettype==1,__FUNCTION__": SubsetType is incorrect"))
return;
//--- Determine network and dataset properties
MLPProperties(network,nin,nout,wcount);
if(MLPIsSoftMax(network))
rowsize=nin+1;
else
rowsize=nin+nout;
//--- Chunked processing
CHPCCores::HPCPrepareChunkedGradient(network.m_weights,wcount,MLPNTotal(network),nin,nout,pbuf);
cstart=subset0;
while(cstart<subset1)
{
//--- Determine size of current chunk and copy it to PBuf.XY
csize=MathMin(subset1,cstart+pbuf.m_ChunkSize)-cstart;
for(int j=0; j<csize; j++)
{
srcidx=-1;
if(subsettype==0)
srcidx=cstart+j;
if(subsettype==1)
srcidx=idx[cstart+j];
if(!CAp::Assert(srcidx>=0,__FUNCTION__": internal error"))
return;
if(datasettype==0)
pbuf.m_XY.Row(j,densexy,srcidx);
if(datasettype==1)
{
CSparse::SparseGetRow(sparsexy,srcidx,pbuf.m_XYRow);
pbuf.m_XY.Row(j,pbuf.m_XYRow);
}
}
pbuf.m_XY.Resize(csize,rowsize);
//--- Process chunk and advance line pointer
MLPChunkedGradient(network,pbuf.m_XY,0,csize,pbuf.m_Batch4Buf,pbuf.m_HPCBuf,e,false);
cstart=cstart+pbuf.m_ChunkSize;
}
CHPCCores::HPCFinalizeChunkedGradient(pbuf,grad);
}
//+------------------------------------------------------------------+
//| Batch gradient calculation for a set of inputs/outputs |
//| (natural error function is used) |
//| INPUT PARAMETERS: |
//| Network - network initialized with one of the network |
//| creation funcs |
//| XY - set of inputs/outputs; one sample=one row; |
//| first NIn columns contain inputs, |
//| next NOut columns - desired outputs. |
//| SSize - number of elements in XY |
//| Grad - possibly preallocated array. If size of array is |
//| smaller than WCount, it will be reallocated. It |
//| is recommended to reuse previously allocated |
//| array to reduce allocation overhead. |
//| OUTPUT PARAMETERS: |
//| E - error function, sum-of-squares for regression |
//| networks, cross-entropy for classification |
//| networks. |
//| Grad - gradient of E with respect to weights of network,|
//| array[WCount] |
//+------------------------------------------------------------------+
void CMLPBase::MLPGradNBatch(CMultilayerPerceptron &network,
CMatrixDouble &xy,const int ssize,
double &e,double &grad[])
{
CRowDouble Grad;
MLPGradNBatch(network,xy,ssize,e,Grad);
Grad.ToArray(grad);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CMLPBase::MLPGradNBatch(CMultilayerPerceptron &network,
CMatrixDouble &xy,const int ssize,
double &e,CRowDouble &grad)
{
//--- create variables
int i=0;
int nin=0;
int nout=0;
int wcount=0;
CMLPBuffers pbuf;
//--- function call
MLPProperties(network,nin,nout,wcount);
CHPCCores::HPCPrepareChunkedGradient(network.m_weights,wcount,MLPNTotal(network),nin,nout,pbuf);
//--- initialization
i=0;
e=0;
grad=vector<double>::Zeros(wcount);
//--- calculation
while(i<ssize)
{
MLPChunkedGradient(network,xy,i,MathMin(ssize,i+pbuf.m_ChunkSize)-i,pbuf.m_Batch4Buf,pbuf.m_HPCBuf,e,true);
i+=pbuf.m_ChunkSize;
}
CHPCCores::HPCFinalizeChunkedGradient(pbuf,grad);
}
//+------------------------------------------------------------------+
//| Batch Hessian calculation (natural error function) using |
//| R-algorithm. Internal subroutine. |
//| Hessian calculation based on R-algorithm described in |
//| "Fast Exact Multiplication by the Hessian", |
//| B. A. Pearlmutter, |
//| Neural Computation, 1994. |
//+------------------------------------------------------------------+
void CMLPBase::MLPHessianNBatch(CMultilayerPerceptron &network,
CMatrixDouble &xy,const int ssize,
double &e,double &grad[],
CMatrixDouble &h)
{
//---
CRowDouble Grad;
//--- initialization
e=0;
//--- function call
MLPHessianBatchInternal(network,xy,ssize,true,e,Grad,h);
//--- return result
Grad.ToArray(grad);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CMLPBase::MLPHessianNBatch(CMultilayerPerceptron &network,
CMatrixDouble &xy,const int ssize,
double &e,CRowDouble &grad,
CMatrixDouble &h)
{
//--- initialization
e=0;
//--- function call
MLPHessianBatchInternal(network,xy,ssize,true,e,grad,h);
}
//+------------------------------------------------------------------+
//| Batch Hessian calculation using R-algorithm. |
//| Internal subroutine. |
//| Hessian calculation based on R-algorithm described in |
//| "Fast Exact Multiplication by the Hessian", |
//| B. A. Pearlmutter, |
//| Neural Computation, 1994. |
//+------------------------------------------------------------------+
void CMLPBase::MLPHessianBatch(CMultilayerPerceptron &network,
CMatrixDouble &xy,const int ssize,
double &e,double &grad[],
CMatrixDouble &h)
{
CRowDouble Grad;
//--- initialization
e=0;
//--- function call
MLPHessianBatchInternal(network,xy,ssize,false,e,Grad,h);
//--- return result
Grad.ToArray(grad);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CMLPBase::MLPHessianBatch(CMultilayerPerceptron &network,
CMatrixDouble &xy,const int ssize,
double &e,CRowDouble &grad,
CMatrixDouble &h)
{
//--- initialization
e=0;
//--- function call
MLPHessianBatchInternal(network,xy,ssize,false,e,grad,h);
}
//+------------------------------------------------------------------+
//| Internal subroutine, shouldn't be called by user. |
//+------------------------------------------------------------------+
void CMLPBase::MLPInternalProcessVector(int &structinfo[],double &weights[],
double &columnmeans[],
double &columnsigmas[],
double &neurons[],
double &dfdnet[],
double &x[],double &y[])
{
//--- create variables
CRowInt StructInfo=structinfo;
CRowDouble Weights=weights;
CRowDouble ColumnMeans=columnmeans;
CRowDouble ColumnSigmas=columnsigmas;
CRowDouble X=x;
CRowDouble Neurons;
CRowDouble dFdnet;
CRowDouble Y;
//--- function call
MLPInternalProcessVector(StructInfo,Weights,ColumnMeans,ColumnSigmas,Neurons,dFdnet,X,Y);
//--- return result
Y.ToArray(y);
Neurons.ToArray(neurons);
dFdnet.ToArray(dfdnet);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CMLPBase::MLPInternalProcessVector(CRowInt &structinfo,CRowDouble &weights,
CRowDouble &columnmeans,
CRowDouble &columnsigmas,
CRowDouble &neurons,
CRowDouble &dfdnet,
CRowDouble &x,CRowDouble &y)
{
//--- create variables
int i=0;
int n1=0;
int n2=0;
int w1=0;
int w2=0;
int ntotal=0;
int nin=0;
int nout=0;
int istart=0;
int offs=0;
double net=0;
double f=0;
double df=0;
double d2f=0;
double mx=0;
bool perr;
int i_=0;
int i1_=0;
//--- Read network geometry
nin=structinfo[1];
nout=structinfo[2];
ntotal=structinfo[3];
istart=structinfo[5];
//--- Inputs standartisation and putting in the network
for(i=0; i<nin; i++)
{
//--- check
if(columnsigmas[i]!=0.0)
neurons.Set(i,(x[i]-columnmeans[i])/columnsigmas[i]);
else
neurons.Set(i,x[i]-columnmeans[i]);
}
//--- Allocate
neurons.Resize(ntotal);
dfdnet.Resize(ntotal);
y.Resize(nout);
//--- Process network
for(i=0; i<ntotal; i++)
{
offs=istart+i*m_nfieldwidth;
//--- check
if(structinfo[offs]>0 || structinfo[offs]==-5)
{
//--- Activation function
MLPActivationFunction(neurons[structinfo[offs+2]],structinfo[offs],f,df,d2f);
//--- change values
neurons.Set(i,f);
dfdnet.Set(i,df);
continue;
}
//--- check
if(structinfo[offs]==0)
{
//--- Adaptive summator
n1=structinfo[offs+2];
w1=structinfo[offs+3];
w2=w1+structinfo[offs+1];
i1_=(n1)-(w1);
net=0.0;
//--- calculation
for(i_=w1; i_<w2; i_++)
net+=weights[i_]*neurons[i_+i1_];
neurons.Set(i,net);
dfdnet.Set(i,1.0);
continue;
}
//--- check
if(structinfo[offs]<0)
{
perr=true;
//--- check
switch(structinfo[offs])
{
case -2:
//--- input neuron,left unchanged
perr=false;
break;
//--- check
case -3:
//--- "-1" neuron
neurons.Set(i,-1);
perr=false;
break;
//--- check
case -4:
//--- "0" neuron
neurons.Set(i,0);
perr=false;
break;
}
//--- check
if(!CAp::Assert(!perr,__FUNCTION__+": internal error - unknown neuron type!"))
return;
continue;
}
}
//--- Extract result
i1_=ntotal-nout;
for(i_=0; i_<nout; i_++)
y.Set(i_,neurons[i_+i1_]);
//--- Softmax post-processing or standardisation if needed
if(!CAp::Assert(structinfo[6]==0 || structinfo[6]==1,__FUNCTION__+": unknown normalization type!"))
return;
//--- check
if(structinfo[6]==1)
{
//--- Softmax
mx=y.Max();
//--- calculation
y=MathExp(y-mx+0);
net=y.Sum();
y/= net;
}
else
{
//--- Standardisation
for(i=0; i<nout; i++)
y.Set(i,y[i]*columnsigmas[nin+i]+columnmeans[nin+i]);
}
}
//+------------------------------------------------------------------+
//| Serializer: allocation |
//+------------------------------------------------------------------+
void CMLPBase::MLPAlloc(CSerializer &s,CMultilayerPerceptron &network)
{
//--- create variables
int i=0;
int j=0;
int k=0;
int fkind=0;
double threshold=0;
double v0=0;
double v1=0;
int nin=0;
int nout=0;
//--- initialization
nin=network.m_hllayersizes[0];
nout=network.m_hllayersizes[network.m_hllayersizes.Size()-1];
//--- preparation to serialize
s.Alloc_Entry();
s.Alloc_Entry();
s.Alloc_Entry();
//--- function call
CApServ::AllocIntegerArray(s,network.m_hllayersizes,-1);
for(i=1; i<network.m_hllayersizes.Size(); i++)
{
for(j=0; j<network.m_hllayersizes[i]; j++)
{
//--- function call
MLPGetNeuronInfo(network,i,j,fkind,threshold);
//--- preparation to serialize
s.Alloc_Entry();
s.Alloc_Entry();
for(k=0; k<network.m_hllayersizes[i-1]; k++)
s.Alloc_Entry();
}
}
for(j=0; j<nin; j++)
{
//--- function call
MLPGetInputScaling(network,j,v0,v1);
//--- preparation to serialize
s.Alloc_Entry();
s.Alloc_Entry();
}
for(j=0; j<nout; j++)
{
//--- function call
MLPGetOutputScaling(network,j,v0,v1);
//--- preparation to serialize
s.Alloc_Entry();
s.Alloc_Entry();
}
}
//+------------------------------------------------------------------+
//| Serializer: serialization |
//+------------------------------------------------------------------+
void CMLPBase::MLPSerialize(CSerializer &s,CMultilayerPerceptron &network)
{
//--- create variables
int i=0;
int j=0;
int k=0;
int fkind=0;
double threshold=0;
double v0=0;
double v1=0;
int nin=0;
int nout=0;
//--- change values
nin=network.m_hllayersizes[0];
nout=network.m_hllayersizes[network.m_hllayersizes.Size()-1];
//--- serializetion
s.Serialize_Int(CSCodes::GetMLPSerializationCode());
s.Serialize_Int(m_mlpfirstversion);
s.Serialize_Bool(MLPIsSoftMax(network));
//--- function call
CApServ::SerializeIntegerArray(s,network.m_hllayersizes,-1);
for(i=1; i<CAp::Len(network.m_hllayersizes); i++)
{
for(j=0; j<network.m_hllayersizes[i]; j++)
{
//--- function call
MLPGetNeuronInfo(network,i,j,fkind,threshold);
//--- serializetion
s.Serialize_Int(fkind);
s.Serialize_Double(threshold);
for(k=0; k<network.m_hllayersizes[i-1]; k++)
s.Serialize_Double(MLPGetWeight(network,i-1,k,i,j));
}
}
for(j=0; j<nin; j++)
{
//--- function call
MLPGetInputScaling(network,j,v0,v1);
//--- serializetion
s.Serialize_Double(v0);
s.Serialize_Double(v1);
}
for(j=0; j<nout; j++)
{
//--- function call
MLPGetOutputScaling(network,j,v0,v1);
//--- serializetion
s.Serialize_Double(v0);
s.Serialize_Double(v1);
}
}
//+------------------------------------------------------------------+
//| Serializer: unserialization |
//+------------------------------------------------------------------+
void CMLPBase::MLPUnserialize(CSerializer &s,CMultilayerPerceptron &network)
{
//--- create variables
int i0=0;
int i1=0;
int i=0;
int j=0;
int k=0;
int fkind=0;
double threshold=0;
double v0=0;
double v1=0;
int nin=0;
int nout=0;
bool issoftmax;
//--- create array
CRowInt layersizes;
//--- check correctness of header
i0=s.Unserialize_Int();
//--- check
if(!CAp::Assert(i0==CSCodes::GetMLPSerializationCode(),__FUNCTION__+": stream header corrupted"))
return;
//--- unserializetion
i1=s.Unserialize_Int();
//--- check
if(!CAp::Assert(i1==m_mlpfirstversion,__FUNCTION__+": stream header corrupted"))
return;
//--- Create network
issoftmax=s.Unserialize_Bool();
//--- function call
CApServ::UnserializeIntegerArray(s,layersizes);
//--- change values
nin=layersizes[0];
nout=layersizes[layersizes.Size()-1];
//--- check
switch(layersizes.Size())
{
case 2:
//--- check
if(issoftmax)
MLPCreateC0(layersizes[0],layersizes[1],network);
else
MLPCreate0(layersizes[0],layersizes[1],network);
break;
case 3:
//--- check
if(issoftmax)
MLPCreateC1(layersizes[0],layersizes[1],layersizes[2],network);
else
MLPCreate1(layersizes[0],layersizes[1],layersizes[2],network);
break;
case 4:
//--- check
if(issoftmax)
MLPCreateC2(layersizes[0],layersizes[1],layersizes[2],layersizes[3],network);
else
MLPCreate2(layersizes[0],layersizes[1],layersizes[2],layersizes[3],network);
break;
default:
//--- check
if(!CAp::Assert(layersizes.Size()==2 || layersizes.Size()==3 || layersizes.Size()==4,__FUNCTION__+": too many hidden layers!"))
return;
}
//--- Load neurons and weights
for(i=1; i<layersizes.Size(); i++)
{
for(j=0; j<layersizes[i]; j++)
{
//--- unserializetion
fkind=s.Unserialize_Int();
threshold=s.Unserialize_Double();
//--- function call
MLPSetNeuronInfo(network,i,j,fkind,threshold);
//--- unserializetion
for(k=0; k<layersizes[i-1]; k++)
{
v0=s.Unserialize_Double();
//--- function call
MLPSetWeight(network,i-1,k,i,j,v0);
}
}
}
//
//--- Load standartizator
//
for(j=0; j<nin; j++)
{
//--- unserializetion
v0=s.Unserialize_Double();
v1=s.Unserialize_Double();
//--- function call
MLPSetInputScaling(network,j,v0,v1);
}
for(j=0; j<nout; j++)
{
//--- unserializetion
v0=s.Unserialize_Double();
v1=s.Unserialize_Double();
//--- function call
MLPSetOutputScaling(network,j,v0,v1);
}
}
//+------------------------------------------------------------------+
//| Calculation of all types of errors on subset of dataset. |
//| INPUT PARAMETERS: |
//| Network - network initialized with one of the network |
//| creation funcs |
//| XY - original dataset; one sample = one row; |
//| first NIn columns contain inputs, next NOut |
//| columns - desired outputs. |
//| SetSize - real size of XY, SetSize>=0; |
//| Subset - subset of SubsetSize elements, array[SubsetSize]; |
//| SubsetSize- number of elements in Subset[] array: |
//| * if SubsetSize>0, rows of XY with indices |
//| Subset[0]......Subset[SubsetSize-1] are processed|
//| * if SubsetSize=0, zeros are returned |
//| * if SubsetSize<0, entire dataset is processed; |
//| Subset[] array is ignored in this case. |
//| OUTPUT PARAMETERS: |
//| Rep - it contains all type of errors. |
//+------------------------------------------------------------------+
void CMLPBase::MLPAllErrorsSubset(CMultilayerPerceptron &network,
CMatrixDouble &xy,int setsize,
CRowInt &subset,int subsetsize,
CModelErrors &rep)
{
//--- create variables
int idx0=0;
int idx1=0;
int idxtype=0;
//--- check
if(!CAp::Assert(xy.Rows()>=setsize,__FUNCTION__": XY has less than SetSize rows"))
return;
if(setsize>0)
{
if(MLPIsSoftMax(network))
{
if(!CAp::Assert(xy.Cols()>MLPGetInputsCount(network),__FUNCTION__": XY has less than NIn+1 columns"))
return;
}
else
if(!CAp::Assert(xy.Cols()>=MLPGetInputsCount(network)+MLPGetOutputsCount(network),__FUNCTION__": XY has less than NIn+NOut columns"))
return;
}
if(subsetsize>=0)
{
idx0=0;
idx1=subsetsize;
idxtype=1;
}
else
{
idx0=0;
idx1=setsize;
idxtype=0;
}
MLPAllErrorsX(network,xy,network.m_dummysxy,setsize,0,subset,idx0,idx1,idxtype,rep);
}
//+------------------------------------------------------------------+
//| Calculation of all types of errors on subset of dataset. |
//| INPUT PARAMETERS: |
//| Network - network initialized with one of the network |
//| creation funcs |
//| XY - original dataset given by sparse matrix; |
//| one sample = one row; first NIn columns contain |
//| inputs, next NOut columns - desired outputs. |
//| SetSize - real size of XY, SetSize>=0; |
//| Subset - subset of SubsetSize elements, array[SubsetSize]; |
//| SubsetSize- number of elements in Subset[] array: |
//| * if SubsetSize>0, rows of XY with indices |
//| Subset[0]......Subset[SubsetSize-1] are processed|
//| * if SubsetSize=0, zeros are returned |
//| * if SubsetSize<0, entire dataset is processed; |
//| Subset[] array is ignored in this case. |
//| OUTPUT PARAMETERS: |
//| Rep - it contains all type of errors. |
//+------------------------------------------------------------------+
void CMLPBase::MLPAllErrorsSparseSubset(CMultilayerPerceptron &network,
CSparseMatrix &xy,int setsize,
CRowInt &subset,int subsetsize,
CModelErrors &rep)
{
//--- create variables
int idx0=0;
int idx1=0;
int idxtype=0;
//--- check
if(!CAp::Assert(CSparse::SparseIsCRS(xy),__FUNCTION__": XY is not in CRS format."))
return;
if(!CAp::Assert(CSparse::SparseGetNRows(xy)>=setsize,__FUNCTION__": XY has less than SetSize rows"))
return;
if(setsize>0)
{
if(MLPIsSoftMax(network))
{
if(!CAp::Assert(CSparse::SparseGetNCols(xy)>MLPGetInputsCount(network),__FUNCTION__": XY has less than NIn+1 columns"))
return;
}
else
if(!CAp::Assert(CSparse::SparseGetNCols(xy)>=MLPGetInputsCount(network)+MLPGetOutputsCount(network),__FUNCTION__": XY has less than NIn+NOut columns"))
return;
}
if(subsetsize>=0)
{
idx0=0;
idx1=subsetsize;
idxtype=1;
}
else
{
idx0=0;
idx1=setsize;
idxtype=0;
}
MLPAllErrorsX(network,network.m_dummydxy,xy,setsize,1,subset,idx0,idx1,idxtype,rep);
}
//+------------------------------------------------------------------+
//| Error of the neural network on subset of dataset. |
//| INPUT PARAMETERS: |
//| Network - neural network; |
//| XY - training set, see below for information on the |
//| training set format; |
//| SetSize - real size of XY, SetSize>=0; |
//| Subset - subset of SubsetSize elements, array[SubsetSize]; |
//| SubsetSize- number of elements in Subset[] array: |
//| * if SubsetSize>0, rows of XY with indices |
//| Subset[0]......Subset[SubsetSize-1] are processed|
//| * if SubsetSize=0, zeros are returned |
//| * if SubsetSize<0, entire dataset is processed; |
//| Subset[] array is ignored in this case. |
//| RESULT: |
//| sum-of-squares error, SUM(sqr(y[i]-desired_y[i])/2) |
//| DATASET FORMAT: |
//| This function uses two different dataset formats - one for |
//| regression networks, another one for classification networks. |
//| For regression networks with NIn inputs and NOut outputs |
//| following dataset format is used: |
//| * dataset is given by NPoints*(NIn+NOut) matrix |
//| * each row corresponds to one example |
//| * first NIn columns are inputs, next NOut columns are |
//| outputs |
//| For classification networks with NIn inputs and NClasses clases|
//| following dataset format is used: |
//| * dataset is given by NPoints*(NIn+1) matrix |
//| * each row corresponds to one example |
//| * first NIn columns are inputs, last column stores class |
//| number (from 0 to NClasses-1). |
//+------------------------------------------------------------------+
double CMLPBase::MLPErrorSubset(CMultilayerPerceptron &network,
CMatrixDouble &xy,int setsize,
CRowInt &subset,int subsetsize)
{
//--- create variables
double result=0;
int idx0=0;
int idx1=0;
int idxtype=0;
//--- check
if(!CAp::Assert(xy.Rows()>=setsize,__FUNCTION__": XY has less than SetSize rows"))
return(0);
if(setsize>0)
{
if(MLPIsSoftMax(network))
{
if(!CAp::Assert(xy.Cols()>MLPGetInputsCount(network),__FUNCTION__": XY has less than NIn+1 columns"))
return(0);
}
else
if(!CAp::Assert(xy.Cols()>=MLPGetInputsCount(network)+MLPGetOutputsCount(network),__FUNCTION__": XY has less than NIn+NOut columns"))
return(0);
}
if(subsetsize>=0)
{
idx0=0;
idx1=subsetsize;
idxtype=1;
}
else
{
idx0=0;
idx1=setsize;
idxtype=0;
}
//--- function call
MLPAllErrorsX(network,xy,network.m_dummysxy,setsize,0,subset,idx0,idx1,idxtype,network.m_err);
result=CMath::Sqr(network.m_err.m_RMSError)*(idx1-idx0)*MLPGetOutputsCount(network)/2;
//--- return result
return(result);
}
//+------------------------------------------------------------------+
//| Error of the neural network on subset of sparse dataset. |
//| INPUT PARAMETERS: |
//| Network - neural network; |
//| XY - training set, see below for information on the |
//| training set format. This function checks |
//| correctness of the dataset (no NANs/INFs, class |
//| numbers are correct) and throws exception when |
//| incorrect dataset is passed. Sparse matrix must use|
//| CRS format for storage. |
//| SetSize - real size of XY, SetSize>=0; it is used when |
//| SubsetSize<0; |
//| Subset - subset of SubsetSize elements, array[SubsetSize]; |
//| SubsetSize- number of elements in Subset[] array: |
//| * if SubsetSize>0, rows of XY with indices |
//| Subset[0]......Subset[SubsetSize-1] are processed|
//| * if SubsetSize=0, zeros are returned |
//| * if SubsetSize<0, entire dataset is processed; |
//| Subset[] array is ignored in this case. |
//| RESULT: |
//| sum-of-squares error, SUM(sqr(y[i]-desired_y[i])/2) |
//| DATASET FORMAT: |
//| This function uses two different dataset formats - one for |
//| regression networks, another one for classification networks. |
//| For regression networks with NIn inputs and NOut outputs |
//| following dataset format is used: |
//| * dataset is given by NPoints*(NIn+NOut) matrix |
//| * each row corresponds to one example |
//| * first NIn columns are inputs, next NOut columns are |
//| outputs |
//| For classification networks with NIn inputs and NClasses clases|
//| following dataset format is used: |
//| * dataset is given by NPoints*(NIn+1) matrix |
//| * each row corresponds to one example |
//| * first NIn columns are inputs, last column stores class |
//| number (from 0 to NClasses-1). |
//+------------------------------------------------------------------+
double CMLPBase::MLPErrorSparseSubset(CMultilayerPerceptron &network,
CSparseMatrix &xy,int setsize,
CRowInt &subset,int subsetsize)
{
//--- create variables
double result=0;
int idx0=0;
int idx1=0;
int idxtype=0;
//--- check
if(!CAp::Assert(CSparse::SparseIsCRS(xy),__FUNCTION__": XY is not in CRS format."))
return(0);
if(!CAp::Assert(CSparse::SparseGetNRows(xy)>=setsize,__FUNCTION__": XY has less than SetSize rows"))
return(0);
if(setsize>0)
{
if(MLPIsSoftMax(network))
{
if(!CAp::Assert(CSparse::SparseGetNCols(xy)>=MLPGetInputsCount(network)+1,__FUNCTION__": XY has less than NIn+1 columns"))
return(0);
}
else
if(!CAp::Assert(CSparse::SparseGetNCols(xy)>=MLPGetInputsCount(network)+MLPGetOutputsCount(network),__FUNCTION__": XY has less than NIn+NOut columns"))
return(0);
}
if(subsetsize>=0)
{
idx0=0;
idx1=subsetsize;
idxtype=1;
}
else
{
idx0=0;
idx1=setsize;
idxtype=0;
}
//--- function call
MLPAllErrorsX(network,network.m_dummydxy,xy,setsize,1,subset,idx0,idx1,idxtype,network.m_err);
result=CMath::Sqr(network.m_err.m_RMSError)*(idx1-idx0)*MLPGetOutputsCount(network)/2;
//--- return result
return(result);
}
//+------------------------------------------------------------------+
//| Calculation of all types of errors at once for a subset or full |
//| dataset, which can be represented in different formats. |
//| THIS INTERNAL FUNCTION IS NOT INTENDED TO BE USED BY ALGLIB USERS|
//+------------------------------------------------------------------+
void CMLPBase::MLPAllErrorsX(CMultilayerPerceptron &network,
CMatrixDouble &densexy,
CSparseMatrix &sparsexy,
int datasetsize,
int datasettype,
CRowInt &idx,
int subset0,
int subset1,
int subsettype,
CModelErrors &rep)
{
//--- create variables
int nin=0;
int nout=0;
int wcount=0;
int rowsize=0;
bool iscls;
int srcidx=0;
int cstart=0;
int csize=0;
int len0=0;
int len1=0;
double problemcost=0;
CMLPBuffers pbuf;
CModelErrors rep0;
CModelErrors rep1;
//--- check
if(!CAp::Assert(datasetsize>=0,__FUNCTION__": SetSize<0"))
return;
if(!CAp::Assert(datasettype==0 || datasettype==1,__FUNCTION__": DatasetType is incorrect"))
return;
if(!CAp::Assert(subsettype==0 || subsettype==1,__FUNCTION__": SubsetType is incorrect"))
return;
//--- Determine network properties
MLPProperties(network,nin,nout,wcount);
iscls=MLPIsSoftMax(network);
//--- Retrieve and prepare
if(iscls)
{
rowsize=nin+1;
CBdSS::DSErrAllocate(nout,pbuf.m_Tmp0);
}
else
{
rowsize=nin+nout;
CBdSS::DSErrAllocate(-nout,pbuf.m_Tmp0);
}
//--- Processing
CHPCCores::HPCPrepareChunkedGradient(network.m_weights,wcount,MLPNTotal(network),nin,nout,pbuf);
cstart=subset0;
while(cstart<subset1)
{
//--- Determine size of current chunk and copy it to PBuf.XY
csize=MathMin(subset1,cstart+pbuf.m_ChunkSize)-cstart;
for(int j=0; j<csize; j++)
{
srcidx=-1;
if(subsettype==0)
srcidx=cstart+j;
else
if(subsettype==1)
srcidx=idx[cstart+j];
if(!CAp::Assert(srcidx>=0,__FUNCTION__": internal error"))
return;
if(datasettype==0)
pbuf.m_XY.Row(j,densexy,srcidx);
else
if(datasettype==1)
{
CSparse::SparseGetRow(sparsexy,srcidx,pbuf.m_XYRow);
pbuf.m_XY.Row(j,pbuf.m_XYRow);
}
}
//--- Unpack XY and process (temporary code, to be replaced by chunked processing)
pbuf.m_XY2=pbuf.m_XY;
MLPChunkedProcess(network,pbuf.m_XY2,0,csize,pbuf.m_Batch4Buf,pbuf.m_HPCBuf);
ulong parts[]={nin,nout};
matrix<double> splitted[];
matrix<double> splitted2[];
//--- Split
pbuf.m_XY2.Split(parts,1,splitted2);
if(iscls)
parts[1]=1;
pbuf.m_XY.Split(parts,1,splitted);
for(int j=0; j<csize; j++)
{
pbuf.m_X=splitted2[0].Row(j);
pbuf.m_Y=splitted2[1].Row(j);
pbuf.m_Desiredy=splitted[1].Row(j);
CBdSS::DSErrAccumulate(pbuf.m_Tmp0,pbuf.m_Y,pbuf.m_Desiredy);
}
//--- Process chunk and advance line pointer
cstart+=pbuf.m_ChunkSize;
}
CBdSS::DSErrFinish(pbuf.m_Tmp0);
rep.m_RelCLSError=pbuf.m_Tmp0[0];
rep.m_AvgCE=pbuf.m_Tmp0[1]/MathLog(2);
rep.m_RMSError=pbuf.m_Tmp0[2];
rep.m_AvgError=pbuf.m_Tmp0[3];
rep.m_AvgRelError=pbuf.m_Tmp0[4];
}
//+------------------------------------------------------------------+
//| Internal subroutine: adding new input layer to network |
//+------------------------------------------------------------------+
void CMLPBase::AddInputLayer(const int ncount,CRowInt &lsizes,
CRowInt &ltypes,CRowInt &lconnfirst,
CRowInt &lconnlast,int &lastproc)
{
//--- change values
lsizes.Set(0,ncount);
ltypes.Set(0,-2);
lconnfirst.Set(0,0);
lconnlast.Set(0,0);
lastproc=0;
}
//+------------------------------------------------------------------+
//| Internal subroutine: adding new summator layer to network |
//+------------------------------------------------------------------+
void CMLPBase::AddBiasedSummatorLayer(const int ncount,CRowInt &lsizes,
CRowInt &ltypes,CRowInt &lconnfirst,
CRowInt &lconnlast,int &lastproc)
{
//--- change values
lsizes.Set(lastproc+1,1);
ltypes.Set(lastproc+1,-3);
lconnfirst.Set(lastproc+1,0);
lconnlast.Set(lastproc+1,0);
lsizes.Set(lastproc+2,ncount);
ltypes.Set(lastproc+2,0);
lconnfirst.Set(lastproc+2,lastproc);
lconnlast.Set(lastproc+2,lastproc+1);
lastproc=lastproc+2;
}
//+------------------------------------------------------------------+
//| Internal subroutine: adding new summator layer to network |
//+------------------------------------------------------------------+
void CMLPBase::AddActivationLayer(const int functype,CRowInt &lsizes,
CRowInt &ltypes,CRowInt &lconnfirst,
CRowInt &lconnlast,int &lastproc)
{
//--- check
if(!CAp::Assert(functype>0 || functype==-5,__FUNCTION__+": incorrect function type"))
return;
//--- change values
lsizes.Set(lastproc+1,lsizes[lastproc]);
ltypes.Set(lastproc+1,functype);
lconnfirst.Set(lastproc+1,lastproc);
lconnlast.Set(lastproc+1,lastproc);
lastproc=lastproc+1;
}
//+------------------------------------------------------------------+
//| Internal subroutine: adding new zero layer to network |
//+------------------------------------------------------------------+
void CMLPBase::AddZeroLayer(CRowInt &lsizes,CRowInt &ltypes,
CRowInt &lconnfirst,CRowInt &lconnlast,
int &lastproc)
{
//--- change values
lsizes.Set(lastproc+1,1);
ltypes.Set(lastproc+1,-4);
lconnfirst.Set(lastproc+1,0);
lconnlast.Set(lastproc+1,0);
lastproc=lastproc+1;
}
//+------------------------------------------------------------------+
//| This routine adds input layer to the high-level description of |
//| the network. |
//| It modifies Network.HLConnections and Network.HLNeurons and |
//| assumes that these arrays have enough place to store data. |
//| It accepts following parameters: |
//| Network - network |
//| ConnIdx - index of the first free entry in the |
//| HLConnections |
//| NeuroIdx - index of the first free entry in the |
//| HLNeurons |
//| StructInfoIdx- index of the first entry in the low level |
//| description of the current layer (in the |
//| StructInfo array) |
//| NIn - number of inputs |
//| It modified Network and indices. |
//+------------------------------------------------------------------+
void CMLPBase::HLAddInputLayer(CMultilayerPerceptron &network,
int &connidx,int &neuroidx,
int &structinfoidx,int nin)
{
//--- initialization
int offs=m_hlm_nfieldwidth*neuroidx;
//--- change values
for(int i=0; i<nin; i++)
{
network.m_hlneurons.Set(offs,0);
network.m_hlneurons.Set(offs+1,i);
network.m_hlneurons.Set(offs+2,-1);
network.m_hlneurons.Set(offs+3,-1);
offs=offs+m_hlm_nfieldwidth;
}
//--- change values
neuroidx=neuroidx+nin;
structinfoidx=structinfoidx+nin;
}
//+------------------------------------------------------------------+
//| This routine adds output layer to the high-level description of |
//| the network. |
//| It modifies Network.HLConnections and Network. HLNeurons and |
//| assumes that these arrays have enough place to store data. It |
//| accepts following parameters: |
//| Network - network |
//| ConnIdx - index of the first free entry in the |
//| HLConnections |
//| NeuroIdx - index of the first free entry in the |
//| HLNeurons |
//| StructInfoIdx- index of the first entry in the low level |
//| description of the current layer (in the |
//| StructInfo array) |
//| WeightsIdx - index of the first entry in the Weights |
//| array which corresponds to the current layer |
//| K - current layer index |
//| NPrev - number of neurons in the previous layer |
//| NOut - number of outputs |
//| IsCls - is it classifier network? |
//| IsLinear - is it network with linear output? |
//| It modified Network and ConnIdx/NeuroIdx/StructInfoIdx/WeightsIdx|
//+------------------------------------------------------------------+
void CMLPBase::HLAddOutputLayer(CMultilayerPerceptron &network,
int &connidx,int &neuroidx,
int &structinfoidx,int &weightsidx,
const int k,const int nprev,
const int nout,const bool iscls,
const bool islinearout)
{
//--- create variables
int i=0;
int j=0;
int neurooffs=0;
int connoffs=0;
//--- check
if(!CAp::Assert((iscls && islinearout) || !iscls,__FUNCTION__+": internal error"))
return;
//--- initialization
neurooffs=m_hlm_nfieldwidth*neuroidx;
connoffs=m_hlconm_nfieldwidth*connidx;
//--- check
if(!iscls)
{
//--- Regression network
for(i=0; i<nout; i++)
{
//--- change values
network.m_hlneurons.Set(neurooffs,k);
network.m_hlneurons.Set(neurooffs+1,i);
network.m_hlneurons.Set(neurooffs+2,structinfoidx+1+nout+i);
network.m_hlneurons.Set(neurooffs+3,weightsidx+nprev+(nprev+1)*i);
neurooffs=neurooffs+m_hlm_nfieldwidth;
}
for(i=0; i<nprev; i++)
{
for(j=0; j<nout; j++)
{
//--- change values
network.m_hlconnections.Set(connoffs,k-1);
network.m_hlconnections.Set(connoffs+1,i);
network.m_hlconnections.Set(connoffs+2,k);
network.m_hlconnections.Set(connoffs+3,j);
network.m_hlconnections.Set(connoffs+4,weightsidx+i+j*(nprev+1));
connoffs=connoffs+m_hlconm_nfieldwidth;
}
}
//--- change values
connidx=connidx+nprev*nout;
neuroidx=neuroidx+nout;
structinfoidx=structinfoidx+2*nout+1;
weightsidx=weightsidx+nout*(nprev+1);
}
else
{
//--- Classification network
for(i=0; i<nout-1; i++)
{
//--- change values
network.m_hlneurons.Set(neurooffs,k);
network.m_hlneurons.Set(neurooffs+1,i);
network.m_hlneurons.Set(neurooffs+2,-1);
network.m_hlneurons.Set(neurooffs+3,weightsidx+nprev+(nprev+1)*i);
neurooffs=neurooffs+m_hlm_nfieldwidth;
}
//--- change values
network.m_hlneurons.Set(neurooffs,k);
network.m_hlneurons.Set(neurooffs+1,i);
network.m_hlneurons.Set(neurooffs+2,-1);
network.m_hlneurons.Set(neurooffs+3,-1);
for(i=0; i<nprev; i++)
{
for(j=0; j<nout-1; j++)
{
//--- change values
network.m_hlconnections.Set(connoffs,k-1);
network.m_hlconnections.Set(connoffs+1,i);
network.m_hlconnections.Set(connoffs+2,k);
network.m_hlconnections.Set(connoffs+3,j);
network.m_hlconnections.Set(connoffs+4,weightsidx+i+j*(nprev+1));
connoffs=connoffs+m_hlconm_nfieldwidth;
}
}
//--- change values
connidx=connidx+nprev*(nout-1);
neuroidx=neuroidx+nout;
structinfoidx=structinfoidx+nout+2;
weightsidx=weightsidx+(nout-1)*(nprev+1);
}
}
//+------------------------------------------------------------------+
//| This routine adds hidden layer to the high-level description of |
//| the network. |
//| It modifies Network.HLConnections and Network.HLNeurons and |
//| assumes that these arrays have enough place to store data. It |
//| accepts following parameters: |
//| Network - network |
//| ConnIdx - index of the first free entry in the |
//| HLConnections |
//| NeuroIdx - index of the first free entry in the |
//| HLNeurons |
//| StructInfoIdx- index of the first entry in the low level |
//| description of the current layer (in the |
//| StructInfo array) |
//| WeightsIdx - index of the first entry in the Weights |
//| array which corresponds to the current layer |
//| K - current layer index |
//| NPrev - number of neurons in the previous layer |
//| NCur - number of neurons in the current layer |
//| It modified Network and ConnIdx/NeuroIdx/StructInfoIdx/WeightsIdx|
//+------------------------------------------------------------------+
void CMLPBase::HLAddHiddenLayer(CMultilayerPerceptron &network,
int &connidx,int &neuroidx,
int &structinfoidx,int &weightsidx,
const int k,const int nprev,
const int ncur)
{
//--- create variables
int neurooffs=m_hlm_nfieldwidth*neuroidx;
int connoffs=m_hlconm_nfieldwidth*connidx;
for(int i=0; i<ncur; i++)
{
//--- change values
network.m_hlneurons.Set(neurooffs,k);
network.m_hlneurons.Set(neurooffs+1,i);
network.m_hlneurons.Set(neurooffs+2,structinfoidx+1+ncur+i);
network.m_hlneurons.Set(neurooffs+3,weightsidx+nprev+(nprev+1)*i);
neurooffs=neurooffs+m_hlm_nfieldwidth;
}
for(int i=0; i<nprev; i++)
{
for(int j=0; j<ncur; j++)
{
//--- change values
network.m_hlconnections.Set(connoffs,k-1);
network.m_hlconnections.Set(connoffs+1,i);
network.m_hlconnections.Set(connoffs+2,k);
network.m_hlconnections.Set(connoffs+3,j);
network.m_hlconnections.Set(connoffs+4,weightsidx+i+j*(nprev+1));
connoffs=connoffs+m_hlconm_nfieldwidth;
}
}
//--- change values
connidx=connidx+nprev*ncur;
neuroidx=neuroidx+ncur;
structinfoidx=structinfoidx+2*ncur+1;
weightsidx=weightsidx+ncur*(nprev+1);
}
//+------------------------------------------------------------------+
//| This function fills high level information about network created |
//| using internal MLPCreate() function. |
//| This function does NOT examine StructInfo for low level |
//| information, it just expects that network has following |
//| structure: |
//| input neuron \ |
//| ... | input layer |
//| input neuron / |
//| "-1" neuron \ |
//| biased summator | |
//| ... | |
//| biased summator | hidden layer(s), if there are |
//| activation function | exists any |
//| ... | |
//| activation function / |
//| "-1" neuron \ |
//| biased summator | output layer: |
//| ... | * we have NOut summators/activators|
//| biased summator | for regression networks |
//| activation function | * we have only NOut-1 summators and|
//| ... | no activators for classifiers |
//| activation function |*we have "0" neuron only when we |
//| "0" neuron / have classifier |
//+------------------------------------------------------------------+
void CMLPBase::FillHighLevelInformation(CMultilayerPerceptron &network,
const int nin,const int nhid1,
const int nhid2,const int nout,
const bool iscls,const bool islinearout)
{
//--- create variables
int idxweights=0;
int idxstruct=0;
int idxneuro=0;
int idxconn=0;
//--- check
if(!CAp::Assert((iscls && islinearout) || !iscls,__FUNCTION__+": internal error"))
return;
//--- Preparations common to all types of networks
network.m_hlnetworktype=0;
//--- network without hidden layers
if(nhid1==0)
{
//--- allocation
network.m_hllayersizes.Resize(2);
//--- change values
network.m_hllayersizes.Set(0,nin);
network.m_hllayersizes.Set(1,nout);
//--- check
if(!iscls)
{
//--- allocation
network.m_hlconnections.Resize(m_hlconm_nfieldwidth*nin*nout);
network.m_hlneurons.Resize(m_hlm_nfieldwidth*(nin+nout));
network.m_hlnormtype=0;
}
else
{
//--- allocation
network.m_hlconnections.Resize(m_hlconm_nfieldwidth*nin*(nout-1));
network.m_hlneurons.Resize(m_hlm_nfieldwidth*(nin+nout));
network.m_hlnormtype=1;
}
//--- function call
HLAddInputLayer(network,idxconn,idxneuro,idxstruct,nin);
//--- function call
HLAddOutputLayer(network,idxconn,idxneuro,idxstruct,idxweights,1,nin,nout,iscls,islinearout);
//--- exit the function
return;
}
//--- network with one hidden layers
if(nhid2==0)
{
//--- allocation
network.m_hllayersizes.Resize(3);
//--- change values
network.m_hllayersizes.Set(0,nin);
network.m_hllayersizes.Set(1,nhid1);
network.m_hllayersizes.Set(2,nout);
//--- check
if(!iscls)
{
//--- allocation
network.m_hlconnections.Resize(m_hlconm_nfieldwidth*(nin*nhid1+nhid1*nout));
network.m_hlneurons.Resize(m_hlm_nfieldwidth*(nin+nhid1+nout));
network.m_hlnormtype=0;
}
else
{
//--- allocation
network.m_hlconnections.Resize(m_hlconm_nfieldwidth*(nin*nhid1+nhid1*(nout-1)));
network.m_hlneurons.Resize(m_hlm_nfieldwidth*(nin+nhid1+nout));
network.m_hlnormtype=1;
}
//--- function call
HLAddInputLayer(network,idxconn,idxneuro,idxstruct,nin);
//--- function call
HLAddHiddenLayer(network,idxconn,idxneuro,idxstruct,idxweights,1,nin,nhid1);
//--- function call
HLAddOutputLayer(network,idxconn,idxneuro,idxstruct,idxweights,2,nhid1,nout,iscls,islinearout);
//--- exit the function
return;
}
//--- Two hidden layers
network.m_hllayersizes.Resize(4);
//--- change values
network.m_hllayersizes.Set(0,nin);
network.m_hllayersizes.Set(1,nhid1);
network.m_hllayersizes.Set(2,nhid2);
network.m_hllayersizes.Set(3,nout);
//--- check
if(!iscls)
{
//--- allocation
network.m_hlconnections.Resize(m_hlconm_nfieldwidth*(nin*nhid1+nhid1*nhid2+nhid2*nout));
network.m_hlneurons.Resize(m_hlm_nfieldwidth*(nin+nhid1+nhid2+nout));
network.m_hlnormtype=0;
}
else
{
//--- allocation
network.m_hlconnections.Resize(m_hlconm_nfieldwidth*(nin*nhid1+nhid1*nhid2+nhid2*(nout-1)));
network.m_hlneurons.Resize(m_hlm_nfieldwidth*(nin+nhid1+nhid2+nout));
network.m_hlnormtype=1;
}
//--- function call
HLAddInputLayer(network,idxconn,idxneuro,idxstruct,nin);
//--- function call
HLAddHiddenLayer(network,idxconn,idxneuro,idxstruct,idxweights,1,nin,nhid1);
//--- function call
HLAddHiddenLayer(network,idxconn,idxneuro,idxstruct,idxweights,2,nhid1,nhid2);
//--- function call
HLAddOutputLayer(network,idxconn,idxneuro,idxstruct,idxweights,3,nhid2,nout,iscls,islinearout);
}
//+------------------------------------------------------------------+
//| Internal subroutine. |
//+------------------------------------------------------------------+
void CMLPBase::MLPCreate(const int nin,const int nout,CRowInt &lsizes,
CRowInt &ltypes,CRowInt &lconnfirst,CRowInt &lconnlast,
const int layerscount,const bool isclsnet,
CMultilayerPerceptron &network)
{
//--- create variables
int i=0;
int j=0;
int ssize=0;
int ntotal=0;
int wcount=0;
int offs=0;
int nprocessed=0;
int wallocated=0;
//--- creating arrays
int localtemp[];
int lnfirst[];
int lnsyn[];
CMLPBuffers buf;
CSMLPGrad sgrad;
//--- Check
if(!CAp::Assert(layerscount>0,__FUNCTION__+": wrong parameters!"))
return;
//--- check
if(!CAp::Assert(ltypes[0]==-2,__FUNCTION__+": wrong LTypes[0] (must be -2)!"))
return;
for(i=0; i<layerscount; i++)
{
//--- check
if(!CAp::Assert(lsizes[i]>0,__FUNCTION__+": wrong LSizes!"))
return;
//--- check
if(!CAp::Assert(lconnfirst[i]>=0 && (lconnfirst[i]<i || i==0),__FUNCTION__+": wrong LConnFirst!"))
return;
//--- check
if(!CAp::Assert(lconnlast[i]>=lconnfirst[i] && (lconnlast[i]<i || i==0),__FUNCTION__+": wrong LConnLast!"))
return;
}
//--- Build network geometry
ArrayResizeAL(lnfirst,layerscount);
ArrayResizeAL(lnsyn,layerscount);
//--- initialization
ntotal=0;
wcount=0;
//--- calculation
for(i=0; i<layerscount; i++)
{
//--- Analyze connections.
//--- This code must throw an assertion in case of unknown LTypes[I]
lnsyn[i]=-1;
//--- check
if(ltypes[i]>=0 || ltypes[i]==-5)
{
lnsyn[i]=0;
for(j=lconnfirst[i]; j<=lconnlast[i]; j++)
lnsyn[i]+=lsizes[j];
}
else
{
//--- check
if((ltypes[i]==-2 || ltypes[i]==-3) || ltypes[i]==-4)
lnsyn[i]=0;
}
//--- check
if(!CAp::Assert(lnsyn[i]>=0,__FUNCTION__+": internal error #0!"))
return;
//--- Other info
lnfirst[i]=ntotal;
ntotal+=lsizes[i];
//--- check
if(ltypes[i]==0)
wcount+=lnsyn[i]*lsizes[i];
}
ssize=7+ntotal*m_nfieldwidth;
//--- Allocate
network.m_structinfo.Resize(ssize);
network.m_weights.Resize(wcount);
//--- check
if(isclsnet)
{
//--- allocation
network.m_columnmeans.Resize(nin);
network.m_columnsigmas.Resize(nin);
}
else
{
//--- allocation
network.m_columnmeans.Resize(nin+nout);
network.m_columnsigmas.Resize(nin+nout);
}
//--- allocation
network.m_neurons.Resize(ntotal);
network.m_nwbuf.Resize(MathMax(wcount,2*nout));
network.m_integerbuf.Resize(4);
network.m_dfdnet.Resize(ntotal);
network.m_x.Resize(nin);
network.m_y.Resize(nout);
network.m_derror.Resize(ntotal);
//--- Fill structure:
//--- * first, fill by dummy values to avoid spurious reports by Valgrind
//--- * then fill global info header
network.m_structinfo.Fill(-999999);
network.m_structinfo.Set(0,ssize);
network.m_structinfo.Set(1,nin);
network.m_structinfo.Set(2,nout);
network.m_structinfo.Set(3,ntotal);
network.m_structinfo.Set(4,wcount);
network.m_structinfo.Set(5,7);
//--- check
if(isclsnet)
network.m_structinfo.Set(6,1);
else
network.m_structinfo.Set(6,0);
//--- Fill structure: neuron connections
nprocessed=0;
wallocated=0;
//--- calculation
for(i=0; i<layerscount; i++)
{
for(j=0; j<lsizes[i]; j++)
{
offs=network.m_structinfo[5]+nprocessed*m_nfieldwidth;
network.m_structinfo.Set(offs,ltypes[i]);
//--- check
if(ltypes[i]==0)
{
//--- Adaptive summator:
//--- * connections with weights to previous neurons
network.m_structinfo.Set(offs+1,lnsyn[i]);
network.m_structinfo.Set(offs+2,lnfirst[lconnfirst[i]]);
network.m_structinfo.Set(offs+3,wallocated);
wallocated=wallocated+lnsyn[i];
nprocessed=nprocessed+1;
}
//--- check
if(ltypes[i]>0 || ltypes[i]==-5)
{
//--- Activation layer:
//--- * each neuron connected to one (only one) of previous neurons.
//--- * no weights
network.m_structinfo.Set(offs+1,1);
network.m_structinfo.Set(offs+2,lnfirst[lconnfirst[i]]+j);
network.m_structinfo.Set(offs+3,-1);
nprocessed=nprocessed+1;
}
//--- check
if((ltypes[i]==-2 || ltypes[i]==-3) || ltypes[i]==-4)
nprocessed++;
}
}
//--- check
if(!CAp::Assert(wallocated==wcount,__FUNCTION__+": internal error #1!"))
return;
//--- check
if(!CAp::Assert(nprocessed==ntotal,__FUNCTION__+": internal error #2!"))
return;
//--- Fill weights by small random values
//--- Initialize means and sigmas
network.m_columnmeans.Fill(0);
network.m_columnsigmas.Fill(1);
MLPRandomize(network);
}
//+------------------------------------------------------------------+
//| Internal subroutine for Hessian calculation. |
//| WARNING!!! Unspeakable math far beyong human capabilities :) |
//+------------------------------------------------------------------+
void CMLPBase::MLPHessianBatchInternal(CMultilayerPerceptron &network,
CMatrixDouble &xy,const int ssize,
const bool naturalerr,double &e,
CRowDouble &grad,CMatrixDouble &h)
{
//--- create variables
int nin=0;
int nout=0;
int wcount=0;
int ntotal=0;
int istart=0;
int i=0;
int j=0;
int k=0;
int kl=0;
int offs=0;
int n1=0;
int n2=0;
int w1=0;
int w2=0;
double s=0;
double t=0;
double v=0;
double et=0;
bool bflag;
double f=0;
double df=0;
double d2f=0;
double deidyj=0;
double mx=0;
double q=0;
double z=0;
double s2=0;
double expi=0;
double expj=0;
int i_=0;
int i1_=0;
//--- creating arrays
CRowDouble x;
CRowDouble desiredy;
CRowDouble gt;
//--- create matrix
CMatrixDouble rx;
CMatrixDouble ry;
CMatrixDouble rdx;
CMatrixDouble rdy;
matrix<double> splittedXY[];
//--- initialization
e=0;
//--- function call
MLPProperties(network,nin,nout,wcount);
//--- initialization
ntotal=network.m_structinfo[3];
istart=network.m_structinfo[5];
//--- Prepare
rdx.Resize(ntotal+nout,wcount);
rdy.Resize(ntotal+nout,wcount);
//--- initialization
e=0;
grad=vector<double>::Zeros(wcount);
h=matrix<double>::Zeros(wcount,wcount);
ulong parts[2];
parts[0]=nin;
parts[1]=(MLPIsSoftMax(network)?1:nout);
xy.Split(parts,1,splittedXY);
//--- Process
for(k=0; k<ssize ; k++)
{
//--- Process vector with MLPGradN.
//--- Now Neurons,DFDNET and DError contains results of the last run.
x=splittedXY[0].Row(k);
if(MLPIsSoftMax(network))
{
kl=(int)MathRound(splittedXY[1][k,0]);
desiredy=vector<double>::Zeros(nout);
desiredy.Set(kl,1);
}
else
desiredy=splittedXY[1].Row(k);
//--- check
if(naturalerr)
MLPGradN(network,x,desiredy,et,gt);
else
MLPGrad(network,x,desiredy,et,gt);
//--- grad,error
e+=et;
grad+=gt;
//--- Hessian.
//--- Forward pass of the R-algorithm
rx=matrix<double>::Zeros(ntotal,wcount);
ry=rx;
for(i=0; i<ntotal; i++)
{
offs=istart+i*m_nfieldwidth;
//--- check
if(network.m_structinfo[offs]>0 || network.m_structinfo[offs]==-5)
{
//--- Activation function
n1=network.m_structinfo[offs+2];
rx.Row(i,ry,n1);
//--- calculation
v=network.m_dfdnet[i];
ry.Row(i,rx[i]*v);
continue;
}
//--- check
if(network.m_structinfo[offs]==0)
{
//--- Adaptive summator
n1=network.m_structinfo[offs+2];
n2=n1+network.m_structinfo[offs+1];
w1=network.m_structinfo[offs+3];
w2=w1+network.m_structinfo[offs+1];
//--- calculation
for(j=n1; j<n2; j++)
{
v=network.m_weights[w1+j-n1];
rx.Row(i,rx[i]+ry[j]*v);
rx.Add(i,w1+j-n1,network.m_neurons[j]);
}
ry.Row(i,rx,i);
continue;
}
//--- check
if(network.m_structinfo[offs]<0)
{
bflag=true;
//--- check
switch(network.m_structinfo[offs])
{
case -2:
//--- input neuron,left unchanged
bflag=false;
break;
case -3:
//--- "-1" neuron,left unchanged
bflag=false;
break;
case -4:
//--- "0" neuron,left unchanged
bflag=false;
break;
}
//--- check
if(!CAp::Assert(!bflag,__FUNCTION__+": internal error - unknown neuron type!"))
return;
continue;
}
}
//--- Hessian. Backward pass of the R-algorithm.
//--- Stage 1. Initialize RDY
rdy=matrix<double>::Zeros(ntotal+nout,wcount);
rx.Resize(ntotal+nout,wcount);
ry.Resize(ntotal+nout,wcount);
//--- check
if(network.m_structinfo[6]==0)
{
//--- Standardisation.
//--- In context of the Hessian calculation standardisation
//--- is considered as additional layer with weightless
//--- activation function:
//--- F(NET) :=Sigma*NET
//--- So we add one more layer to forward pass,and
//--- make forward/backward pass through this layer.
for(i=0; i<nout; i++)
{
n1=ntotal-nout+i;
n2=ntotal+i;
//--- Forward pass from N1 to N2
rx.Row(n2,ry,n1);
v=network.m_columnsigmas[nin+i];
ry.Row(n2,rx[n2]*v);
//--- Initialization of RDY
rdy.Row(n2,ry,n2);
//--- Backward pass from N2 to N1:
//--- 1. Calculate R(dE/dX).
//--- 2. No R(dE/dWij) is needed since weight of activation neuron
//--- is fixed to 1. So we can update R(dE/dY) for
//--- the connected neuron (note that Vij=0,Wij=1)
df=network.m_columnsigmas[nin+i];
rdx.Row(n2,rdy[n2]*df);
rdy.Row(n1,rdy[n1]+rdx[n2]);
}
}
else
{
//--- Softmax.
//--- Initialize RDY using generalized expression for ei'(yi)
//--- (see expression (9) from p. 5 of "Fast Exact Multiplication by the Hessian").
//--- When we are working with softmax network,generalized
//--- expression for ei'(yi) is used because softmax
//--- normalization leads to ei,which depends on all y's
if(naturalerr)
{
//--- softmax + cross-entropy.
//--- We have:
//--- S=sum(exp(yk)),
//--- ei=sum(trn)*exp(yi)/S-trn_i
//--- j=i: d(ei)/d(yj)=T*exp(yi)*(S-exp(yi))/S^2
//--- j<>i: d(ei)/d(yj)=-T*exp(yi)*exp(yj)/S^2
t=desiredy.Sum();
mx=network.m_neurons[ntotal-nout];
//--- calculation
for(i=1; i<nout; i++)
mx=MathMax(mx,network.m_neurons[ntotal-nout+i]);
s=0;
for(i=0; i<nout; i++)
{
network.m_nwbuf.Set(i,MathExp(network.m_neurons[ntotal-nout+i]-mx));
s=s+network.m_nwbuf[i];
}
//--- calculation
for(i=0; i<nout; i++)
{
for(j=0; j<nout; j++)
{
//--- check
if(j==i)
{
deidyj=t*network.m_nwbuf[i]*(s-network.m_nwbuf[i])/CMath::Sqr(s);
rdy.Row(ntotal-nout+i,rdy[ntotal-nout+i]+ry[ntotal-nout+i]*deidyj);
}
else
{
deidyj=-(t*network.m_nwbuf[i]*network.m_nwbuf[j]/CMath::Sqr(s));
rdy.Row(ntotal-nout+i,rdy[ntotal-nout+i]+ry[ntotal-nout+j]*deidyj);
}
}
}
}
else
{
//--- For a softmax + squared error we have expression
//--- far beyond human imagination so we dont even try
//--- to comment on it. Just enjoy the code...
//--- P.S. That's why "natural error" is called "natural" -
//--- compact beatiful expressions,fast code....
mx=network.m_neurons[ntotal-nout];
for(i=1; i<nout; i++)
mx=MathMax(mx,network.m_neurons[ntotal-nout+i]);
//--- calculation
s=0;
s2=0;
for(i=0; i<nout; i++)
{
network.m_nwbuf.Set(i,MathExp(network.m_neurons[ntotal-nout+i]-mx));
s+=network.m_nwbuf[i];
s2+=CMath::Sqr(network.m_nwbuf[i]);
}
//--- calculation
q=0;
for(i=0; i<nout; i++)
q+=(network.m_y[i]-desiredy[i])*network.m_nwbuf[i];
for(i=0; i<nout; i++)
{
//--- change values
z=-q+(network.m_y[i]-desiredy[i])*s;
expi=network.m_nwbuf[i];
for(j=0; j<nout; j++)
{
expj=network.m_nwbuf[j];
//--- check
if(j==i)
deidyj=expi/CMath::Sqr(s)*((z+expi)*(s-2*expi)/s+expi*s2/CMath::Sqr(s));
else
deidyj=expi*expj/CMath::Sqr(s)*(s2/CMath::Sqr(s)-2*z/s-(expi+expj)/s+(network.m_y[i]-desiredy[i])-(network.m_y[j]-desiredy[j]));
rdy.Row(ntotal-nout+i,rdy[ntotal-nout+i]+ry[ntotal-nout+j]*deidyj);
}
}
}
}
//--- Hessian. Backward pass of the R-algorithm
//--- Stage 2. Process.
for(i=ntotal-1; i>=0; i--)
{
//--- Possible variants:
//--- 1. Activation function
//--- 2. Adaptive summator
//--- 3. Special neuron
offs=istart+i*m_nfieldwidth;
//--- check
if(network.m_structinfo[offs]>0 || network.m_structinfo[offs]==-5)
{
n1=network.m_structinfo[offs+2];
//--- First,calculate R(dE/dX).
MLPActivationFunction(network.m_neurons[n1],network.m_structinfo[offs],f,df,d2f);
v=d2f*network.m_derror[i];
rdx.Row(i,rdy[i]*df);
rdx.Row(i,rdx[i]+rx[i]*v);
//--- No R(dE/dWij) is needed since weight of activation neuron
//--- is fixed to 1.
//--- So we can update R(dE/dY) for the connected neuron.
//--- (note that Vij=0,Wij=1)
rdy.Row(n1,rdy[n1]+rdx[i]);
continue;
}
//--- check
if(network.m_structinfo[offs]==0)
{
//--- Adaptive summator
n1=network.m_structinfo[offs+2];
n2=n1+network.m_structinfo[offs+1]-1;
w1=network.m_structinfo[offs+3];
w2=w1+network.m_structinfo[offs+1]-1;
//--- First,calculate R(dE/dX).
rdx.Row(i,rdy,i);
//--- Then,calculate R(dE/dWij)
for(j=w1; j<=w2; j++)
{
v=network.m_neurons[n1+j-w1];
h.Row(j,h[j]+rdx[i]*v);
//--- calculation
v=network.m_derror[i];
h.Row(j,h[j]+ry[n1+j-w1]*v);
}
//--- And finally,update R(dE/dY) for connected neurons.
for(j=w1; j<=w2; j++)
{
v=network.m_weights[j];
rdy.Row(n1+j-w1,rdy[n1+j-w1]+ rdx[i]*v);
rdy.Set(n1+j-w1,j,rdy[n1+j-w1][j]+network.m_derror[i]);
}
continue;
}
//--- check
if(network.m_structinfo[offs]<0)
{
bflag=false;
//--- check
if((network.m_structinfo[offs]==-2 || network.m_structinfo[offs]==-3) || network.m_structinfo[offs]==-4)
{
//--- Special neuron type,no back-propagation required
bflag=true;
}
//--- check
if(!CAp::Assert(bflag,__FUNCTION__+": unknown neuron type!"))
return;
continue;
}
}
}
}
//+------------------------------------------------------------------+
//| Internal subroutine |
//| Network must be processed by MLPProcess on X |
//+------------------------------------------------------------------+
void CMLPBase::MLPInternalCalculateGradient(CMultilayerPerceptron &network,
CRowDouble &neurons,
CRowDouble &weights,
CRowDouble &derror,
CRowDouble &grad,
const bool naturalerrorfunc)
{
//--- create variables
int i=0;
int n1=0;
int n2=0;
int w1=0;
int w2=0;
int ntotal=network.m_structinfo[3];
int istart=network.m_structinfo[5];
int nin=network.m_structinfo[1];
int nout=network.m_structinfo[2];
int offs=0;
double dedf=0;
double dfdnet=0;
double v=0;
double fown=0;
double deown=0;
double net=0;
double mx=0;
bool bflag;
int i_=0;
int i1_=0;
//--- Pre-processing of dError/dOut:
//--- from dError/dOut(normalized) to dError/dOut(non-normalized)
if(!CAp::Assert(network.m_structinfo[6]==0 || network.m_structinfo[6]==1,__FUNCTION__+": unknown normalization type!"))
return;
//--- check
if(network.m_structinfo[6]==1)
{
//--- Softmax
if(!naturalerrorfunc)
{
mx=network.m_neurons[ntotal-nout];
for(i=1; i<nout; i++)
mx=MathMax(mx,network.m_neurons[ntotal-nout+i]);
net=0;
for(i=0; i<nout; i++)
{
network.m_nwbuf.Set(i,MathExp(network.m_neurons[ntotal-nout+i]-mx));
net+=network.m_nwbuf[i];
}
//--- calculation
i1_=ntotal-nout;
v=0.0;
for(i=0; i<nout; i++)
v+=network.m_derror[i+i1_]*network.m_nwbuf[i];
for(i=0; i<nout; i++)
{
fown=network.m_nwbuf[i];
deown=network.m_derror[i+i1_];
network.m_nwbuf.Set(nout+i,(-v+deown*fown+deown*(net-fown))*fown/CMath::Sqr(net));
network.m_derror.Set(i+i1_,network.m_nwbuf[nout+i]);
}
}
}
else
{
//--- Un-standardisation
for(i=0; i<nout; i++)
network.m_derror.Set(ntotal-nout+i,network.m_derror[ntotal-nout+i]*network.m_columnsigmas[nin+i]);
}
//--- Backpropagation
for(i=ntotal-1; i>=0; i--)
{
//--- Extract info
offs=istart+i*m_nfieldwidth;
//--- check
if(network.m_structinfo[offs]>0 || network.m_structinfo[offs]==-5)
{
//--- Activation function
dedf=network.m_derror[i];
dfdnet=network.m_dfdnet[i];
derror.Set(network.m_structinfo[offs+2],derror[network.m_structinfo[offs+2]]+dedf*dfdnet);
continue;
}
//--- check
if(network.m_structinfo[offs]==0)
{
//--- Adaptive summator
n1=network.m_structinfo[offs+2];
n2=n1+network.m_structinfo[offs+1];
w1=network.m_structinfo[offs+3];
w2=w1+network.m_structinfo[offs+1];
dedf=network.m_derror[i];
dfdnet=1.0;
v=dedf*dfdnet;
i1_=n1-w1;
//--- calculation
for(i_=w1; i_<w2; i_++)
grad.Set(i_,v*neurons[i_+i1_]);
i1_=w1-n1;
for(i_=n1; i_<n2; i_++)
derror.Add(i_,v*weights[i_+i1_]);
continue;
}
//--- check
if(network.m_structinfo[offs]<0)
{
bflag=false;
//--- check
if((network.m_structinfo[offs]==-2 || network.m_structinfo[offs]==-3) || network.m_structinfo[offs]==-4)
{
//--- Special neuron type,no back-propagation required
bflag=true;
}
//--- check
if(!CAp::Assert(bflag,__FUNCTION__+": unknown neuron type!"))
return;
continue;
}
}
}
//+------------------------------------------------------------------+
//| Internal subroutine, chunked gradient |
//+------------------------------------------------------------------+
void CMLPBase::MLPChunkedGradient(CMultilayerPerceptron &network,
CMatrixDouble &xy,const int cstart,
const int csize,CRowDouble &batch4buf,
CRowDouble &hpcbuf,
double &e,const bool naturalerrorfunc)
{
//--- create variables
int i=0;
int j=0;
int k=0;
int kl=0;
int ntotal=0;
int nin=0;
int nout=0;
int offs=0;
double f=0;
double df=0;
double d2f=0;
double v=0;
double vv=0;
double s=0;
double fown=0;
double deown=0;
bool bflag;
int istart=0;
int entrysize=0;
int dfoffs=0;
int derroroffs=0;
int entryoffs=0;
int neuronidx=0;
int srcentryoffs=0;
int srcneuronidx=0;
int srcweightidx=0;
int neurontype=0;
int nweights=0;
int offs0=0;
int offs1=0;
int offs2=0;
double v0=0;
double v1=0;
double v2=0;
double v3=0;
double s0=0;
double s1=0;
double s2=0;
double s3=0;
int chunksize=4;
//--- check
if(!CAp::Assert(csize<=chunksize,__FUNCTION__": internal error (CSize>ChunkSize)"))
return;
//--- Read network geometry, prepare data
nin=network.m_structinfo[1];
nout=network.m_structinfo[2];
ntotal=network.m_structinfo[3];
istart=network.m_structinfo[5];
entrysize=12;
dfoffs=4;
derroroffs=8;
//--- Fill Batch4Buf by zeros.
//--- THIS STAGE IS VERY IMPORTANT!
//--- We fill all components of entry - neuron values, dF/dNET, dError/dF.
//--- It allows us to easily handle situations when CSize<ChunkSize by
//--- simply working with ALL components of Batch4Buf, without ever
//--- looking at CSize. The idea is that dError/dF for absent components
//--- will be initialized by zeros - and won't be rewritten by non-zero
//--- values during backpropagation.
batch4buf.Fill(0);
//--- Forward pass:
//--- 1. Load data into Batch4Buf. If CSize<ChunkSize, data are padded by zeros.
//--- 2. Perform forward pass through network
for(i=0; i<nin; i++)
{
entryoffs=entrysize*i;
for(j=0; j<csize; j++)
{
if(network.m_columnsigmas[i]!=0.0)
batch4buf.Set(entryoffs+j,(xy.Get(cstart+j,i)-network.m_columnmeans[i])/network.m_columnsigmas[i]);
else
batch4buf.Set(entryoffs+j,xy.Get(cstart+j,i)-network.m_columnmeans[i]);
}
}
for(neuronidx=0; neuronidx<ntotal; neuronidx++)
{
entryoffs=entrysize*neuronidx;
offs=istart+neuronidx*m_nfieldwidth;
neurontype=network.m_structinfo[offs+0];
if(neurontype>0 || neurontype==-5)
{
//--- "activation function" neuron, which takes value of neuron SrcNeuronIdx
//--- and applies activation function to it.
//--- This neuron has no weights and no tunable parameters.
srcneuronidx=network.m_structinfo[offs+2];
srcentryoffs=entrysize*srcneuronidx;
for(i=0; i<4; i++)
{
MLPActivationFunction(batch4buf[srcentryoffs+i],neurontype,f,df,d2f);
batch4buf.Set(entryoffs+i,f);
batch4buf.Set(entryoffs+dfoffs+i,df);
}
continue;
}
if(neurontype==0)
{
//--- "adaptive summator" neuron, whose output is a weighted sum of inputs.
//--- It has weights, but has no activation function.
nweights=network.m_structinfo[offs+1];
srcneuronidx=network.m_structinfo[offs+2];
srcentryoffs=entrysize*srcneuronidx;
srcweightidx=network.m_structinfo[offs+3];
v0=0;
v1=0;
v2=0;
v3=0;
for(j=0; j<nweights; j++)
{
v=network.m_weights[srcweightidx];
srcweightidx++;
v0+=v*batch4buf[srcentryoffs];
v1+=v*batch4buf[srcentryoffs+1];
v2+=v*batch4buf[srcentryoffs+2];
v3+=v*batch4buf[srcentryoffs+3];
srcentryoffs+=entrysize;
}
batch4buf.Set(entryoffs,v0);
batch4buf.Set(entryoffs+1,v1);
batch4buf.Set(entryoffs+2,v2);
batch4buf.Set(entryoffs+3,v3);
batch4buf.Set(entryoffs+dfoffs,1);
batch4buf.Set(entryoffs+1+dfoffs,1);
batch4buf.Set(entryoffs+2+dfoffs,1);
batch4buf.Set(entryoffs+3+dfoffs,1);
continue;
}
if(neurontype<0)
{
bflag=false;
switch(neurontype)
{
case -2:
//--- Input neuron, left unchanged
bflag=true;
break;
case -3:
//--- "-1" neuron
batch4buf.Set(entryoffs,-1);
batch4buf.Set(entryoffs+1,-1);
batch4buf.Set(entryoffs+2,-1);
batch4buf.Set(entryoffs+3,-1);
batch4buf.Set(entryoffs+dfoffs,0);
batch4buf.Set(entryoffs+1+dfoffs,0);
batch4buf.Set(entryoffs+2+dfoffs,0);
batch4buf.Set(entryoffs+3+dfoffs,0);
bflag=true;
break;
case -4:
//--- "0" neuron
batch4buf.Set(entryoffs,0);
batch4buf.Set(entryoffs+1,0);
batch4buf.Set(entryoffs+2,0);
batch4buf.Set(entryoffs+3,0);
batch4buf.Set(entryoffs+dfoffs,0);
batch4buf.Set(entryoffs+1+dfoffs,0);
batch4buf.Set(entryoffs+2+dfoffs,0);
batch4buf.Set(entryoffs+3+dfoffs,0);
bflag=true;
break;
}
if(!CAp::Assert(bflag,__FUNCTION__": internal error - unknown neuron type!"))
return;
continue;
}
}
//--- Intermediate phase between forward and backward passes.
//--- For regression networks:
//--- * forward pass is completely done (no additional post-processing is
//--- needed).
//--- * before starting backward pass, we have to calculate dError/dOut
//--- for output neurons. We also update error at this phase.
//--- For classification networks:
//--- * in addition to forward pass we apply SOFTMAX normalization to
//--- output neurons.
//--- * after applying normalization, we have to calculate dError/dOut,
//--- which is calculated in two steps:
//--- * first, we calculate derivative of error with respect to SOFTMAX
//--- normalized outputs (normalized dError)
//--- * then, we calculate derivative of error with respect to values
//--- of outputs BEFORE normalization was applied to them
if(!CAp::Assert(network.m_structinfo[6]==0 || network.m_structinfo[6]==1,__FUNCTION__": unknown normalization type!"))
return;
if(network.m_structinfo[6]==1)
{
//--- SOFTMAX-normalized network.
//--- First, calculate (V0,V1,V2,V3) - component-wise maximum
//--- of output neurons. This vector of maximum values will be
//--- used for normalization of outputs prior to calculating
//--- exponentials.
//--- NOTE: the only purpose of this stage is to prevent overflow
//--- during calculation of exponentials. With this stage
//--- we make sure that all exponentials are calculated
//--- with non-positive argument. If you load (0,0,0,0) to
//--- (V0,V1,V2,V3), your program will continue working -
//--- although with less robustness.
entryoffs=entrysize*(ntotal-nout);
v0=batch4buf[entryoffs];
v1=batch4buf[entryoffs+1];
v2=batch4buf[entryoffs+2];
v3=batch4buf[entryoffs+3];
entryoffs+=entrysize;
for(i=1; i<nout; i++)
{
v=batch4buf[entryoffs];
if(v>v0)
v0=v;
v=batch4buf[entryoffs+1];
if(v>v1)
v1=v;
v=batch4buf[entryoffs+2];
if(v>v2)
v2=v;
v=batch4buf[entryoffs+3];
if(v>v3)
v3=v;
entryoffs=entryoffs+entrysize;
}
//--- Then, calculate exponentials and place them to part of the
//--- array which is located past the last entry. We also
//--- calculate sum of exponentials which will be stored past the
//--- exponentials.
entryoffs=entrysize*(ntotal-nout);
offs0=entrysize*ntotal;
s0=0;
s1=0;
s2=0;
s3=0;
for(i=0; i<nout; i++)
{
v=MathExp(batch4buf[entryoffs]-v0);
s0+=v;
batch4buf.Set(offs0,v);
v=MathExp(batch4buf[entryoffs+1]-v1);
s1+=v;
batch4buf.Set(offs0+1,v);
v=MathExp(batch4buf[entryoffs+2]-v2);
s2+=v;
batch4buf.Set(offs0+2,v);
v=MathExp(batch4buf[entryoffs+3]-v3);
s3+=v;
batch4buf.Set(offs0+3,v);
entryoffs=entryoffs+entrysize;
offs0=offs0+chunksize;
}
offs0=entrysize*ntotal+2*nout*chunksize;
batch4buf.Set(offs0,s0);
batch4buf.Set(offs0+1,s1);
batch4buf.Set(offs0+2,s2);
batch4buf.Set(offs0+3,s3);
//--- Now we have:
//--- * Batch4Buf[0...EntrySize*NTotal-1] stores:
//--- * NTotal*ChunkSize neuron output values (SOFTMAX normalization
//--- was not applied to these values),
//--- * NTotal*ChunkSize values of dF/dNET (derivative of neuron
//--- output with respect to its input)
//--- * NTotal*ChunkSize zeros in the elements which correspond to
//--- dError/dOut (derivative of error with respect to neuron output).
//--- * Batch4Buf[EntrySize*NTotal...EntrySize*NTotal+ChunkSize*NOut-1] -
//--- stores exponentials of last NOut neurons.
//--- * Batch4Buf[EntrySize*NTotal+ChunkSize*NOut-1...EntrySize*NTotal+ChunkSize*2*NOut-1]
//--- - can be used for temporary calculations
//--- * Batch4Buf[EntrySize*NTotal+ChunkSize*2*NOut...EntrySize*NTotal+ChunkSize*2*NOut+ChunkSize-1]
//--- - stores sum-of-exponentials
//--- Block below calculates derivatives of error function with respect
//--- to non-SOFTMAX-normalized output values of last NOut neurons.
//--- It is quite complicated; we do not describe algebra behind it,
//--- but if you want you may check it yourself :)
if(naturalerrorfunc)
{
//--- Calculate derivative of error with respect to values of
//--- output neurons PRIOR TO SOFTMAX NORMALIZATION. Because we
//--- use natural error function (cross-entropy), we can do so
//--- very easy.
offs0=entrysize*ntotal+2*nout*chunksize;
for(k=0; k<csize; k++)
{
s=batch4buf[offs0+k];
kl=(int)MathRound(xy.Get(cstart+k,nin));
offs1=(ntotal-nout)*entrysize+derroroffs+k;
offs2=entrysize*ntotal+k;
for(i=0; i<nout; i++)
{
v=(double)(i==kl);
vv=batch4buf[offs2];
batch4buf.Set(offs1,vv/s-v);
e+=SafeCrossEntropy(v,vv/s);
offs1+=entrysize;
offs2+=chunksize;
}
}
}
else
{
//--- SOFTMAX normalization makes things very difficult.
//--- Sorry, we do not dare to describe this esoteric math
//--- in details.
offs0=entrysize*ntotal+chunksize*2*nout;
for(k=0; k<csize; k++)
{
s=batch4buf[offs0+k];
kl=(int)MathRound(xy.Get(cstart+k,nin));
vv=0;
offs1=entrysize*ntotal+k;
offs2=entrysize*ntotal+nout*chunksize+k;
for(i=0; i<nout; i++)
{
fown=batch4buf[offs1];
if(i==kl)
deown=fown/s-1;
else
deown=fown/s;
batch4buf.Set(offs2,deown);
vv+=deown*fown;
e+=deown*deown/2;
offs1+=chunksize;
offs2+=chunksize;
}
offs1=entrysize*ntotal+k;
offs2=entrysize*ntotal+nout*chunksize+k;
for(i=0; i<nout; i++)
{
fown=batch4buf[offs1];
deown=batch4buf[offs2];
batch4buf.Set((ntotal-nout+i)*entrysize+derroroffs+k,(-vv+deown*fown+deown*(s-fown))*fown/CMath::Sqr(s));
offs1+=chunksize;
offs2+=chunksize;
}
}
}
}
else
{
//--- Regression network with sum-of-squares function.
//--- For each NOut of last neurons:
//--- * calculate difference between actual and desired output
//--- * calculate dError/dOut for this neuron (proportional to difference)
//--- * store in in last 4 components of entry (these values are used
//--- to start backpropagation)
//--- * update error
for(i=0; i<nout; i++)
{
v0=network.m_columnsigmas[nin+i];
v1=network.m_columnmeans[nin+i];
entryoffs=entrysize*(ntotal-nout+i);
offs0=entryoffs;
offs1=entryoffs+derroroffs;
for(j=0; j<csize; j++)
{
v=batch4buf[offs0+j]*v0+v1-xy.Get(cstart+j,nin+i);
batch4buf.Set(offs1+j,v*v0);
e+=v*v/2;
}
}
}
//--- Backpropagation
for(neuronidx=ntotal-1; neuronidx>=0; neuronidx--)
{
entryoffs=entrysize*neuronidx;
offs=istart+neuronidx*m_nfieldwidth;
neurontype=network.m_structinfo[offs+0];
if(neurontype>0 || neurontype==-5)
{
//--- Activation function
srcneuronidx=network.m_structinfo[offs+2];
srcentryoffs=entrysize*srcneuronidx;
offs0=srcentryoffs+derroroffs;
offs1=entryoffs+derroroffs;
offs2=entryoffs+dfoffs;
batch4buf.Set(offs0,batch4buf[offs0]+batch4buf[offs1]*batch4buf[offs2]);
batch4buf.Set(offs0+1,batch4buf[offs0+1]+batch4buf[offs1+1]*batch4buf[offs2+1]);
batch4buf.Set(offs0+2,batch4buf[offs0+2]+batch4buf[offs1+2]*batch4buf[offs2+2]);
batch4buf.Set(offs0+3,batch4buf[offs0+3]+batch4buf[offs1+3]*batch4buf[offs2+3]);
continue;
}
if(neurontype==0)
{
//--- Adaptive summator
nweights=network.m_structinfo[offs+1];
srcneuronidx=network.m_structinfo[offs+2];
srcentryoffs=entrysize*srcneuronidx;
srcweightidx=network.m_structinfo[offs+3];
v0=batch4buf[entryoffs+derroroffs];
v1=batch4buf[entryoffs+derroroffs+1];
v2=batch4buf[entryoffs+derroroffs+2];
v3=batch4buf[entryoffs+derroroffs+3];
for(j=0; j<nweights; j++)
{
offs0=srcentryoffs;
offs1=srcentryoffs+derroroffs;
v=network.m_weights[srcweightidx];
hpcbuf.Set(srcweightidx,hpcbuf[srcweightidx]+batch4buf[offs0]*v0+batch4buf[offs0+1]*v1+batch4buf[offs0+2]*v2+batch4buf[offs0+3]*v3);
batch4buf.Set(offs1,batch4buf[offs1+0]+v*v0);
batch4buf.Set(offs1+1,batch4buf[offs1+1]+v*v1);
batch4buf.Set(offs1+2,batch4buf[offs1+2]+v*v2);
batch4buf.Set(offs1+3,batch4buf[offs1+3]+v*v3);
srcentryoffs+=entrysize;
srcweightidx ++;
}
continue;
}
if(neurontype<0)
{
bflag=false;
if((neurontype==-2 || neurontype==-3) || neurontype==-4)
{
//--- Special neuron type, no back-propagation required
bflag=true;
}
if(!CAp::Assert(bflag,"MLPInternalCalculateGradient: unknown neuron type!"))
return;
continue;
}
}
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CMLPBase::MLPChunkedProcess(CMultilayerPerceptron &network,
CMatrixDouble &xy,
int cstart,int csize,
CRowDouble &batch4buf,
CRowDouble &hpcbuf)
{
//--- create variables
int i=0;
int j=0;
int ntotal=0;
int nin=0;
int nout=0;
int offs=0;
double f=0;
double df=0;
double d2f=0;
double v=0;
bool bflag;
int istart=0;
int entrysize=0;
int entryoffs=0;
int neuronidx=0;
int srcentryoffs=0;
int srcneuronidx=0;
int srcweightidx=0;
int neurontype=0;
int nweights=0;
int offs0=0;
double v0=0;
double v1=0;
double v2=0;
double v3=0;
double s0=0;
double s1=0;
double s2=0;
double s3=0;
int chunksize= 4;
//--- check
if(!CAp::Assert(csize<=chunksize,__FUNCTION__": internal error (CSize>ChunkSize)"))
return;
//--- Read network geometry, prepare data
nin=network.m_structinfo[1];
nout=network.m_structinfo[2];
ntotal=network.m_structinfo[3];
istart=network.m_structinfo[5];
entrysize=4;
//--- Fill Batch4Buf by zeros.
//--- THIS STAGE IS VERY IMPORTANT!
//--- We fill all components of entry - neuron values, dF/dNET, dError/dF.
//--- It allows us to easily handle situations when CSize<ChunkSize by
//--- simply working with ALL components of Batch4Buf, without ever
//--- looking at CSize.
for(i=0; i<entrysize*ntotal; i++)
batch4buf.Fill(0);
//--- Forward pass:
//--- 1. Load data into Batch4Buf. If CSize<ChunkSize, data are padded by zeros.
//--- 2. Perform forward pass through network
for(i=0; i<nin; i++)
{
entryoffs=entrysize*i;
for(j=0; j<csize; j++)
{
if((double)(network.m_columnsigmas[i])!=0.0)
batch4buf.Set(entryoffs+j,(xy.Get(cstart+j,i)-network.m_columnmeans[i])/network.m_columnsigmas[i]);
else
batch4buf.Set(entryoffs+j,xy.Get(cstart+j,i)-network.m_columnmeans[i]);
}
}
for(neuronidx=0; neuronidx<ntotal; neuronidx++)
{
entryoffs=entrysize*neuronidx;
offs=istart+neuronidx*m_nfieldwidth;
neurontype=network.m_structinfo[offs+0];
if(neurontype>0 || neurontype==-5)
{
//--- "activation function" neuron, which takes value of neuron SrcNeuronIdx
//--- and applies activation function to it.
//--- This neuron has no weights and no tunable parameters.
srcneuronidx=network.m_structinfo[offs+2];
srcentryoffs=entrysize*srcneuronidx;
for(i=0; i<4; i++)
{
MLPActivationFunction(batch4buf[srcentryoffs+i],neurontype,f,df,d2f);
batch4buf.Set(entryoffs+i,f);
}
continue;
}
if(neurontype==0)
{
//--- "adaptive summator" neuron, whose output is a weighted sum of inputs.
//--- It has weights, but has no activation function.
nweights=network.m_structinfo[offs+1];
srcneuronidx=network.m_structinfo[offs+2];
srcentryoffs=entrysize*srcneuronidx;
srcweightidx=network.m_structinfo[offs+3];
v0=0;
v1=0;
v2=0;
v3=0;
for(j=0; j<nweights; j++)
{
v=network.m_weights[srcweightidx];
srcweightidx++;
v0+=v*batch4buf[srcentryoffs];
v1+=v*batch4buf[srcentryoffs+1];
v2+=v*batch4buf[srcentryoffs+2];
v3+=v*batch4buf[srcentryoffs+3];
srcentryoffs+=entrysize;
}
batch4buf.Set(entryoffs,v0);
batch4buf.Set(entryoffs+1,v1);
batch4buf.Set(entryoffs+2,v2);
batch4buf.Set(entryoffs+3,v3);
continue;
}
if(neurontype<0)
{
bflag=false;
switch(neurontype)
{
case -2:
//--- Input neuron, left unchanged
bflag=true;
break;
case -3:
//--- "-1" neuron
batch4buf.Set(entryoffs,-1);
batch4buf.Set(entryoffs+1,-1);
batch4buf.Set(entryoffs+2,-1);
batch4buf.Set(entryoffs+3,-1);
bflag=true;
break;
case -4:
//--- "0" neuron
batch4buf.Set(entryoffs,0);
batch4buf.Set(entryoffs+1,0);
batch4buf.Set(entryoffs+2,0);
batch4buf.Set(entryoffs+3,0);
bflag=true;
break;
}
if(!CAp::Assert(bflag,__FUNCTION__": internal error - unknown neuron type!"))
return;
continue;
}
}
//--- SOFTMAX normalization or scaling.
if(!CAp::Assert(network.m_structinfo[6]==0 || network.m_structinfo[6]==1,__FUNCTION__": unknown normalization type!"))
return;
if(network.m_structinfo[6]==1)
{
//--- SOFTMAX-normalized network.
//--- First, calculate (V0,V1,V2,V3) - component-wise maximum
//--- of output neurons. This vector of maximum values will be
//--- used for normalization of outputs prior to calculating
//--- exponentials.
//--- NOTE: the only purpose of this stage is to prevent overflow
//--- during calculation of exponentials. With this stage
//--- we make sure that all exponentials are calculated
//--- with non-positive argument. If you load (0,0,0,0) to
//--- (V0,V1,V2,V3), your program will continue working -
//--- although with less robustness.
entryoffs=entrysize*(ntotal-nout);
v0=batch4buf[entryoffs+0];
v1=batch4buf[entryoffs+1];
v2=batch4buf[entryoffs+2];
v3=batch4buf[entryoffs+3];
entryoffs+=entrysize;
for(i=1; i<nout; i++)
{
v=batch4buf[entryoffs];
if(v>v0)
v0=v;
v=batch4buf[entryoffs+1];
if(v>v1)
v1=v;
v=batch4buf[entryoffs+2];
if(v>v2)
v2=v;
v=batch4buf[entryoffs+3];
if(v>v3)
v3=v;
entryoffs+=entrysize;
}
//--- Then, calculate exponentials and place them to part of the
//--- array which is located past the last entry. We also
//--- calculate sum of exponentials.
entryoffs=entrysize*(ntotal-nout);
offs0=entrysize*ntotal;
s0=0;
s1=0;
s2=0;
s3=0;
for(i=0; i<nout; i++)
{
v=MathExp(batch4buf[entryoffs]-v0);
s0+=v;
batch4buf.Set(offs0,v);
v=MathExp(batch4buf[entryoffs+1]-v1);
s1+=v;
batch4buf.Set(offs0+1,v);
v=MathExp(batch4buf[entryoffs+2]-v2);
s2+=v;
batch4buf.Set(offs0+2,v);
v=MathExp(batch4buf[entryoffs+3]-v3);
s3+=v;
batch4buf.Set(offs0+3,v);
entryoffs+=entrysize;
offs0+=chunksize;
}
//--- Write SOFTMAX-normalized values to the output array.
offs0=entrysize*ntotal;
for(i=0; i<nout; i++)
{
if(csize>0)
xy.Set(cstart,nin+i,batch4buf[offs0]/s0);
if(csize>1)
xy.Set(cstart+1,nin+i,batch4buf[offs0+1]/s1);
if(csize>2)
xy.Set(cstart+2,nin+i,batch4buf[offs0+2]/s2);
if(csize>3)
xy.Set(cstart+3,nin+i,batch4buf[offs0+3]/s3);
offs0+=chunksize;
}
}
else
{
//--- Regression network with sum-of-squares function.
//--- For each NOut of last neurons:
//--- * calculate difference between actual and desired output
//--- * calculate dError/dOut for this neuron (proportional to difference)
//--- * store in in last 4 components of entry (these values are used
//--- to start backpropagation)
//--- * update error
for(i=0; i<nout; i++)
{
v0=network.m_columnsigmas[nin+i];
v1=network.m_columnmeans[nin+i];
entryoffs=entrysize*(ntotal-nout+i);
for(j=0; j<csize; j++)
xy.Set(cstart+j,nin+i,batch4buf[entryoffs+j]*v0+v1);
}
}
}
//+------------------------------------------------------------------+
//| Returns T*Ln(T/Z), guarded against overflow/underflow. |
//| Internal subroutine. |
//+------------------------------------------------------------------+
double CMLPBase::SafeCrossEntropy(const double t,const double z)
{
//--- create variables
double result=0;
double r=0;
//--- check
if(t==0.0)
result=0;
else
{
//--- check
if(MathAbs(z)>1.0)
{
//--- Shouldn't be the case with softmax,
//--- but we just want to be sure.
if(t/z==0.0)
r=CMath::m_minrealnumber;
else
r=t/z;
}
else
{
//--- Normal case
if(z==0.0 || MathAbs(t)>=CMath::m_maxrealnumber*MathAbs(z))
r=CMath::m_maxrealnumber;
else
r=t/z;
}
//--- get result
result=t*MathLog(r);
}
//--- return result
return(result);
}
//+------------------------------------------------------------------+
//| This function performs backward pass of neural network |
//| randimization: |
//| * it assumes that Network.Weights stores standard deviation of |
//| weights (weights are not generated yet, only their deviations|
//| are present) |
//| * it sets deviations of weights which feed NeuronIdx-th neuron |
//| to specified value |
//| * it recursively passes to deeper neuron and modifies their |
//| weights |
//| * it stops after encountering nonlinear neurons, linear |
//| activation function, input neurons, "0" and "-1" neurons |
//+------------------------------------------------------------------+
void CMLPBase::RandomizeBackwardPass(CMultilayerPerceptron &network,
int neuronidx,double v)
{
//--- create variables
int istart=0;
int neurontype=0;
int n1=0;
int n2=0;
int w1=0;
int w2=0;
int offs=0;
istart=network.m_structinfo[5];
neurontype=network.m_structinfo[istart+neuronidx*m_nfieldwidth];
switch(neurontype)
{
case -2:
//--- Input neuron - stop
break;
case -3:
//--- "-1" neuron: stop
break;
case -4:
//--- "0" neuron: stop
break;
case 0:
//--- Adaptive summator neuron:
//--- * modify deviations of its weights
//--- * recursively call this function for its inputs
offs=istart+neuronidx*m_nfieldwidth;
n1=network.m_structinfo[offs+2];
n2=n1+network.m_structinfo[offs+1];
w1=network.m_structinfo[offs+3];
w2=w1+network.m_structinfo[offs+1];
for(int i=w1; i<w2; i++)
network.m_weights.Set(i,v);
for(int i=n1; i<n2; i++)
RandomizeBackwardPass(network,i,v);
break;
case -5:
//--- Linear activation function: stop
break;
default:
if(!CAp::Assert(neurontype>0,__FUNCTION__": unexpected neuron type"))
return;
break;
}
}
//+------------------------------------------------------------------+
//| Auxiliary class for CLogit |
//+------------------------------------------------------------------+
class CLogitModel
{
public:
CRowDouble m_w;
//--- constructor, destructor
CLogitModel(void) {}
~CLogitModel(void) {}
//--- copy
void Copy(const CLogitModel &obj);
//--- overloading
void operator=(const CLogitModel &obj) { Copy(obj); }
};
//+------------------------------------------------------------------+
//| Copy |
//+------------------------------------------------------------------+
void CLogitModel::Copy(const CLogitModel &obj)
{
//--- copy array
m_w=obj.m_w;
}
//+------------------------------------------------------------------+
//| This class is a shell for class CLogitModel |
//+------------------------------------------------------------------+
class CLogitModelShell
{
private:
CLogitModel m_innerobj;
public:
//--- constructors, destructor
CLogitModelShell(void) {}
CLogitModelShell(CLogitModel &obj) { m_innerobj.Copy(obj); }
~CLogitModelShell(void) {}
//--- method
CLogitModel *GetInnerObj(void) { return(GetPointer(m_innerobj)); }
};
//+------------------------------------------------------------------+
//| Auxiliary class for CLogit |
//+------------------------------------------------------------------+
struct CLogitMCState
{
bool m_brackt;
bool m_stage1;
int m_infoc;
double m_dg;
double m_dgm;
double m_dginit;
double m_dgtest;
double m_dgx;
double m_dgxm;
double m_dgy;
double m_dgym;
double m_finit;
double m_ftest1;
double m_fm;
double m_fx;
double m_fxm;
double m_fy;
double m_fym;
double m_stx;
double m_sty;
double m_stmin;
double m_stmax;
double m_width;
double m_width1;
double m_xtrapf;
//--- constructor, destructor
CLogitMCState(void) { ZeroMemory(this); }
~CLogitMCState(void) {}
//---
void Copy(const CLogitMCState &obj);
//--- overloading
void operator=(const CLogitMCState &obj) { Copy(obj); }
};
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CLogitMCState::Copy(const CLogitMCState &obj)
{
m_brackt=obj.m_brackt;
m_stage1=obj.m_stage1;
m_infoc=obj.m_infoc;
m_dg=obj.m_dg;
m_dgm=obj.m_dgm;
m_dginit=obj.m_dginit;
m_dgtest=obj.m_dgtest;
m_dgx=obj.m_dgx;
m_dgxm=obj.m_dgxm;
m_dgy=obj.m_dgy;
m_dgym=obj.m_dgym;
m_finit=obj.m_finit;
m_ftest1=obj.m_ftest1;
m_fm=obj.m_fm;
m_fx=obj.m_fx;
m_fxm=obj.m_fxm;
m_fy=obj.m_fy;
m_fym=obj.m_fym;
m_stx=obj.m_stx;
m_sty=obj.m_sty;
m_stmin=obj.m_stmin;
m_stmax=obj.m_stmax;
m_width=obj.m_width;
m_width1=obj.m_width1;
m_xtrapf=obj.m_xtrapf;
}
//+------------------------------------------------------------------+
//| MNLReport structure contains information about training process: |
//| * NGrad - number of gradient calculations |
//| * NHess - number of Hessian calculations |
//+------------------------------------------------------------------+
class CMNLReport
{
public:
//--- variables
int m_ngrad;
int m_nhess;
//--- constructor, destructor
CMNLReport(void) { ZeroMemory(this); }
~CMNLReport(void) {}
//--- copy
void Copy(CMNLReport &obj);
};
//+------------------------------------------------------------------+
//| Copy |
//+------------------------------------------------------------------+
void CMNLReport::Copy(CMNLReport &obj)
{
//--- copy variables
m_ngrad=obj.m_ngrad;
m_nhess=obj.m_nhess;
}
//+------------------------------------------------------------------+
//| MNLReport structure contains information about training process: |
//| * NGrad - number of gradient calculations |
//| * NHess - number of Hessian calculations |
//+------------------------------------------------------------------+
class CMNLReportShell
{
private:
CMNLReport m_innerobj;
public:
//--- constructors, destructor
CMNLReportShell(void) {}
CMNLReportShell(CMNLReport &obj) { m_innerobj.Copy(obj); }
~CMNLReportShell(void) {}
//--- methods
int GetNGrad(void);
void SetNGrad(const int i);
int GetNHess(void);
void SetNHess(const int i);
CMNLReport *GetInnerObj(void);
};
//+------------------------------------------------------------------+
//| Returns the value of the variable ngrad |
//+------------------------------------------------------------------+
int CMNLReportShell::GetNGrad(void)
{
return(m_innerobj.m_ngrad);
}
//+------------------------------------------------------------------+
//| Changing the value of the variable ngrad |
//+------------------------------------------------------------------+
void CMNLReportShell::SetNGrad(const int i)
{
m_innerobj.m_ngrad=i;
}
//+------------------------------------------------------------------+
//| Returns the value of the variable nhess |
//+------------------------------------------------------------------+
int CMNLReportShell::GetNHess(void)
{
return(m_innerobj.m_nhess);
}
//+------------------------------------------------------------------+
//| Changing the value of the variable nhess |
//+------------------------------------------------------------------+
void CMNLReportShell::SetNHess(const int i)
{
m_innerobj.m_nhess=i;
}
//+------------------------------------------------------------------+
//| Return object of class |
//+------------------------------------------------------------------+
CMNLReport *CMNLReportShell::GetInnerObj(void)
{
return(GetPointer(m_innerobj));
}
//+------------------------------------------------------------------+
//| Class logit model |
//+------------------------------------------------------------------+
class CLogit
{
public:
//--- variables
static const double m_xtol;
static const double m_ftol;
static const double m_gtol;
static const int m_maxfev;
static const double m_stpmin;
static const double m_stpmax;
static const int m_logitvnum;
//--- public methods
static void MNLTrainH(CMatrixDouble &xy,const int npoints,const int nvars,const int nclasses,int &info,CLogitModel &lm,CMNLReport &rep);
static void MNLProcess(CLogitModel &lm,double &x[],double &y[]);
static void MNLProcess(CLogitModel &lm,CRowDouble &x,CRowDouble &y);
static void MNLProcessI(CLogitModel &lm,double &x[],double &y[]);
static void MNLProcessI(CLogitModel &lm,CRowDouble &x,CRowDouble &y);
static void MNLUnpack(CLogitModel &lm,CMatrixDouble &a,int &nvars,int &nclasses);
static void MNLPack(CMatrixDouble &a,const int nvars,const int nclasses,CLogitModel &lm);
static void MNLCopy(CLogitModel &lm1,CLogitModel &lm2);
static double MNLAvgCE(CLogitModel &lm,CMatrixDouble &xy,const int npoints);
static double MNLRelClsError(CLogitModel &lm,CMatrixDouble &xy,const int npoints);
static double MNLRMSError(CLogitModel &lm,CMatrixDouble &xy,const int npoints);
static double MNLAvgError(CLogitModel &lm,CMatrixDouble &xy,const int npoints);
static double MNLAvgRelError(CLogitModel &lm,CMatrixDouble &xy,const int ssize);
static int MNLClsError(CLogitModel &lm,CMatrixDouble &xy,const int npoints);
private:
static void MNLIExp(CRowDouble &w,CRowDouble &x);
static void MNLAllErrors(CLogitModel &lm,CMatrixDouble &xy,const int npoints,double &relcls,double &avgce,double &rms,double &avg,double &avgrel);
static void MNLMCSrch(const int n,CRowDouble &x,double &f,CRowDouble &g,CRowDouble &s,double &stp,int &info,int &nfev,CRowDouble &wa,CLogitMCState &state,int &stage);
static void MNLMCStep(double &stx,double &fx,double &dx,double &sty,double &fy,double &dy,double &stp,const double fp,const double dp,bool &brackt,const double stmin,const double stmax,int &info);
};
//+------------------------------------------------------------------+
//| Initialize constants |
//+------------------------------------------------------------------+
const double CLogit::m_xtol=100*CMath::m_machineepsilon;
const double CLogit::m_ftol=0.0001;
const double CLogit::m_gtol=0.3;
const int CLogit::m_maxfev=20;
const double CLogit::m_stpmin=1.0E-2;
const double CLogit::m_stpmax=1.0E5;
const int CLogit::m_logitvnum=6;
//+------------------------------------------------------------------+
//| This subroutine trains logit model. |
//| INPUT PARAMETERS: |
//| XY - training set, array[0..NPoints-1,0..NVars] |
//| First NVars columns store values of |
//| independent variables, next column stores |
//| number of class (from 0 to NClasses-1) which |
//| dataset element belongs to. Fractional values|
//| are rounded to nearest integer. |
//| NPoints - training set size, NPoints>=1 |
//| NVars - number of independent variables, NVars>=1 |
//| NClasses - number of classes, NClasses>=2 |
//| OUTPUT PARAMETERS: |
//| Info - return code: |
//| * -2, if there is a point with class number |
//| outside of [0..NClasses-1]. |
//| * -1, if incorrect parameters was passed |
//| (NPoints<NVars+2, NVars<1, NClasses<2).|
//| * 1, if task has been solved |
//| LM - model built |
//| Rep - training report |
//+------------------------------------------------------------------+
void CLogit::MNLTrainH(CMatrixDouble &xy,const int npoints,
const int nvars,const int nclasses,
int &info,CLogitModel &lm,CMNLReport &rep)
{
//--- create variables
int i=0;
int j=0;
int k=0;
int ssize=0;
bool allsame;
int offs=0;
double threshold=0;
double wminstep=0;
double decay=0;
int wdim=0;
int expoffs=0;
double v=0;
double s=0;
int nin=0;
int nout=0;
int wcount=0;
double e=0;
bool spd;
double wstep=0;
int mcstage=0;
int mcinfo=0;
int mcnfev=0;
int solverinfo=0;
int i_=0;
int i1_=0;
//--- creating arrays
CRowDouble g;
vector<double> x;
vector<double> y;
vector<double> wbase;
CRowDouble wdir;
CRowDouble work;
//--- create matrix
CMatrixDouble h;
//--- create objects of classes
CLogitMCState mcstate;
CDenseSolverReport solverrep;
CMultilayerPerceptron network;
//--- initialization
info=0;
threshold=1000*CMath::m_machineepsilon;
wminstep=0.001;
decay=0.001;
//--- Test for inputs
if((npoints<nvars+2 || nvars<1) || nclasses<2)
{
info=-1;
return;
}
for(i=0; i<npoints; i++)
{
//--- check
if((int)MathRound(xy.Get(i,nvars))<0 || (int)MathRound(xy.Get(i,nvars))>=nclasses)
{
info=-2;
return;
}
}
//--- change value
info=1;
//--- Initialize data
rep.m_ngrad=0;
rep.m_nhess=0;
//--- Allocate array
wdim=(nvars+1)*(nclasses-1);
offs=5;
expoffs=offs+wdim;
ssize=5+(nvars+1)*(nclasses-1)+nclasses;
//--- allocation
lm.m_w.Resize(ssize);
//--- change values
lm.m_w.Set(0,ssize);
lm.m_w.Set(1,m_logitvnum);
lm.m_w.Set(2,nvars);
lm.m_w.Set(3,nclasses);
lm.m_w.Set(4,offs);
//--- Degenerate case: all outputs are equal
allsame=true;
for(i=1; i<npoints; i++)
{
//--- check
if((int)MathRound(xy.Get(i,nvars))!=(int)MathRound(xy.Get(i-1,nvars)))
allsame=false;
}
//--- check
if(allsame)
{
for(i=0; i<=(nvars+1)*(nclasses-1)-1; i++)
lm.m_w.Set(offs+i,0);
//--- change values
v=-(2*MathLog(CMath::m_minrealnumber));
k=(int)MathRound(xy.Get(0,nvars));
//--- check
if(k==nclasses-1)
{
for(i=0; i<nclasses-1; i++)
lm.m_w.Set(offs+i*(nvars+1)+nvars,-v);
}
else
{
for(i=0; i<nclasses-1; i++)
//--- check
if(i==k)
lm.m_w.Set(offs+i*(nvars+1)+nvars,v);
else
lm.m_w.Set(offs+i*(nvars+1)+nvars,0);
}
//--- exit the function
return;
}
//--- General case.
//--- Prepare task and network. Allocate space.
CMLPBase::MLPCreateC0(nvars,nclasses,network);
//--- function call
CMLPBase::MLPInitPreprocessor(network,xy,npoints);
//--- function call
CMLPBase::MLPProperties(network,nin,nout,wcount);
for(i=0; i<=wcount-1; i++)
network.m_weights.Set(i,(2*CMath::RandomReal()-1)/nvars);
//--- allocation
g.Resize(wcount);
h.Resize(wcount,wcount);
wbase.Resize(wcount);
wdir.Resize(wcount);
work.Resize(wcount);
//--- First stage: optimize in gradient direction.
for(k=0; k<=wcount/3+10; k++)
{
//--- Calculate gradient in starting point
CMLPBase::MLPGradNBatch(network,xy,npoints,e,g);
v=network.m_weights.Dot(network.m_weights);
//--- change value
e=e+0.5*decay*v;
g+=network.m_weights*decay+0;
rep.m_ngrad++;
//--- Setup optimization scheme
wdir=g*(-1.0)+0;
v=wdir.Dot(wdir);
//--- change values
wstep=MathSqrt(v);
wdir/=MathSqrt(v);
mcstage=0;
//--- function call
MNLMCSrch(wcount,network.m_weights,e,g,wdir,wstep,mcinfo,mcnfev,work,mcstate,mcstage);
//--- cycle
while(mcstage!=0)
{
//--- function call
CMLPBase::MLPGradNBatch(network,xy,npoints,e,g);
v=network.m_weights.Dot(network.m_weights);
//--- change value
e=e+0.5*decay*v;
g+=network.m_weights*decay+0;
rep.m_ngrad++;
//--- function call
MNLMCSrch(wcount,network.m_weights,e,g,wdir,wstep,mcinfo,mcnfev,work,mcstate,mcstage);
}
}
//--- Second stage: use Hessian when we are close to the minimum
while(true)
{
//--- Calculate and update E/G/H
CMLPBase::MLPHessianNBatch(network,xy,npoints,e,g,h);
v=network.m_weights.Dot(network.m_weights);
//--- change value
e=e+0.5*decay*v;
g+=network.m_weights*decay+0;
h.Diag(h.Diag(0)+decay,0);
rep.m_nhess++;
//--- Select step direction
//--- NOTE: it is important to use lower-triangle Cholesky
//--- factorization since it is much faster than higher-triangle version.
spd=CTrFac::SPDMatrixCholesky(h,wcount,false);
//--- function call
CDenseSolver::SPDMatrixCholeskySolve(h,wcount,false,g,solverinfo,solverrep,wdir);
spd=solverinfo>0;
//--- check
if(spd)
{
//--- H is positive definite.
//--- Step in Newton direction.
wdir*=(-1.0);
spd=true;
}
else
{
//--- H is indefinite.
//--- Step in gradient direction.
wdir=g;
wdir*=(-1.0);
spd=false;
}
//--- Optimize in WDir direction
v=wdir.Dot(wdir);
//--- change values
wstep=MathSqrt(v);
wdir/=MathSqrt(v);
mcstage=0;
//--- function call
MNLMCSrch(wcount,network.m_weights,e,g,wdir,wstep,mcinfo,mcnfev,work,mcstate,mcstage);
//--- cycle
while(mcstage!=0)
{
//--- function call
CMLPBase::MLPGradNBatch(network,xy,npoints,e,g);
v=network.m_weights.Dot(network.m_weights);
//--- change value
e=e+0.5*decay*v;
g+=network.m_weights*decay+0;
rep.m_ngrad++;
//--- function call
MNLMCSrch(wcount,network.m_weights,e,g,wdir,wstep,mcinfo,mcnfev,work,mcstate,mcstage);
}
//--- check
if(spd && ((mcinfo==2 || mcinfo==4) || mcinfo==6))
break;
}
//--- Convert from NN format to MNL format
i1_=-offs;
for(i_=offs; i_<offs+wcount; i_++)
lm.m_w.Set(i_,network.m_weights[i_+i1_]);
for(k=0; k<nvars; k++)
{
for(i=0; i<nclasses-1; i++)
{
s=network.m_columnsigmas[k];
//--- check
if(s==0.0)
s=1;
//--- change values
j=offs+(nvars+1)*i;
v=lm.m_w[j+k];
lm.m_w.Set(j+k,v/s);
lm.m_w.Add(j+nvars,v*network.m_columnmeans[k]/s);
}
}
//--- calculation
for(k=0; k<nclasses-1; k++)
lm.m_w.Mul(offs+(nvars+1)*k+nvars,-1.0);
}
//+------------------------------------------------------------------+
//| Procesing |
//| INPUT PARAMETERS: |
//| LM - logit model, passed by non-constant reference |
//| (some fields of structure are used as temporaries|
//| when calculating model output). |
//| X - input vector, array[0..NVars-1]. |
//| Y - (possibly) preallocated buffer; if size of Y is |
//| less than NClasses, it will be reallocated.If it |
//| is large enough, it is NOT reallocated, so we |
//| can save some time on reallocation. |
//| OUTPUT PARAMETERS: |
//| Y - result, array[0..NClasses-1] |
//| Vector of posterior probabilities for |
//| classification task. |
//+------------------------------------------------------------------+
void CLogit::MNLProcess(CLogitModel &lm,double &x[],double &y[])
{
CRowDouble X=x;
CRowDouble Y;
MNLProcess(lm,X,Y);
Y.ToArray(y);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CLogit::MNLProcess(CLogitModel &lm,CRowDouble &x,CRowDouble &y)
{
//--- create variables
int nvars=(int)MathRound(lm.m_w[2]);
int nclasses=(int)MathRound(lm.m_w[3]);
int offs=(int)MathRound(lm.m_w[4]);
int i1=0;
double s=0;
//--- check
if(!CAp::Assert(lm.m_w[1]==m_logitvnum,__FUNCTION__+": unexpected model version"))
return;
//--- function call
MNLIExp(lm.m_w,x);
s=0;
//--- calculation
i1=offs+(nvars+1)*(nclasses-1);
for(int i=i1; i<(i1+nclasses); i++)
s+=lm.m_w[i];
//--- check
if(CAp::Len(y)<nclasses)
y.Resize(nclasses);
//--- change values
for(int i=0; i<=nclasses-1; i++)
y.Set(i,lm.m_w[i1+i]/s);
}
//+------------------------------------------------------------------+
//| 'interactive' variant of MNLProcess for languages like Python |
//| which support constructs like "Y=MNLProcess(LM,X)" and |
//| interactive mode of the interpreter |
//| This function allocates new array on each call, so it is |
//| significantly slower than its 'non-interactive' counterpart, |
//| but it is more convenient when you call it from command line. |
//+------------------------------------------------------------------+
void CLogit::MNLProcessI(CLogitModel &lm,double &x[],double &y[])
{
MNLProcess(lm,x,y);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CLogit::MNLProcessI(CLogitModel &lm,CRowDouble &x,CRowDouble &y)
{
MNLProcess(lm,x,y);
}
//+------------------------------------------------------------------+
//| Unpacks coefficients of logit model. Logit model have form: |
//| P(class=i) = S(i) / (S(0) + S(1) + ... +S(M-1)) |
//| S(i) = Exp(A[i,0]*X[0] + ... + A[i,N-1]*X[N-1] + A[i,N]), |
//| when i<M-1 |
//| S(M-1) = 1 |
//| INPUT PARAMETERS: |
//| LM - logit model in ALGLIB format |
//| OUTPUT PARAMETERS: |
//| V - coefficients, array[0..NClasses-2,0..NVars] |
//| NVars - number of independent variables |
//| NClasses - number of classes |
//+------------------------------------------------------------------+
void CLogit::MNLUnpack(CLogitModel &lm,CMatrixDouble &a,int &nvars,
int &nclasses)
{
//--- create variables
int offs=0;
int i1_=0;
//--- initialization
nvars=0;
nclasses=0;
//--- check
if(!CAp::Assert(lm.m_w[1]==m_logitvnum,__FUNCTION__+": unexpected model version"))
return;
//--- initialization
nvars=(int)MathRound(lm.m_w[2]);
nclasses=(int)MathRound(lm.m_w[3]);
offs=(int)MathRound(lm.m_w[4]);
//--- allocation
a.Resize(nclasses-1,nvars+1);
//--- calculation
for(int i=0; i<=nclasses-2; i++)
{
i1_=offs+i*(nvars+1);
for(int i_=0; i_<=nvars; i_++)
a.Set(i,i_,lm.m_w[i_+i1_]);
}
}
//+------------------------------------------------------------------+
//| "Packs" coefficients and creates logit model in ALGLIB format |
//| (MNLUnpack reversed). |
//| INPUT PARAMETERS: |
//| A - model (see MNLUnpack) |
//| NVars - number of independent variables |
//| NClasses - number of classes |
//| OUTPUT PARAMETERS: |
//| LM - logit model. |
//+------------------------------------------------------------------+
void CLogit::MNLPack(CMatrixDouble &a,const int nvars,const int nclasses,
CLogitModel &lm)
{
//--- create variables
int offs=5;
int wdim=(nvars+1)*(nclasses-1);
int ssize=5+(nvars+1)*(nclasses-1)+nclasses;
int i1_=0;
//--- allocation
lm.m_w.Resize(ssize);
//--- initialization
lm.m_w.Set(0,ssize);
lm.m_w.Set(1,m_logitvnum);
lm.m_w.Set(2,nvars);
lm.m_w.Set(3,nclasses);
lm.m_w.Set(4,offs);
//--- calculation
for(int i=0; i<nclasses-1; i++)
{
i1_=-(offs+i*(nvars+1));
for(int i_=offs+i*(nvars+1); i_<=offs+i*(nvars+1)+nvars; i_++)
lm.m_w.Set(i_,a.Get(i,i_+i1_));
}
}
//+------------------------------------------------------------------+
//| Copying of LogitModel strucure |
//| INPUT PARAMETERS: |
//| LM1 - original |
//| OUTPUT PARAMETERS: |
//| LM2 - copy |
//+------------------------------------------------------------------+
void CLogit::MNLCopy(CLogitModel &lm1,CLogitModel &lm2)
{
//--- initialization
int k=(int)MathRound(lm1.m_w[0]);
//--- allocation
lm2.m_w.Resize(k);
//--- copy
lm2.m_w=lm1.m_w;
}
//+------------------------------------------------------------------+
//| Average cross-entropy (in bits per element) on the test set |
//| INPUT PARAMETERS: |
//| LM - logit model |
//| XY - test set |
//| NPoints - test set size |
//| RESULT: |
//| CrossEntropy/(NPoints*ln(2)). |
//+------------------------------------------------------------------+
double CLogit::MNLAvgCE(CLogitModel &lm,CMatrixDouble &xy,const int npoints)
{
//--- create variables
double result=0;
int nvars=(int)MathRound(lm.m_w[2]);
int nclasses=(int)MathRound(lm.m_w[3]);
//--- creating arrays
CRowDouble workx;
CRowDouble worky;
//--- check
if(!CAp::Assert(lm.m_w[1]==m_logitvnum,__FUNCTION__+": unexpected model version"))
return(EMPTY_VALUE);
//--- allocation
worky.Resize(nclasses);
//--- calculation
for(int i=0; i<npoints; i++)
{
//--- check
if(!CAp::Assert((int)MathRound(xy[i][nvars])>=0 && (int)MathRound(xy[i][nvars])<nclasses,__FUNCTION__+": incorrect class number!"))
return(EMPTY_VALUE);
//--- Process
workx=xy[i]+0;
//--- function call
MNLProcess(lm,workx,worky);
//--- check
if(worky[(int)MathRound(xy.Get(i,nvars))]>0.0)
result-=MathLog(worky[(int)MathRound(xy.Get(i,nvars))]);
else
result-=MathLog(CMath::m_minrealnumber);
}
//--- return result
return(result/(npoints*MathLog(2)));
}
//+------------------------------------------------------------------+
//| Relative classification error on the test set |
//| INPUT PARAMETERS: |
//| LM - logit model |
//| XY - test set |
//| NPoints - test set size |
//| RESULT: |
//| percent of incorrectly classified cases. |
//+------------------------------------------------------------------+
double CLogit::MNLRelClsError(CLogitModel &lm,CMatrixDouble &xy,
const int npoints)
{
return((double)MNLClsError(lm,xy,npoints)/(double)npoints);
}
//+------------------------------------------------------------------+
//| RMS error on the test set |
//| INPUT PARAMETERS: |
//| LM - logit model |
//| XY - test set |
//| NPoints - test set size |
//| RESULT: |
//| root mean square error (error when estimating posterior |
//| probabilities). |
//+------------------------------------------------------------------+
double CLogit::MNLRMSError(CLogitModel &lm,CMatrixDouble &xy,
const int npoints)
{
//--- create variables
double relcls=0;
double avgce=0;
double rms=0;
double avg=0;
double avgrel=0;
//--- check
if(!CAp::Assert((int)MathRound(lm.m_w[1])==m_logitvnum,__FUNCTION__+": Incorrect MNL version!"))
return(EMPTY_VALUE);
//--- function call
MNLAllErrors(lm,xy,npoints,relcls,avgce,rms,avg,avgrel);
//--- return result
return(rms);
}
//+------------------------------------------------------------------+
//| Average error on the test set |
//| INPUT PARAMETERS: |
//| LM - logit model |
//| XY - test set |
//| NPoints - test set size |
//| RESULT: |
//| average error (error when estimating posterior |
//| probabilities). |
//+------------------------------------------------------------------+
double CLogit::MNLAvgError(CLogitModel &lm,CMatrixDouble &xy,
const int npoints)
{
//--- create variables
double relcls=0;
double avgce=0;
double rms=0;
double avg=0;
double avgrel=0;
//--- check
if(!CAp::Assert((int)MathRound(lm.m_w[1])==m_logitvnum,__FUNCTION__+": Incorrect MNL version!"))
return(EMPTY_VALUE);
//--- function call
MNLAllErrors(lm,xy,npoints,relcls,avgce,rms,avg,avgrel);
//--- return result
return(avg);
}
//+------------------------------------------------------------------+
//| Average relative error on the test set |
//| INPUT PARAMETERS: |
//| LM - logit model |
//| XY - test set |
//| NPoints - test set size |
//| RESULT: |
//| average relative error (error when estimating posterior |
//| probabilities). |
//+------------------------------------------------------------------+
double CLogit::MNLAvgRelError(CLogitModel &lm,CMatrixDouble &xy,
const int ssize)
{
//--- create variables
double relcls=0;
double avgce=0;
double rms=0;
double avg=0;
double avgrel=0;
//--- check
if(!CAp::Assert((int)MathRound(lm.m_w[1])==m_logitvnum,__FUNCTION__+": Incorrect MNL version!"))
return(EMPTY_VALUE);
//--- function call
MNLAllErrors(lm,xy,ssize,relcls,avgce,rms,avg,avgrel);
//--- return result
return(avgrel);
}
//+------------------------------------------------------------------+
//| Classification error on test set = MNLRelClsError*NPoints |
//+------------------------------------------------------------------+
int CLogit::MNLClsError(CLogitModel &lm,CMatrixDouble &xy,const int npoints)
{
//--- create variables
int result=0;
int nvars=(int)MathRound(lm.m_w[2]);
int nclasses=(int)MathRound(lm.m_w[3]);
int nmax=0;
//--- creating arrays
CRowDouble workx;
CRowDouble worky;
//--- check
if(!CAp::Assert(lm.m_w[1]==m_logitvnum,__FUNCTION__+": unexpected model version"))
return(-1);
//--- allocation
worky.Resize(nclasses);
//--- calculation
for(int i=0; i<npoints; i++)
{
//--- Process
workx=xy[i]+0;
//--- function call
MNLProcess(lm,workx,worky);
//--- Logit version of the answer
nmax=0;
for(int j=0; j<nclasses; j++)
{
//--- check
if(worky[j]>worky[nmax])
nmax=j;
}
//--- compare
if(nmax!=(int)MathRound(xy[i][nvars]))
result++;
}
//--- return result
return(result);
}
//+------------------------------------------------------------------+
//| Internal subroutine. Places exponents of the anti-overflow |
//| shifted internal linear outputs into the service part of the W |
//| array. |
//+------------------------------------------------------------------+
void CLogit::MNLIExp(CRowDouble &w,CRowDouble &x)
{
//--- create variables
int nvars=(int)MathRound(w[2]);
int nclasses=(int)MathRound(w[3]);
int offs=(int)MathRound(w[4]);
int i1=0;
double v=0;
double mx=0;
int i1_=0;
//--- check
if(!CAp::Assert(w[1]==m_logitvnum,__FUNCTION__+": unexpected model version"))
return;
//--- calculation
i1=offs+(nvars+1)*(nclasses-1);
for(int i=0; i<nclasses-1; i++)
{
//--- change values
i1_=-(offs+i*(nvars+1));
v=0.0;
for(int i_=offs+i*(nvars+1); i_<=offs+i*(nvars+1)+nvars-1; i_++)
v+=w[i_]*x[i_+i1_];
w.Set(i1+i,v+w[offs+i*(nvars+1)+nvars]);
}
//--- change values
w.Set(i1+nclasses-1,0);
mx=0;
//--- calculation
for(int i=i1; i<i1+nclasses; i++)
mx=MathMax(mx,w[i]);
for(int i=i1; i<i1+nclasses; i++)
w.Set(i,MathExp(w[i]-mx));
}
//+------------------------------------------------------------------+
//| Calculation of all types of errors |
//+------------------------------------------------------------------+
void CLogit::MNLAllErrors(CLogitModel &lm,CMatrixDouble &xy,
const int npoints,double &relcls,
double &avgce,double &rms,double &avg,
double &avgrel)
{
//--- create variables
int nvars=(int)MathRound(lm.m_w[2]);
int nclasses=(int)MathRound(lm.m_w[3]);
//--- creating arrays
CRowDouble buf;
CRowDouble workx;
CRowDouble y;
CRowDouble dy;
//--- initialization
relcls=0;
avgce=0;
rms=0;
avg=0;
avgrel=0;
//--- check
if(!CAp::Assert((int)MathRound(lm.m_w[1])==m_logitvnum,__FUNCTION__+": Incorrect MNL version!"))
return;
//--- allocation
workx.Resize(nvars);
y.Resize(nclasses);
dy.Resize(1);
//--- function call
CBdSS::DSErrAllocate(nclasses,buf);
for(int i=0; i<npoints; i++)
{
workx=xy[i]+0;
//--- function call
MNLProcess(lm,workx,y);
//--- change value
dy.Set(0,xy.Get(i,nvars));
//--- function call
CBdSS::DSErrAccumulate(buf,y,dy);
}
//--- function call
CBdSS::DSErrFinish(buf);
//--- change values
relcls=buf[0];
avgce=buf[1];
rms=buf[2];
avg=buf[3];
avgrel=buf[4];
}
//+------------------------------------------------------------------+
//| The purpose of mcsrch is to find a step which satisfies a |
//| sufficient decrease condition and a curvature condition. |
//| At each stage the subroutine updates an interval of uncertainty |
//| with endpoints stx and sty. The interval of uncertainty is |
//| initially chosen so that it contains a minimizer of the modified |
//| function |
//| f(x+stp*s) - f(x) - ftol*stp*(gradf(x)'s). |
//| If a step is obtained for which the modified function has a |
//| nonpositive function value and nonnegative derivative, then the |
//| interval of uncertainty is chosen so that it contains a minimizer|
//| of f(x+stp*s). |
//| The algorithm is designed to find a step which satisfies the |
//| sufficient decrease condition |
//| f(x+stp*s) .le. f(x) + ftol*stp*(gradf(x)'s), |
//| and the curvature condition |
//| abs(gradf(x+stp*s)'s)) .le. gtol*abs(gradf(x)'s). |
//| If ftol is less than gtol and if, for example, the function is |
//| bounded below, then there is always a step which satisfies both |
//| conditions. If no step can be found which satisfies both |
//| conditions, then the algorithm usually stops when rounding |
//| errors prevent further progress. In this case stp only satisfies |
//| the sufficient decrease condition. |
//| Parameters descriprion |
//| N is a positive integer input variable set to the number of |
//| variables. |
//| X is an array of length n. on input it must contain the base |
//| point for the line search. on output it contains x+stp*s. |
//| F is a variable. on input it must contain the value of f at x. On|
//| output it contains the value of f at x + stp*s. |
//| G is an array of length n. on input it must contain the gradient |
//| of f at x. On output it contains the gradient of f at x + stp*s. |
//| s is an input array of length n which specifies the search |
//| direction. |
//| Stp is a nonnegative variable. on input stp contains an initial|
//| estimate of a satisfactory step. on output stp contains the final|
//| estimate. |
//| Ftol and gtol are nonnegative input variables. termination occurs|
//| when the sufficient decrease condition and the directional |
//| derivative condition are satisfied. |
//| Xtol is a nonnegative input variable. termination occurs when the|
//| relative width of the interval of uncertainty is at most xtol. |
//| Stpmin and stpmax are nonnegative input variables which specify |
//| lower and upper bounds for the step. |
//| Maxfev is a positive integer input variable. termination occurs |
//| when the number of calls to fcn is at least maxfev by the end of |
//| an iteration. |
//| Info is an integer output variable set as follows: |
//| info = 0 improper input parameters. |
//| info = 1 the sufficient decrease condition and the |
//| directional derivative condition hold. |
//| info = 2 relative width of the interval of uncertainty |
//| is at most xtol. |
//| info = 3 number of calls to fcn has reached maxfev. |
//| info = 4 the step is at the lower bound stpmin. |
//| info = 5 the step is at the upper bound stpmax. |
//| info = 6 rounding errors prevent further progress. |
//| there may not be a step which satisfies the |
//| sufficient decrease and curvature conditions. |
//| tolerances may be too small. |
//| Nfev is an integer output variable set to the number of calls to |
//| fcn. |
//| wa is a work array of length n. |
//| argonne national laboratory. minpack project. june 1983 |
//| Jorge J. More', David J. Thuente |
//+------------------------------------------------------------------+
void CLogit::MNLMCSrch(const int n,CRowDouble &x,double &f,CRowDouble &g,
CRowDouble &s,double &stp,int &info,int &nfev,
CRowDouble &wa,CLogitMCState &state,int &stage)
{
//--- create variables
double v=0;
double p5=0;
double p66=0;
double zero=0;
//--- init
p5=0.5;
p66=0.66;
state.m_xtrapf=4.0;
zero=0;
//--- Main cycle
while(true)
{
//--- check
switch(stage)
{
case 0:
//--- NEXT
stage=2;
break;
//--- check
case 2:
state.m_infoc=1;
info=0;
//--- CHECK THE INPUT PARAMETERS FOR ERRORS.
if(n<=0 || stp<=0.0 || m_ftol<0.0 || m_gtol<zero || m_xtol<zero || m_stpmin<zero || m_stpmax<m_stpmin || m_maxfev<=0)
{
stage=0;
return;
}
//--- compute the initial gradient in the search direction
//--- and check that s is a descent direction.
v=g.Dot(s);
state.m_dginit=v;
//--- check
if(state.m_dginit>=0.0)
{
stage=0;
return;
}
//--- initialize local variables.
state.m_brackt=false;
state.m_stage1=true;
nfev=0;
state.m_finit=f;
state.m_dgtest=m_ftol*state.m_dginit;
state.m_width=m_stpmax-m_stpmin;
state.m_width1=state.m_width/p5;
wa=x;
//--- the variables stx,fx,dgx contain the values of the step,
//--- function,and directional derivative at the best step.
//--- the variables sty,fy,dgy contain the value of the step,
//--- function,and derivative at the other endpoint of
//--- the interval of uncertainty.
//--- the variables stp,f,dg contain the values of the step,
//--- function,and derivative at the current step.
state.m_stx=0;
state.m_fx=state.m_finit;
state.m_dgx=state.m_dginit;
state.m_sty=0;
state.m_fy=state.m_finit;
state.m_dgy=state.m_dginit;
//--- NEXT
stage=3;
break;
case 3:
//--- start of iteration.
//--- set the minimum and maximum steps to correspond
//--- to the present interval of uncertainty.
if(state.m_brackt)
{
//--- check
if(state.m_stx<state.m_sty)
{
state.m_stmin=state.m_stx;
state.m_stmax=state.m_sty;
}
else
{
state.m_stmin=state.m_sty;
state.m_stmax=state.m_stx;
}
}
else
{
state.m_stmin=state.m_stx;
state.m_stmax=stp+state.m_xtrapf*(stp-state.m_stx);
}
//--- force the step to be within the bounds stpmax and stpmin.
if(stp>m_stpmax)
stp=m_stpmax;
//--- check
if(stp<m_stpmin)
stp=m_stpmin;
//--- if an unusual termination is to occur then let
//--- stp be the lowest point obtained so far.
if((((state.m_brackt && (stp<=state.m_stmin || stp>=state.m_stmax)) || nfev>=m_maxfev-1) || state.m_infoc==0) || (state.m_brackt && state.m_stmax-state.m_stmin<=m_xtol*state.m_stmax))
stp=state.m_stx;
//--- evaluate the function and gradient at stp
//--- and compute the directional derivative.
x=wa+s*stp+0;
//--- next
stage=4;
return;
case 4:
info=0;
nfev++;
//--- calculation
v=g.Dot(s);
state.m_dg=v;
state.m_ftest1=state.m_finit+stp*state.m_dgtest;
//--- test for convergence.
if((state.m_brackt && (stp<=state.m_stmin || stp>=state.m_stmax)) || state.m_infoc==0)
info=6;
//--- check
if((stp==m_stpmax && f<=state.m_ftest1) && state.m_dg<=state.m_dgtest)
info=5;
//--- check
if(stp==m_stpmin && (f>state.m_ftest1 || state.m_dg>=state.m_dgtest))
info=4;
//--- check
if(nfev>=m_maxfev)
info=3;
//--- check
if(state.m_brackt && state.m_stmax-state.m_stmin<=m_xtol*state.m_stmax)
info=2;
//--- check
if(f<=state.m_ftest1 && MathAbs(state.m_dg)<=-(m_gtol*state.m_dginit))
info=1;
//--- check for termination.
if(info!=0)
{
stage=0;
return;
}
//--- in the first stage we seek a step for which the modified
//--- function has a nonpositive value and nonnegative derivative.
if((state.m_stage1 && f<=state.m_ftest1) && state.m_dg>=MathMin(m_ftol,m_gtol)*state.m_dginit)
state.m_stage1=false;
//--- a modified function is used to predict the step only if
//--- we have not obtained a step for which the modified
//--- function has a nonpositive function value and nonnegative
//--- derivative,and if a lower function value has been
//--- obtained but the decrease is not sufficient.
if((state.m_stage1 && f<=state.m_fx) && f>state.m_ftest1)
{
//--- define the modified function and derivative values.
state.m_fm=f-stp*state.m_dgtest;
state.m_fxm=state.m_fx-state.m_stx*state.m_dgtest;
state.m_fym=state.m_fy-state.m_sty*state.m_dgtest;
state.m_dgm=state.m_dg-state.m_dgtest;
state.m_dgxm=state.m_dgx-state.m_dgtest;
state.m_dgym=state.m_dgy-state.m_dgtest;
//--- call cstep to update the interval of uncertainty
//--- and to compute the new step.
MNLMCStep(state.m_stx,state.m_fxm,state.m_dgxm,state.m_sty,state.m_fym,state.m_dgym,stp,state.m_fm,state.m_dgm,state.m_brackt,state.m_stmin,state.m_stmax,state.m_infoc);
//--- reset the function and gradient values for f.
state.m_fx=state.m_fxm+state.m_stx*state.m_dgtest;
state.m_fy=state.m_fym+state.m_sty*state.m_dgtest;
state.m_dgx=state.m_dgxm+state.m_dgtest;
state.m_dgy=state.m_dgym+state.m_dgtest;
}
else
{
//--- call mcstep to update the interval of uncertainty
//--- and to compute the new step.
MNLMCStep(state.m_stx,state.m_fx,state.m_dgx,state.m_sty,state.m_fy,state.m_dgy,stp,f,state.m_dg,state.m_brackt,state.m_stmin,state.m_stmax,state.m_infoc);
}
//--- force a sufficient decrease in the size of the
//--- interval of uncertainty.
if(state.m_brackt)
{
//--- check
if(MathAbs(state.m_sty-state.m_stx)>=p66*state.m_width1)
stp=state.m_stx+p5*(state.m_sty-state.m_stx);
state.m_width1=state.m_width;
state.m_width=MathAbs(state.m_sty-state.m_stx);
}
//--- next.
stage=3;
break;
}
}
}
//+------------------------------------------------------------------+
//| Auxiliary function for MNLMCSrch |
//+------------------------------------------------------------------+
void CLogit::MNLMCStep(double &stx,double &fx,double &dx,double &sty,
double &fy,double &dy,double &stp,const double fp,
const double dp,bool &brackt,const double stmin,
const double stmax,int &info)
{
//--- create variables
bool bound;
double gamma=0;
double p=0;
double q=0;
double r=0;
double s=0;
double sgnd=0;
double stpc=0;
double stpf=0;
double stpq=0;
double theta=0;
//--- initialization
info=0;
//--- check the input parameters for errors.
if(((brackt && (stp<=MathMin(stx,sty) || stp>=MathMax(stx,sty))) || dx*(stp-stx)>=0.0) || stmax<stmin)
return;
//--- determine if the derivatives have opposite sign.
sgnd=dp*(dx/MathAbs(dx));
//--- first case. a higher function value.
//--- the minimum is bracketed. if the cubic step is closer
//--- to stx than the quadratic step,the cubic step is taken,
//--- else the average of the cubic and quadratic steps is taken.
if(fp>fx)
{
//--- change value
info=1;
bound=true;
theta=3*(fx-fp)/(stp-stx)+dx+dp;
s=MathMax(MathAbs(theta),MathMax(MathAbs(dx),MathAbs(dp)));
gamma=s*MathSqrt(CMath::Sqr(theta/s)-dx/s*(dp/s));
//--- check
if(stp<stx)
gamma=-gamma;
//--- change value
p=gamma-dx+theta;
q=gamma-dx+gamma+dp;
r=p/q;
stpc=stx+r*(stp-stx);
stpq=stx+dx/((fx-fp)/(stp-stx)+dx)/2*(stp-stx);
//--- check
if(MathAbs(stpc-stx)<MathAbs(stpq-stx))
stpf=stpc;
else
stpf=stpc+(stpq-stpc)/2;
brackt=true;
}
else
{
//--- check
if(sgnd<0.0)
{
//--- second case. a lower function value and derivatives of
//--- opposite sign. the minimum is bracketed. if the cubic
//--- step is closer to stx than the quadratic (secant) step,
//--- the cubic step is taken,else the quadratic step is taken.
info=2;
bound=false;
theta=3*(fx-fp)/(stp-stx)+dx+dp;
s=MathMax(MathAbs(theta),MathMax(MathAbs(dx),MathAbs(dp)));
gamma=s*MathSqrt(CMath::Sqr(theta/s)-dx/s*(dp/s));
//--- check
if(stp>stx)
gamma=-gamma;
//--- change values
p=gamma-dp+theta;
q=gamma-dp+gamma+dx;
r=p/q;
stpc=stp+r*(stx-stp);
stpq=stp+dp/(dp-dx)*(stx-stp);
//--- check
if(MathAbs(stpc-stp)>MathAbs(stpq-stp))
stpf=stpc;
else
stpf=stpq;
brackt=true;
}
else
{
//--- check
if(MathAbs(dp)<MathAbs(dx))
{
//--- third case. a lower function value,derivatives of the
//--- same sign,and the magnitude of the derivative decreases.
//--- the cubic step is only used if the cubic tends to infinity
//--- in the direction of the step or if the minimum of the cubic
//--- is beyond stp. otherwise the cubic step is defined to be
//--- either stpmin or stpmax. the quadratic (secant) step is also
//--- computed and if the minimum is bracketed then the the step
//--- closest to stx is taken,else the step farthest away is taken.
info=3;
bound=true;
theta=3*(fx-fp)/(stp-stx)+dx+dp;
s=MathMax(MathAbs(theta),MathMax(MathAbs(dx),MathAbs(dp)));
//--- the case gamma=0 only arises if the cubic does not tend
//--- to infinity in the direction of the step.
gamma=s*MathSqrt(MathMax(0,CMath::Sqr(theta/s)-dx/s*(dp/s)));
//--- check
if(stp>stx)
gamma=-gamma;
p=gamma-dp+theta;
q=gamma+(dx-dp)+gamma;
r=p/q;
//--- check
if(r<0.0 && gamma!=0.0)
stpc=stp+r*(stx-stp);
else
{
//--- check
if(stp>stx)
stpc=stmax;
else
stpc=stmin;
}
stpq=stp+dp/(dp-dx)*(stx-stp);
//--- check
if(brackt)
{
//--- check
if(MathAbs(stp-stpc)<MathAbs(stp-stpq))
stpf=stpc;
else
stpf=stpq;
}
else
{
//--- check
if(MathAbs(stp-stpc)>MathAbs(stp-stpq))
stpf=stpc;
else
stpf=stpq;
}
}
else
{
//--- fourth case. a lower function value,derivatives of the
//--- same sign,and the magnitude of the derivative does
//--- not decrease. if the minimum is not bracketed,the step
//--- is either stpmin or stpmax,else the cubic step is taken.
info=4;
bound=false;
//--- check
if(brackt)
{
//--- change values
theta=3*(fp-fy)/(sty-stp)+dy+dp;
s=MathMax(MathAbs(theta),MathMax(MathAbs(dy),MathAbs(dp)));
gamma=s*MathSqrt(CMath::Sqr(theta/s)-dy/s*(dp/s));
//--- check
if(stp>sty)
gamma=-gamma;
//--- change values
p=gamma-dp+theta;
q=gamma-dp+gamma+dy;
r=p/q;
stpc=stp+r*(sty-stp);
stpf=stpc;
}
else
{
//--- check
if(stp>stx)
stpf=stmax;
else
stpf=stmin;
}
}
}
}
//--- update the interval of uncertainty. this update does not
//--- depend on the new step or the case analysis above.
if(fp>fx)
{
sty=stp;
fy=fp;
dy=dp;
}
else
{
//--- check
if(sgnd<0.0)
{
sty=stx;
fy=fx;
dy=dx;
}
//--- change values
stx=stp;
fx=fp;
dx=dp;
}
//--- compute the new step and safeguard it.
stpf=MathMin(stmax,stpf);
stpf=MathMax(stmin,stpf);
stp=stpf;
//--- check
if(brackt && bound)
{
//--- check
if(sty>stx)
stp=MathMin(stx+0.66*(sty-stx),stp);
else
stp=MathMax(stx+0.66*(sty-stx),stp);
}
}
//+------------------------------------------------------------------+
//| This structure is a MCPD (Markov Chains for Population Data) |
//| solver. You should use ALGLIB functions in order to work with |
//| this object. |
//+------------------------------------------------------------------+
class CMCPDState
{
public:
//--- variables
int m_n;
int m_npairs;
int m_ccnt;
double m_regterm;
int m_repinneriterationscount;
int m_repouteriterationscount;
int m_repnfev;
int m_repterminationtype;
CMinBLEICState m_bs;
CMinBLEICReport m_br;
//--- arrays
CRowInt m_states;
CRowInt m_ct;
CRowDouble m_pw;
CRowDouble m_tmpp;
CRowDouble m_effectivew;
CRowDouble m_effectivebndl;
CRowDouble m_effectivebndu;
CRowInt m_effectivect;
CRowDouble m_h;
//--- matrices
CMatrixDouble m_data;
CMatrixDouble m_ec;
CMatrixDouble m_bndl;
CMatrixDouble m_bndu;
CMatrixDouble m_c;
CMatrixDouble m_priorp;
CMatrixDouble m_effectivec;
CMatrixDouble m_p;
//--- constructor, destructor
CMCPDState(void);
~CMCPDState(void) {}
//--- copy
void Copy(const CMCPDState &obj);
//--- overloading
void operator=(const CMCPDState &obj) { Copy(obj); }
};
//+------------------------------------------------------------------+
//| Constructor |
//+------------------------------------------------------------------+
CMCPDState::CMCPDState(void)
{
//--- copy variables
m_n=0;
m_npairs=0;
m_ccnt=0;
m_regterm=0;
m_repinneriterationscount=0;
m_repouteriterationscount=0;
m_repnfev=0;
m_repterminationtype=0;
}
//+------------------------------------------------------------------+
//| Copy |
//+------------------------------------------------------------------+
void CMCPDState::Copy(const CMCPDState &obj)
{
//--- copy variables
m_n=obj.m_n;
m_npairs=obj.m_npairs;
m_ccnt=obj.m_ccnt;
m_regterm=obj.m_regterm;
m_repinneriterationscount=obj.m_repinneriterationscount;
m_repouteriterationscount=obj.m_repouteriterationscount;
m_repnfev=obj.m_repnfev;
m_repterminationtype=obj.m_repterminationtype;
m_bs.Copy(obj.m_bs);
m_br.Copy(obj.m_br);
//--- copy arrays
m_states=obj.m_states;
m_ct=obj.m_ct;
m_pw=obj.m_pw;
m_tmpp=obj.m_tmpp;
m_effectivew=obj.m_effectivew;
m_effectivebndl=obj.m_effectivebndl;
m_effectivebndu=obj.m_effectivebndu;
m_effectivect=obj.m_effectivect;
m_h=obj.m_h;
//--- copy matrices
m_data=obj.m_data;
m_ec=obj.m_ec;
m_bndl=obj.m_bndl;
m_bndu=obj.m_bndu;
m_c=obj.m_c;
m_priorp=obj.m_priorp;
m_effectivec=obj.m_effectivec;
m_p=obj.m_p;
}
//+------------------------------------------------------------------+
//| This structure is a MCPD (Markov Chains for Population Data) |
//| solver. |
//| You should use ALGLIB functions in order to work with this object|
//+------------------------------------------------------------------+
class CMCPDStateShell
{
private:
CMCPDState m_innerobj;
public:
//--- constructors, destructor
CMCPDStateShell(void) {}
CMCPDStateShell(CMCPDState &obj) { m_innerobj.Copy(obj); }
~CMCPDStateShell(void) {}
//--- method
CMCPDState *GetInnerObj(void) { return(GetPointer(m_innerobj)); }
};
//+------------------------------------------------------------------+
//| This structure is a MCPD training report: |
//| InnerIterationsCount - number of inner iterations of the|
//| underlying optimization algorithm|
//| OuterIterationsCount - number of outer iterations of the|
//| underlying optimization algorithm|
//| NFEV - number of merit function |
//| evaluations |
//| TerminationType - termination type |
//| (same as for MinBLEIC optimizer, |
//| positive values denote success, |
//| negative ones - failure) |
//+------------------------------------------------------------------+
class CMCPDReport
{
public:
//--- variables
int m_inneriterationscount;
int m_outeriterationscount;
int m_nfev;
int m_terminationtype;
//--- constructor, destructor
CMCPDReport(void) { ZeroMemory(this); }
~CMCPDReport(void) {}
//--- copy
void Copy(const CMCPDReport &obj);
//--- overloading
void operator=(const CMCPDReport &obj) { Copy(obj); }
};
//+------------------------------------------------------------------+
//| Copy |
//+------------------------------------------------------------------+
void CMCPDReport::Copy(const CMCPDReport &obj)
{
//--- copy variables
m_inneriterationscount=obj.m_inneriterationscount;
m_outeriterationscount=obj.m_outeriterationscount;
m_nfev=obj.m_nfev;
m_terminationtype=obj.m_terminationtype;
}
//+------------------------------------------------------------------+
//| This structure is a MCPD training report: |
//| InnerIterationsCount - number of inner iterations of the|
//| underlying optimization algorithm|
//| OuterIterationsCount - number of outer iterations of the|
//| underlying optimization algorithm|
//| NFEV - number of merit function |
//| evaluations |
//| TerminationType - termination type |
//| (same as for MinBLEIC optimizer, |
//| positive values denote success, |
//| negative ones - failure) |
//+------------------------------------------------------------------+
class CMCPDReportShell
{
private:
CMCPDReport m_innerobj;
public:
//--- constructors, destructor
CMCPDReportShell(void) {}
CMCPDReportShell(CMCPDReport &obj) { m_innerobj.Copy(obj); }
~CMCPDReportShell(void) {}
//--- methods
int GetInnerIterationsCount(void);
void SetInnerIterationsCount(const int i);
int GetOuterIterationsCount(void);
void SetOuterIterationsCount(const int i);
int GetNFev(void);
void SetNFev(const int i);
int GetTerminationType(void);
void SetTerminationType(const int i);
CMCPDReport *GetInnerObj(void);
};
//+------------------------------------------------------------------+
//| Returns the value of the variable inneriterationscount |
//+------------------------------------------------------------------+
int CMCPDReportShell::GetInnerIterationsCount(void)
{
return(m_innerobj.m_inneriterationscount);
}
//+------------------------------------------------------------------+
//| Changing the value of the variable inneriterationscount |
//+------------------------------------------------------------------+
void CMCPDReportShell::SetInnerIterationsCount(const int i)
{
m_innerobj.m_inneriterationscount=i;
}
//+------------------------------------------------------------------+
//| Returns the value of the variable outeriterationscount |
//+------------------------------------------------------------------+
int CMCPDReportShell::GetOuterIterationsCount(void)
{
return(m_innerobj.m_outeriterationscount);
}
//+------------------------------------------------------------------+
//| Changing the value of the variable outeriterationscount |
//+------------------------------------------------------------------+
void CMCPDReportShell::SetOuterIterationsCount(const int i)
{
m_innerobj.m_outeriterationscount=i;
}
//+------------------------------------------------------------------+
//| Returns the value of the variable nfev |
//+------------------------------------------------------------------+
int CMCPDReportShell::GetNFev(void)
{
return(m_innerobj.m_nfev);
}
//+------------------------------------------------------------------+
//| Changing the value of the variable nfev |
//+------------------------------------------------------------------+
void CMCPDReportShell::SetNFev(const int i)
{
m_innerobj.m_nfev=i;
}
//+------------------------------------------------------------------+
//| Returns the value of the variable terminationtype |
//+------------------------------------------------------------------+
int CMCPDReportShell::GetTerminationType(void)
{
return(m_innerobj.m_terminationtype);
}
//+------------------------------------------------------------------+
//| Changing the value of the variable terminationtype |
//+------------------------------------------------------------------+
void CMCPDReportShell::SetTerminationType(const int i)
{
m_innerobj.m_terminationtype=i;
}
//+------------------------------------------------------------------+
//| Return object of class |
//+------------------------------------------------------------------+
CMCPDReport *CMCPDReportShell::GetInnerObj(void)
{
return(GetPointer(m_innerobj));
}
//+------------------------------------------------------------------+
//| Markov chains class |
//+------------------------------------------------------------------+
class CMarkovCPD
{
public:
//--- constant
static const double m_xtol;
//--- public methods
static void MCPDCreate(const int n,CMCPDState &s);
static void MCPDCreateEntry(const int n,const int entrystate,CMCPDState &s);
static void MCPDCreateExit(const int n,const int exitstate,CMCPDState &s);
static void MCPDCreateEntryExit(const int n,const int entrystate,const int exitstate,CMCPDState &s);
static void MCPDAddTrack(CMCPDState &s,CMatrixDouble &xy,const int k);
static void MCPDSetEC(CMCPDState &s,CMatrixDouble &ec);
static void MCPDAddEC(CMCPDState &s,const int i,const int j,const double c);
static void MCPDSetBC(CMCPDState &s,CMatrixDouble &bndl,CMatrixDouble &bndu);
static void MCPDAddBC(CMCPDState &s,const int i,const int j,double bndl,double bndu);
static void MCPDSetLC(CMCPDState &s,CMatrixDouble &c,int &ct[],const int k);
static void MCPDSetLC(CMCPDState &s,CMatrixDouble &c,CRowInt &ct,const int k);
static void MCPDSetTikhonovRegularizer(CMCPDState &s,const double v);
static void MCPDSetPrior(CMCPDState &s,CMatrixDouble &cpp);
static void MCPDSetPredictionWeights(CMCPDState &s,double &pw[]);
static void MCPDSetPredictionWeights(CMCPDState &s,CRowDouble &pw);
static void MCPDSolve(CMCPDState &s);
static void MCPDResults(CMCPDState &s,CMatrixDouble &p,CMCPDReport &rep);
private:
static void MCPDInit(const int n,const int entrystate,const int exitstate,CMCPDState &s);
};
//+------------------------------------------------------------------+
//| Initialize constant |
//+------------------------------------------------------------------+
const double CMarkovCPD::m_xtol=1.0E-8;
//+------------------------------------------------------------------+
//| DESCRIPTION: |
//| This function creates MCPD (Markov Chains for Population Data) |
//| solver. |
//| This solver can be used to find transition matrix P for |
//| N-dimensional prediction problem where transition from X[i] to |
//| X[i+1] is modelled as X[i+1] = P*X[i] |
//| where X[i] and X[i+1] are N-dimensional population vectors |
//| (components of each X are non-negative), and P is a N*N |
//| transition matrix (elements of are non-negative, each column |
//| sums to 1.0). |
//| Such models arise when when: |
//| * there is some population of individuals |
//| * individuals can have different states |
//| * individuals can transit from one state to another |
//| * population size is constant, i.e. there is no new individuals |
//| and no one leaves population |
//| * you want to model transitions of individuals from one state |
//| into another |
//| USAGE: |
//| Here we give very brief outline of the MCPD. We strongly |
//| recommend you to read examples in the ALGLIB Reference Manual |
//| and to read ALGLIB User Guide on data analysis which is |
//| available at http://www.alglib.net/dataanalysis/ |
//| 1. User initializes algorithm state with MCPDCreate() call |
//| 2. User adds one or more tracks - sequences of states which |
//| describe evolution of a system being modelled from different |
//| starting conditions |
//| 3. User may add optional boundary, equality and/or linear |
//| constraints on the coefficients of P by calling one of the |
//| following functions: |
//| * MCPDSetEC() to set equality constraints |
//| * MCPDSetBC() to set bound constraints |
//| * MCPDSetLC() to set linear constraints |
//| 4. Optionally, user may set custom weights for prediction errors |
//| (by default, algorithm assigns non-equal, automatically chosen|
//| weights for errors in the prediction of different components |
//| of X). It can be done with a call of |
//| MCPDSetPredictionWeights() function. |
//| 5. User calls MCPDSolve() function which takes algorithm state |
//| and pointer (delegate, etc.) to callback function which |
//| calculates F/G. |
//| 6. User calls MCPDResults() to get solution |
//| INPUT PARAMETERS: |
//| N - problem dimension, N>=1 |
//| OUTPUT PARAMETERS: |
//| State - structure stores algorithm state |
//+------------------------------------------------------------------+
void CMarkovCPD::MCPDCreate(const int n,CMCPDState &s)
{
//--- check
if(!CAp::Assert(n>=1,__FUNCTION__+": N<1"))
return;
//--- function call
MCPDInit(n,-1,-1,s);
}
//+------------------------------------------------------------------+
//| DESCRIPTION: |
//| This function is a specialized version of MCPDCreate() function, |
//| and we recommend you to read comments for this function for |
//| general information about MCPD solver. |
//| This function creates MCPD (Markov Chains for Population Data) |
//| solver for "Entry-state" model, i.e. model where transition from |
//| X[i] to X[i+1] is modelled as |
//| X[i+1] = P*X[i] |
//| where |
//| X[i] and X[i+1] are N-dimensional state vectors |
//| P is a N*N transition matrix |
//| and one selected component of X[] is called "entry" state and |
//| is treated in a special way: |
//| system state always transits from "entry" state to some |
//| another state |
//| system state can not transit from any state into "entry" |
//| state |
//| Such conditions basically mean that row of P which corresponds to|
//| "entry" state is zero. |
//| Such models arise when: |
//| * there is some population of individuals |
//| * individuals can have different states |
//| * individuals can transit from one state to another |
//| * population size is NOT constant - at every moment of time |
//| there is some (unpredictable) amount of "new" individuals, |
//| which can transit into one of the states at the next turn, but |
//| still no one leaves population |
//| * you want to model transitions of individuals from one state |
//| into another |
//|*but you do NOT want to predict amount of "new" individuals |
//| because it does not depends on individuals already present |
//| (hence system can not transit INTO entry state - it can only |
//| transit FROM it). |
//| This model is discussed in more details in the ALGLIB User Guide |
//| (see http://www.alglib.net/dataanalysis/ for more data). |
//| INPUT PARAMETERS: |
//| N - problem dimension, N>=2 |
//| EntryState- index of entry state, in 0..N-1 |
//| OUTPUT PARAMETERS: |
//| State - structure stores algorithm state |
//+------------------------------------------------------------------+
void CMarkovCPD::MCPDCreateEntry(const int n,const int entrystate,
CMCPDState &s)
{
//--- check
if(!CAp::Assert(n>=2,__FUNCTION__+": N<2"))
return;
//--- check
if(!CAp::Assert(entrystate>=0,__FUNCTION__+": EntryState<0"))
return;
//--- check
if(!CAp::Assert(entrystate<n,__FUNCTION__+": EntryState>=N"))
return;
//--- function call
MCPDInit(n,entrystate,-1,s);
}
//+------------------------------------------------------------------+
//| DESCRIPTION: |
//| This function is a specialized version of MCPDCreate() function, |
//| and we recommend you to read comments for this function for |
//| general information about MCPD solver. |
//| This function creates MCPD (Markov Chains for Population Data) |
//| solver for "Exit-state" model, i.e. model where transition from |
//| X[i] to X[i+1] is modelled as |
//| X[i+1] = P*X[i] |
//| where |
//| X[i] and X[i+1] are N-dimensional state vectors |
//| P is a N*N transition matrix |
//| and one selected component of X[] is called "exit" state and |
//| is treated in a special way: |
//| system state can transit from any state into "exit" state |
//| system state can not transit from "exit" state into any other|
//| state transition operator discards "exit" state (makes it |
//| zero at each turn) |
//| Such conditions basically mean that column of P which |
//| corresponds to "exit" state is zero. Multiplication by such P |
//| may decrease sum of vector components. |
//| Such models arise when: |
//| * there is some population of individuals |
//| * individuals can have different states |
//| * individuals can transit from one state to another |
//| * population size is NOT constant - individuals can move into |
//| "exit" state and leave population at the next turn, but there |
//| are no new individuals |
//| * amount of individuals which leave population can be predicted |
//| * you want to model transitions of individuals from one state |
//| into another (including transitions into the "exit" state) |
//| This model is discussed in more details in the ALGLIB User Guide |
//| (see http://www.alglib.net/dataanalysis/ for more data). |
//| INPUT PARAMETERS: |
//| N - problem dimension, N>=2 |
//| ExitState- index of exit state, in 0..N-1 |
//| OUTPUT PARAMETERS: |
//| State - structure stores algorithm state |
//+------------------------------------------------------------------+
void CMarkovCPD::MCPDCreateExit(const int n,const int exitstate,
CMCPDState &s)
{
//--- check
if(!CAp::Assert(n>=2,__FUNCTION__+": N<2"))
return;
//--- check
if(!CAp::Assert(exitstate>=0,__FUNCTION__+": ExitState<0"))
return;
//--- check
if(!CAp::Assert(exitstate<n,__FUNCTION__+": ExitState>=N"))
return;
//--- function call
MCPDInit(n,-1,exitstate,s);
}
//+------------------------------------------------------------------+
//| DESCRIPTION: |
//| This function is a specialized version of MCPDCreate() function, |
//| and we recommend you to read comments for this function for |
//| general information about MCPD solver. |
//| This function creates MCPD (Markov Chains for Population Data) |
//| solver for "Entry-Exit-states" model, i.e. model where transition|
//| from X[i] to X[i+1] is modelled as |
//| X[i+1] = P*X[i] |
//| where |
//| X[i] and X[i+1] are N-dimensional state vectors |
//| P is a N*N transition matrix |
//| one selected component of X[] is called "entry" state and is a |
//| treated in special way: |
//| system state always transits from "entry" state to some |
//| another state |
//| system state can not transit from any state into "entry" |
//| state |
//| and another one component of X[] is called "exit" state and is |
//| treated in a special way too: |
//| system state can transit from any state into "exit" state |
//| system state can not transit from "exit" state into any other|
//| state transition operator discards "exit" state (makes it |
//| zero at each turn) |
//| Such conditions basically mean that: |
//| row of P which corresponds to "entry" state is zero |
//| column of P which corresponds to "exit" state is zero |
//| Multiplication by such P may decrease sum of vector components. |
//| Such models arise when: |
//| * there is some population of individuals |
//| * individuals can have different states |
//| * individuals can transit from one state to another |
//| * population size is NOT constant |
//| * at every moment of time there is some (unpredictable) amount |
//| of "new" individuals, which can transit into one of the states |
//| at the next turn |
//|*some individuals can move (predictably) into "exit" state |
//| and leave population at the next turn |
//| * you want to model transitions of individuals from one state |
//| into another, including transitions from the "entry" state and |
//| into the "exit" state. |
//|*but you do NOT want to predict amount of "new" individuals |
//| because it does not depends on individuals already present |
//| (hence system can not transit INTO entry state - it can only |
//| transit FROM it). |
//| This model is discussed in more details in the ALGLIB User |
//| Guide (see http://www.alglib.net/dataanalysis/ for more data). |
//| INPUT PARAMETERS: |
//| N - problem dimension, N>=2 |
//| EntryState- index of entry state, in 0..N-1 |
//| ExitState- index of exit state, in 0..N-1 |
//| OUTPUT PARAMETERS: |
//| State - structure stores algorithm state |
//+------------------------------------------------------------------+
void CMarkovCPD::MCPDCreateEntryExit(const int n,const int entrystate,
const int exitstate,CMCPDState &s)
{
//--- check
if(!CAp::Assert(n>=2,__FUNCTION__+": N<2"))
return;
//--- check
if(!CAp::Assert(entrystate>=0,__FUNCTION__+": EntryState<0"))
return;
//--- check
if(!CAp::Assert(entrystate<n,__FUNCTION__+": EntryState>=N"))
return;
//--- check
if(!CAp::Assert(exitstate>=0,__FUNCTION__+": ExitState<0"))
return;
//--- check
if(!CAp::Assert(exitstate<n,__FUNCTION__+": ExitState>=N"))
return;
//--- check
if(!CAp::Assert(entrystate!=exitstate,__FUNCTION__+": EntryState=ExitState"))
return;
//--- function call
MCPDInit(n,entrystate,exitstate,s);
}
//+------------------------------------------------------------------+
//| This function is used to add a track - sequence of system states |
//| at the different moments of its evolution. |
//| You may add one or several tracks to the MCPD solver. In case you|
//| have several tracks, they won't overwrite each other. For |
//| example, if you pass two tracks, A1-A2-A3 (system at t=A+1, t=A+2|
//| and t=A+3) and B1-B2-B3, then solver will try to model |
//| transitions from t=A+1 to t=A+2, t=A+2 to t=A+3, t=B+1 to t=B+2, |
//| t=B+2 to t=B+3. But it WONT mix these two tracks - i.e. it wont |
//| try to model transition from t=A+3 to t=B+1. |
//| INPUT PARAMETERS: |
//| S - solver |
//| XY - track, array[K,N]: |
//| * I-th row is a state at t=I |
//| * elements of XY must be non-negative (exception |
//| will be thrown on negative elements) |
//| K - number of points in a track |
//| * if given, only leading K rows of XY are used |
//| * if not given, automatically determined from |
//| size of XY |
//| NOTES: |
//| 1. Track may contain either proportional or population data: |
//| * with proportional data all rows of XY must sum to 1.0, i.e. |
//| we have proportions instead of absolute population values |
//| * with population data rows of XY contain population counts |
//| and generally do not sum to 1.0 (although they still must be|
//| non-negative) |
//+------------------------------------------------------------------+
void CMarkovCPD::MCPDAddTrack(CMCPDState &s,CMatrixDouble &xy,
const int k)
{
//--- create variables
int n=s.m_n;
double s0=0;
double s1=0;
//--- check
if(!CAp::Assert(k>=0,__FUNCTION__+": K<0"))
return;
//--- check
if(!CAp::Assert((int)CAp::Cols(xy)>=n,__FUNCTION__+": Cols(XY)<N"))
return;
//--- check
if(!CAp::Assert((int)CAp::Rows(xy)>=k,__FUNCTION__+": Rows(XY)<K"))
return;
//--- check
if(!CAp::Assert(CApServ::IsFiniteMatrix(xy,k,n),__FUNCTION__+": XY contains infinite or NaN elements"))
return;
for(int i=0; i<k; i++)
{
for(int j=0; j<n; j++)
{
//--- check
if(!CAp::Assert(xy.Get(i,j)>=0.0,__FUNCTION__+": XY contains negative elements"))
return;
}
}
//--- check
if(k<2)
return;
//--- check
if((int)CAp::Rows(s.m_data)<s.m_npairs+k-1)
CApServ::RMatrixResize(s.m_data,MathMax(2*(int)CAp::Rows(s.m_data),s.m_npairs+k-1),2*n);
//--- calculation
for(int i=0; i<k-1; i++)
{
s0=0;
s1=0;
for(int j=0; j<n; j++)
{
//--- check
if(s.m_states[j]>=0)
s0+=xy.Get(i,j);
//--- check
if(s.m_states[j]<=0)
s1+=xy.Get(i+1,j);
}
//--- check
if(s0>0.0 && s1>0.0)
{
for(int j=0; j<n; j++)
{
//--- check
if(s.m_states[j]>=0)
s.m_data.Set(s.m_npairs,j,xy.Get(i,j)/s0);
else
s.m_data.Set(s.m_npairs,j,0.0);
//--- check
if(s.m_states[j]<=0)
s.m_data.Set(s.m_npairs,n+j,xy.Get(i+1,j)/s1);
else
s.m_data.Set(s.m_npairs,n+j,0.0);
}
//--- change value
s.m_npairs++;
}
}
}
//+------------------------------------------------------------------+
//| This function is used to add equality constraints on the elements|
//| of the transition matrix P. |
//| MCPD solver has four types of constraints which can be placed |
//| on P: |
//| * user-specified equality constraints (optional) |
//| * user-specified bound constraints (optional) |
//| * user-specified general linear constraints (optional) |
//| * basic constraints (always present): |
//| * non-negativity: P[i,j]>=0 |
//| * consistency: every column of P sums to 1.0 |
//| Final constraints which are passed to the underlying optimizer |
//| are calculated as intersection of all present constraints. For |
//| example, you may specify boundary constraint on P[0,0] and |
//| equality one: |
//| 0.1<=P[0,0]<=0.9 |
//| P[0,0]=0.5 |
//| Such combination of constraints will be silently reduced to their|
//| intersection, which is P[0,0]=0.5. |
//| This function can be used to place equality constraints on |
//| arbitrary subset of elements of P. Set of constraints is |
//| specified by EC, which may contain either NAN's or finite numbers|
//| from [0,1]. NAN denotes absence of constraint, finite number |
//| denotes equality constraint on specific element of P. |
//| You can also use MCPDAddEC() function which allows to ADD |
//| equality constraint for one element of P without changing |
//| constraints for other elements. |
//| These functions (MCPDSetEC and MCPDAddEC) interact as follows: |
//| * there is internal matrix of equality constraints which is |
//| stored in the MCPD solver |
//| * MCPDSetEC() replaces this matrix by another one (SET) |
//| * MCPDAddEC() modifies one element of this matrix and leaves |
//| other ones unchanged (ADD) |
//| * thus MCPDAddEC() call preserves all modifications done by |
//| previous calls, while MCPDSetEC() completely discards all |
//| changes done to the equality constraints. |
//| INPUT PARAMETERS: |
//| S - solver |
//| EC - equality constraints, array[N,N]. Elements of EC |
//| can be either NAN's or finite numbers from [0,1].|
//| NAN denotes absence of constraints, while finite |
//| value denotes equality constraint on the |
//| corresponding element of P. |
//| NOTES: |
//| 1. infinite values of EC will lead to exception being thrown. |
//| Values less than 0.0 or greater than 1.0 will lead to error code |
//| being returned after call to MCPDSolve(). |
//+------------------------------------------------------------------+
void CMarkovCPD::MCPDSetEC(CMCPDState &s,CMatrixDouble &ec)
{
int n=s.m_n;
//--- check
if(!CAp::Assert((int)CAp::Cols(ec)>=n,__FUNCTION__+": Cols(EC)<N"))
return;
//--- check
if(!CAp::Assert((int)CAp::Rows(ec)>=n,__FUNCTION__+": Rows(EC)<N"))
return;
//--- calculation
if(!CAp::Assert(CApServ::IsFiniteOrNaNMatrix(ec,n,n),"MCPDSetEC: EC containts infinite elements"))
return;
s.m_ec=ec;
s.m_ec.Resize(n,n);
}
//+------------------------------------------------------------------+
//| This function is used to add equality constraints on the elements|
//| of the transition matrix P. |
//| MCPD solver has four types of constraints which can be placed |
//| on P: |
//| * user-specified equality constraints (optional) |
//| * user-specified bound constraints (optional) |
//| * user-specified general linear constraints (optional) |
//| * basic constraints (always present): |
//| * non-negativity: P[i,j]>=0 |
//| * consistency: every column of P sums to 1.0 |
//| Final constraints which are passed to the underlying optimizer |
//| are calculated as intersection of all present constraints. For |
//| example, you may specify boundary constraint on P[0,0] and |
//| equality one: |
//| 0.1<=P[0,0]<=0.9 |
//| P[0,0]=0.5 |
//| Such combination of constraints will be silently reduced to their|
//| intersection, which is P[0,0]=0.5. |
//| This function can be used to ADD equality constraint for one |
//| element of P without changing constraints for other elements. |
//| You can also use MCPDSetEC() function which allows you to specify|
//| arbitrary set of equality constraints in one call. |
//| These functions (MCPDSetEC and MCPDAddEC) interact as follows: |
//| * there is internal matrix of equality constraints which is |
//| stored in the MCPD solver |
//| * MCPDSetEC() replaces this matrix by another one (SET) |
//| * MCPDAddEC() modifies one element of this matrix and leaves |
//| other ones unchanged (ADD) |
//| * thus MCPDAddEC() call preserves all modifications done by |
//| previous calls, while MCPDSetEC() completely discards all |
//| changes done to the equality constraints. |
//| INPUT PARAMETERS: |
//| S - solver |
//| I - row index of element being constrained |
//| J - column index of element being constrained |
//| C - value (constraint for P[I,J]). Can be either NAN |
//| (no constraint) or finite value from [0,1]. |
//| NOTES: |
//| 1. infinite values of C will lead to exception being thrown. |
//| Values less than 0.0 or greater than 1.0 will lead to error code |
//| being returned after call to MCPDSolve(). |
//+------------------------------------------------------------------+
void CMarkovCPD::MCPDAddEC(CMCPDState &s,const int i,const int j,
const double c)
{
//--- check
if(!CAp::Assert(i>=0,__FUNCTION__+": I<0"))
return;
//--- check
if(!CAp::Assert(i<s.m_n,__FUNCTION__+": I>=N"))
return;
//--- check
if(!CAp::Assert(j>=0,__FUNCTION__+": J<0"))
return;
//--- check
if(!CAp::Assert(j<s.m_n,__FUNCTION__+": J>=N"))
return;
//--- check
if(!CAp::Assert(CInfOrNaN::IsNaN(c) || MathIsValidNumber(c),"MCPDAddEC: C is not finite number or NAN"))
return;
s.m_ec.Set(i,j,c);
}
//+------------------------------------------------------------------+
//| This function is used to add bound constraints on the elements |
//| of the transition matrix P. |
//| MCPD solver has four types of constraints which can be placed |
//| on P: |
//| * user-specified equality constraints (optional) |
//| * user-specified bound constraints (optional) |
//| * user-specified general linear constraints (optional) |
//| * basic constraints (always present): |
//| * non-negativity: P[i,j]>=0 |
//| * consistency: every column of P sums to 1.0 |
//| Final constraints which are passed to the underlying optimizer |
//| are calculated as intersection of all present constraints. For |
//| example, you may specify boundary constraint on P[0,0] and |
//| equality one: |
//| 0.1<=P[0,0]<=0.9 |
//| P[0,0]=0.5 |
//| Such combination of constraints will be silently reduced to their|
//| intersection, which is P[0,0]=0.5. |
//| This function can be used to place bound constraints on arbitrary|
//| subset of elements of P. Set of constraints is specified by |
//| BndL/BndU matrices, which may contain arbitrary combination of |
//| finite numbers or infinities (like -INF<x<=0.5 or 0.1<=x<+INF). |
//| You can also use MCPDAddBC() function which allows to ADD bound |
//| constraint for one element of P without changing constraints for |
//| other elements. |
//| These functions (MCPDSetBC and MCPDAddBC) interact as follows: |
//| * there is internal matrix of bound constraints which is stored |
//| in the MCPD solver |
//| * MCPDSetBC() replaces this matrix by another one (SET) |
//| * MCPDAddBC() modifies one element of this matrix and leaves |
//| other ones unchanged (ADD) |
//| * thus MCPDAddBC() call preserves all modifications done by |
//| previous calls, while MCPDSetBC() completely discards all |
//| changes done to the equality constraints. |
//| INPUT PARAMETERS: |
//| S - solver |
//| BndL - lower bounds constraints, array[N,N]. Elements of|
//| BndL can be finite numbers or -INF. |
//| BndU - upper bounds constraints, array[N,N]. Elements of|
//| BndU can be finite numbers or +INF. |
//+------------------------------------------------------------------+
void CMarkovCPD::MCPDSetBC(CMCPDState &s,CMatrixDouble &bndl,
CMatrixDouble &bndu)
{
int n=s.m_n;
//--- check
if(!CAp::Assert((int)CAp::Cols(bndl)>=n,__FUNCTION__+": Cols(BndL)<N"))
return;
//--- check
if(!CAp::Assert((int)CAp::Rows(bndl)>=n,__FUNCTION__+": Rows(BndL)<N"))
return;
//--- check
if(!CAp::Assert((int)CAp::Cols(bndu)>=n,__FUNCTION__+": Cols(BndU)<N"))
return;
//--- check
if(!CAp::Assert((int)CAp::Rows(bndu)>=n,__FUNCTION__+": Rows(BndU)<N"))
return;
//--- calculation
for(int i=0; i<n; i++)
{
for(int j=0; j<n; j++)
{
//--- check
if(!CAp::Assert(MathIsValidNumber(bndl.Get(i,j)) || CInfOrNaN::IsNegativeInfinity(bndl.Get(i,j)),"MCPDSetBC: BndL containts NAN or +INF"))
return;
//--- check
if(!CAp::Assert(MathIsValidNumber(bndu.Get(i,j)) || CInfOrNaN::IsPositiveInfinity(bndu.Get(i,j)),"MCPDSetBC: BndU containts NAN or -INF"))
return;
//--- change values
s.m_bndl.Set(i,j,bndl.Get(i,j));
s.m_bndu.Set(i,j,bndu.Get(i,j));
}
}
}
//+------------------------------------------------------------------+
//| This function is used to add bound constraints on the elements |
//| of the transition matrix P. |
//| MCPD solver has four types of constraints which can be placed |
//| on P: |
//| * user-specified equality constraints (optional) |
//| * user-specified bound constraints (optional) |
//| * user-specified general linear constraints (optional) |
//| * basic constraints (always present): |
//| * non-negativity: P[i,j]>=0 |
//| * consistency: every column of P sums to 1.0 |
//| Final constraints which are passed to the underlying optimizer |
//| are calculated as intersection of all present constraints. For |
//| example, you may specify boundary constraint on P[0,0] and |
//| equality one: |
//| 0.1<=P[0,0]<=0.9 |
//| P[0,0]=0.5 |
//| Such combination of constraints will be silently reduced to their|
//| intersection, which is P[0,0]=0.5. |
//| This function can be used to ADD bound constraint for one element|
//| of P without changing constraints for other elements. |
//| You can also use MCPDSetBC() function which allows to place bound|
//| constraints on arbitrary subset of elements of P. Set of |
//| constraints is specified by BndL/BndU matrices, which may |
//| contain arbitrary combination of finite numbers or infinities |
//| (like -INF<x<=0.5 or 0.1<=x<+INF). |
//| These functions (MCPDSetBC and MCPDAddBC) interact as follows: |
//| * there is internal matrix of bound constraints which is stored |
//| in the MCPD solver |
//| * MCPDSetBC() replaces this matrix by another one (SET) |
//| * MCPDAddBC() modifies one element of this matrix and leaves |
//| other ones unchanged (ADD) |
//| * thus MCPDAddBC() call preserves all modifications done by |
//| previous calls, while MCPDSetBC() completely discards all |
//| changes done to the equality constraints. |
//| INPUT PARAMETERS: |
//| S - solver |
//| I - row index of element being constrained |
//| J - column index of element being constrained |
//| BndL - lower bound |
//| BndU - upper bound |
//+------------------------------------------------------------------+
void CMarkovCPD::MCPDAddBC(CMCPDState &s,const int i,const int j,
double bndl,double bndu)
{
//--- check
if(!CAp::Assert(i>=0,__FUNCTION__+": I<0"))
return;
//--- check
if(!CAp::Assert(i<s.m_n,__FUNCTION__+": I>=N"))
return;
//--- check
if(!CAp::Assert(j>=0,__FUNCTION__+": J<0"))
return;
//--- check
if(!CAp::Assert(j<s.m_n,__FUNCTION__+": J>=N"))
return;
//--- check
if(!CAp::Assert(MathIsValidNumber(bndl) || CInfOrNaN::IsNegativeInfinity(bndl),"MCPDAddBC: BndL is NAN or +INF"))
return;
//--- check
if(!CAp::Assert(MathIsValidNumber(bndu) || CInfOrNaN::IsPositiveInfinity(bndu),"MCPDAddBC: BndU is NAN or -INF"))
return;
//--- change values
s.m_bndl.Set(i,j,bndl);
s.m_bndu.Set(i,j,bndu);
}
//+------------------------------------------------------------------+
//| This function is used to set linear equality/inequality |
//| constraints on the elements of the transition matrix P. |
//| This function can be used to set one or several general linear |
//| constraints on the elements of P. Two types of constraints are |
//| supported: |
//| * equality constraints |
//| * inequality constraints (both less-or-equal and |
//| greater-or-equal) |
//| Coefficients of constraints are specified by matrix C (one of the|
//| parameters). One row of C corresponds to one constraint. |
//| Because transition matrix P has N*N elements, we need N*N columns|
//| to store all coefficients (they are stored row by row), and |
//| one more column to store right part - hence C has N*N+1 columns. |
//| Constraint kind is stored in the CT array. |
//| Thus, I-th linear constraint is |
//| P[0,0]*C[I,0] + P[0,1]*C[I,1] + .. + P[0,N-1]*C[I,N-1] + |
//| + P[1,0]*C[I,N] + P[1,1]*C[I,N+1] + ... + |
//| + P[N-1,N-1]*C[I,N*N-1] ?=? C[I,N*N] |
//| where ?=? can be either "=" (CT[i]=0), "<=" (CT[i]<0) or ">=" |
//| (CT[i]>0). |
//| Your constraint may involve only some subset of P (less than N*N |
//| elements). |
//| For example it can be something like |
//| P[0,0] + P[0,1] = 0.5 |
//| In this case you still should pass matrix with N*N+1 columns, |
//| but all its elements (except for C[0,0], C[0,1] and C[0,N*N-1]) |
//| will be zero. |
//| INPUT PARAMETERS: |
//| S - solver |
//| C - array[K,N*N+1] - coefficients of constraints |
//| (see above for complete description) |
//| CT - array[K] - constraint types |
//| (see above for complete description) |
//| K - number of equality/inequality constraints, K>=0: |
//| * if given, only leading K elements of C/CT are |
//| used |
//| * if not given, automatically determined from |
//| sizes of C/CT |
//+------------------------------------------------------------------+
void CMarkovCPD::MCPDSetLC(CMCPDState &s,CMatrixDouble &c,int &ct[],
const int k)
{
CRowInt Ct=ct;
MCPDSetLC(s,c,Ct,k);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CMarkovCPD::MCPDSetLC(CMCPDState &s,CMatrixDouble &c,CRowInt &ct,
const int k)
{
int n=s.m_n;
//--- check
if(!CAp::Assert((int)CAp::Cols(c)>=n*n+1,__FUNCTION__+": Cols(C)<N*N+1"))
return;
//--- check
if(!CAp::Assert((int)CAp::Rows(c)>=k,__FUNCTION__+": Rows(C)<K"))
return;
//--- check
if(!CAp::Assert(CAp::Len(ct)>=k,__FUNCTION__+": Len(CT)<K"))
return;
//--- check
if(!CAp::Assert(CApServ::IsFiniteMatrix(c,k,n*n+1),__FUNCTION__+": C contains infinite or NaN values!"))
return;
//--- copy
s.m_c=c;
s.m_ct=ct;
s.m_c.Resize(k,n*n+1);
s.m_ct.Resize(k);
s.m_ccnt=k;
}
//+------------------------------------------------------------------+
//| This function allows to tune amount of Tikhonov regularization |
//| being applied to your problem. |
//| By default, regularizing term is equal to r*||P-prior_P||^2, |
//| where r is a small non-zero value, P is transition matrix, |
//| prior_P is identity matrix, ||X||^2 is a sum of squared elements |
//| of X. |
//| This function allows you to change coefficient r. You can also |
//| change prior values with MCPDSetPrior() function. |
//| INPUT PARAMETERS: |
//| S - solver |
//| V - regularization coefficient, finite non-negative |
//| value. It is not recommended to specify zero |
//| value unless you are pretty sure that you want it.|
//+------------------------------------------------------------------+
void CMarkovCPD::MCPDSetTikhonovRegularizer(CMCPDState &s,const double v)
{
//--- check
if(!CAp::Assert(MathIsValidNumber(v),__FUNCTION__+": V is infinite or NAN"))
return;
//--- check
if(!CAp::Assert(v>=0.0,__FUNCTION__+": V is less than zero"))
return;
//--- change value
s.m_regterm=v;
}
//+------------------------------------------------------------------+
//| This function allows to set prior values used for regularization |
//| of your problem. |
//| By default, regularizing term is equal to r*||P-prior_P||^2, |
//| where r is a small non-zero value, P is transition matrix, |
//| prior_P is identity matrix, ||X||^2 is a sum of squared elements |
//| of X. |
//| This function allows you to change prior values prior_P. You can |
//| also change r with MCPDSetTikhonovRegularizer() function. |
//| INPUT PARAMETERS: |
//| S - solver |
//| PP - array[N,N], matrix of prior values: |
//| 1. elements must be real numbers from [0,1] |
//| 2. columns must sum to 1.0. |
//| First property is checked (exception is thrown |
//| otherwise), while second one is not |
//| checked/enforced. |
//+------------------------------------------------------------------+
void CMarkovCPD::MCPDSetPrior(CMCPDState &s,CMatrixDouble &pp)
{
int n=s.m_n;
//--- check
if(!CAp::Assert((int)CAp::Cols(pp)>=n,__FUNCTION__+": Cols(PP)<N"))
return;
//--- check
if(!CAp::Assert((int)CAp::Rows(pp)>=n,__FUNCTION__+": Rows(PP)<K"))
return;
//--- check
if(!CAp::Assert(CApServ::IsFiniteMatrix(pp,n,n),__FUNCTION__+": PP containts infinite elements"))
return;
//--- check
if(!CAp::Assert(pp.Min()>=0.0 && pp.Max()<=1.0,__FUNCTION__+": PP.Get(i,j) is less than 0.0 or greater than 1.0"))
return;
//--- change value
s.m_priorp=pp;
s.m_priorp.Resize(n,n);
}
//+------------------------------------------------------------------+
//| This function is used to change prediction weights |
//| MCPD solver scales prediction errors as follows |
//| Error(P) = ||W*(y-P*x)||^2 |
//| where |
//| x is a system state at time t |
//| y is a system state at time t+1 |
//| P is a transition matrix |
//| W is a diagonal scaling matrix |
//| By default, weights are chosen in order to minimize relative |
//| prediction error instead of absolute one. For example, if one |
//| component of state is about 0.5 in magnitude and another one is |
//| about 0.05, then algorithm will make corresponding weights equal |
//| to 2.0 and 20.0. |
//| INPUT PARAMETERS: |
//| S - solver |
//| PW - array[N], weights: |
//| * must be non-negative values (exception will be |
//| thrown otherwise) |
//| * zero values will be replaced by automatically |
//| chosen values |
//+------------------------------------------------------------------+
void CMarkovCPD::MCPDSetPredictionWeights(CMCPDState &s,double &pw[])
{
CRowDouble Pw=pw;
MCPDSetPredictionWeights(s,Pw);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CMarkovCPD::MCPDSetPredictionWeights(CMCPDState &s,CRowDouble &pw)
{
int n=s.m_n;
//--- check
if(!CAp::Assert(CAp::Len(pw)>=n,__FUNCTION__+": Length(PW)<N"))
return;
//--- check
if(!CAp::Assert(CApServ::IsFiniteVector(pw,n),__FUNCTION__+": PW containts infinite or NAN elements"))
return;
//--- check
if(!CAp::Assert(pw.Min()>=0.0,__FUNCTION__+": PW containts negative elements"))
return;
s.m_pw=pw;
}
//+------------------------------------------------------------------+
//| This function is used to start solution of the MCPD problem. |
//| After return from this function, you can use MCPDResults() to get|
//| solution and completion code. |
//+------------------------------------------------------------------+
void CMarkovCPD::MCPDSolve(CMCPDState &s)
{
//--- create variables
int n=s.m_n;
int npairs=s.m_npairs;
int ccnt=0;
int i=0;
int j=0;
int k=0;
int k2=0;
double v=0;
double vv=0;
int i_=0;
int i1_=0;
//--- init fields of S
s.m_repterminationtype=0;
s.m_repinneriterationscount=0;
s.m_repouteriterationscount=0;
s.m_repnfev=0;
s.m_p.Fill(AL_NaN);
//--- Generate "effective" weights for prediction and calculate preconditioner
for(i=0; i<n; i++)
{
//--- check
if(s.m_pw[i]==0.0)
{
//--- change values
v=0;
k=0;
for(j=0; j<npairs; j++)
//--- check
if(s.m_data.Get(j,n+i)!=0.0)
{
v+=s.m_data.Get(j,n+i);
k++;
}
//--- check
if(k!=0)
s.m_effectivew.Set(i,k/v);
else
s.m_effectivew.Set(i,1.0);
}
else
s.m_effectivew.Set(i,s.m_pw[i]);
}
//--- calculation
for(i=0; i<n; i++)
{
for(j=0; j<n; j++)
s.m_h.Set(i*n+j,2*s.m_regterm);
}
//--- calculation
for(k=0; k<npairs; k++)
{
for(i=0; i<n; i++)
{
for(j=0; j<n; j++)
s.m_h.Add(i*n+j,2*CMath::Sqr(s.m_effectivew[i])*CMath::Sqr(s.m_data.Get(k,j)));
}
}
//--- calculation
for(i=0; i<n; i++)
for(j=0; j<n; j++)
//--- check
if(s.m_h[i*n+j]==0.0)
s.m_h.Set(i*n+j,1);
//--- Generate "effective" BndL/BndU
for(i=0; i<n; i++)
{
for(j=0; j<n; j++)
{
//--- Set default boundary constraints.
//--- Lower bound is always zero,upper bound is calculated
//--- with respect to entry/exit states.
s.m_effectivebndl.Set(i*n+j,0.0);
//--- check
if(s.m_states[i]>0 || s.m_states[j]<0)
s.m_effectivebndu.Set(i*n+j,0.0);
else
s.m_effectivebndu.Set(i*n+j,1.0);
//--- Calculate intersection of the default and user-specified bound constraints.
//--- This code checks consistency of such combination.
if(MathIsValidNumber(s.m_bndl.Get(i,j)) && s.m_bndl.Get(i,j)>s.m_effectivebndl[i*n+j])
s.m_effectivebndl.Set(i*n+j,s.m_bndl.Get(i,j));
//--- check
if(MathIsValidNumber(s.m_bndu.Get(i,j)) && s.m_bndu.Get(i,j)<s.m_effectivebndu[i*n+j])
s.m_effectivebndu.Set(i*n+j,s.m_bndu.Get(i,j));
//--- check
if(s.m_effectivebndl[i*n+j]>s.m_effectivebndu[i*n+j])
{
s.m_repterminationtype=-3;
return;
}
//--- Calculate intersection of the effective bound constraints
//--- and user-specified equality constraints.
//--- This code checks consistency of such combination.
if(MathIsValidNumber(s.m_ec.Get(i,j)))
{
//--- check
if(s.m_ec.Get(i,j)<s.m_effectivebndl[i*n+j] || s.m_ec.Get(i,j)>s.m_effectivebndu[i*n+j])
{
s.m_repterminationtype=-3;
return;
}
//--- change values
s.m_effectivebndl.Set(i*n+j,s.m_ec.Get(i,j));
s.m_effectivebndu.Set(i*n+j,s.m_ec.Get(i,j));
}
}
}
//--- Generate linear constraints:
//---*"default" sums-to-one constraints (not generated for "exit" states)
CApServ::RMatrixSetLengthAtLeast(s.m_effectivec,s.m_ccnt+n,n*n+1);
//--- function call
CApServ::IVectorSetLengthAtLeast(s.m_effectivect,s.m_ccnt+n);
ccnt=s.m_ccnt;
for(i=0; i<s.m_ccnt; i++)
{
for(j=0; j<=n*n; j++)
s.m_effectivec.Set(i,j,s.m_c.Get(i,j));
s.m_effectivect.Set(i,s.m_ct[i]);
}
//--- calculation
for(i=0; i<n; i++)
{
//--- check
if(s.m_states[i]>=0)
{
for(k=0; k<n*n; k++)
s.m_effectivec.Set(ccnt,k,0);
for(k=0; k<n; k++)
s.m_effectivec.Set(ccnt,k*n+i,1);
//--- change values
s.m_effectivec.Set(ccnt,n*n,1.0);
s.m_effectivect.Set(ccnt,0);
ccnt++;
}
}
//--- create optimizer
for(i=0; i<n; i++)
for(j=0; j<n; j++)
s.m_tmpp.Set(i*n+j,1.0/(double)n);
//--- function calls
CMinBLEIC::MinBLEICRestartFrom(s.m_bs,s.m_tmpp);
CMinBLEIC::MinBLEICSetBC(s.m_bs,s.m_effectivebndl,s.m_effectivebndu);
CMinBLEIC::MinBLEICSetLC(s.m_bs,s.m_effectivec,s.m_effectivect,ccnt);
CMinBLEIC::MinBLEICSetInnerCond(s.m_bs,0,0,m_xtol);
CMinBLEIC::MinBLEICSetOuterCond(s.m_bs,m_xtol,1.0E-5);
CMinBLEIC::MinBLEICSetPrecDiag(s.m_bs,s.m_h);
//--- solve problem
while(CMinBLEIC::MinBLEICIteration(s.m_bs))
{
//--- check
if(!CAp::Assert(s.m_bs.m_needfg,__FUNCTION__+": internal error"))
return;
//--- check
if(s.m_bs.m_needfg)
{
//--- Calculate regularization term
s.m_bs.m_f=0.0;
vv=s.m_regterm;
for(i=0; i<n; i++)
for(j=0; j<n; j++)
{
double delt=s.m_bs.m_x[i*n+j]-s.m_priorp.Get(i,j);
s.m_bs.m_f+=vv*CMath::Sqr(delt);
s.m_bs.m_g.Set(i*n+j,2*vv*delt);
}
//--- calculate prediction error/gradient for K-th pair
for(k=0; k<npairs; k++)
{
for(i=0; i<n; i++)
{
i1_=-(i*n);
v=0.0;
for(i_=i*n; i_<i*n+n; i_++)
v+=s.m_bs.m_x[i_]*s.m_data.Get(k,i_+i1_);
vv=s.m_effectivew[i];
s.m_bs.m_f+=CMath::Sqr(vv*(v-s.m_data.Get(k,n+i)));
for(j=0; j<n; j++)
s.m_bs.m_g.Add(i*n+j,2*vv*vv*(v-s.m_data.Get(k,n+i))*s.m_data.Get(k,j));
}
}
//--- continue
continue;
}
}
//--- function call
CMinBLEIC::MinBLEICResultsBuf(s.m_bs,s.m_tmpp,s.m_br);
for(i=0; i<n; i++)
{
for(j=0; j<n; j++)
s.m_p.Set(i,j,s.m_tmpp[i*n+j]);
}
//--- change values
s.m_repterminationtype=s.m_br.m_terminationtype;
s.m_repinneriterationscount=s.m_br.m_inneriterationscount;
s.m_repouteriterationscount=s.m_br.m_outeriterationscount;
s.m_repnfev=s.m_br.m_nfev;
}
//+------------------------------------------------------------------+
//| MCPD results |
//| INPUT PARAMETERS: |
//| State - algorithm state |
//| OUTPUT PARAMETERS: |
//| P - array[N,N], transition matrix |
//| Rep - optimization report. You should check Rep. |
//| TerminationType in order to distinguish successful|
//| termination from unsuccessful one. Speaking short,|
//| positive values denote success, negative ones are |
//| failures. More information about fields of this |
//| structure can befound in the comments on |
//| MCPDReport datatype. |
//+------------------------------------------------------------------+
void CMarkovCPD::MCPDResults(CMCPDState &s,CMatrixDouble &p,
CMCPDReport &rep)
{
//--- copy
p=s.m_p;
//--- change values
rep.m_terminationtype=s.m_repterminationtype;
rep.m_inneriterationscount=s.m_repinneriterationscount;
rep.m_outeriterationscount=s.m_repouteriterationscount;
rep.m_nfev=s.m_repnfev;
}
//+------------------------------------------------------------------+
//| Internal initialization function |
//+------------------------------------------------------------------+
void CMarkovCPD::MCPDInit(const int n,const int entrystate,
const int exitstate,CMCPDState &s)
{
//--- check
if(!CAp::Assert(n>=1,__FUNCTION__+": N<1"))
return;
//--- initialization
s.m_n=n;
//--- allocation
s.m_states.Resize(n);
s.m_states.Fill(0);
//--- check
if(entrystate>=0)
s.m_states.Set(entrystate,1);
//--- check
if(exitstate>=0)
s.m_states.Set(exitstate,-1);
//--- initialization
s.m_npairs=0;
s.m_regterm=1.0E-8;
s.m_ccnt=0;
//--- allocation
s.m_p=matrix<double>::Zeros(n,n);
s.m_ec=matrix<double>::Full(n,n,CInfOrNaN::NaN());
s.m_bndl=matrix<double>::Full(n,n,CInfOrNaN::NegativeInfinity());
s.m_bndu=matrix<double>::Full(n,n,CInfOrNaN::PositiveInfinity());
s.m_pw=vector<double>::Zeros(n);
s.m_priorp=matrix<double>::Identity(n,n);
s.m_tmpp=vector<double>::Zeros(n*n);
s.m_effectivew.Resize(n);
s.m_effectivebndl.Resize(n*n);
s.m_effectivebndu.Resize(n*n);
s.m_h.Resize(n*n);
//--- allocation
s.m_data=matrix<double>::Zeros(1,2*n);
//--- function call
CMinBLEIC::MinBLEICCreate(n*n,s.m_tmpp,s.m_bs);
}
//+------------------------------------------------------------------+
//| Training report: |
//| * RelCLSError - fraction of misclassified cases. |
//| * AvgCE - acerage cross-entropy |
//| * RMSError - root-mean-square error |
//| * AvgError - average error |
//| * AvgRelError - average relative error |
//| * NGrad - number of gradient calculations |
//| * NHess - number of Hessian calculations |
//| * NCholesky - number of Cholesky decompositions |
//| NOTE 1: RelCLSError/AvgCE are zero on regression problems. |
//| NOTE 2: on classification problems RMSError/AvgError/AvgRelError |
//| contain errors in prediction of posterior probabilities |
//+------------------------------------------------------------------+
class CMLPReport
{
public:
//--- variables
double m_RelCLSError;
double m_AvgCE;
double m_RMSError;
double m_AvgError;
double m_AvgRelError;
int m_ngrad;
int m_nhess;
int m_ncholesky;
//--- constructor, destructor
CMLPReport(void) { ZeroMemory(this); }
~CMLPReport(void) {}
//--- copy
void Copy(const CMLPReport &obj);
//--- overloading
void operator=(const CMLPReport &obj) { Copy(obj); }
};
//+------------------------------------------------------------------+
//| Copy |
//+------------------------------------------------------------------+
void CMLPReport::Copy(const CMLPReport &obj)
{
//--- copy variables
m_RelCLSError=obj.m_RelCLSError;
m_AvgCE=obj.m_AvgCE;
m_RMSError=obj.m_RMSError;
m_AvgError=obj.m_AvgError;
m_AvgRelError=obj.m_AvgRelError;
m_ngrad=obj.m_ngrad;
m_nhess=obj.m_nhess;
m_ncholesky=obj.m_ncholesky;
}
//+------------------------------------------------------------------+
//| Training report: |
//| * NGrad - number of gradient calculations |
//| * NHess - number of Hessian calculations |
//| * NCholesky - number of Cholesky decompositions |
//+------------------------------------------------------------------+
class CMLPReportShell
{
private:
CMLPReport m_innerobj;
public:
//--- constructors, destructor
CMLPReportShell(void) {}
CMLPReportShell(CMLPReport &obj) { m_innerobj.Copy(obj); }
~CMLPReportShell(void) {}
//--- methods
int GetNGrad(void);
void SetNGrad(const int i);
int GetNHess(void);
void SetNHess(const int i);
int GetNCholesky(void);
void SetNCholesky(const int i);
CMLPReport *GetInnerObj(void);
};
//+------------------------------------------------------------------+
//| Returns the value of the variable ngrad |
//+------------------------------------------------------------------+
int CMLPReportShell::GetNGrad(void)
{
return(m_innerobj.m_ngrad);
}
//+------------------------------------------------------------------+
//| Changing the value of the variable ngrad |
//+------------------------------------------------------------------+
void CMLPReportShell::SetNGrad(const int i)
{
m_innerobj.m_ngrad=i;
}
//+------------------------------------------------------------------+
//| Returns the value of the variable nhess |
//+------------------------------------------------------------------+
int CMLPReportShell::GetNHess(void)
{
return(m_innerobj.m_nhess);
}
//+------------------------------------------------------------------+
//| Changing the value of the variable nhess |
//+------------------------------------------------------------------+
void CMLPReportShell::SetNHess(const int i)
{
m_innerobj.m_nhess=i;
}
//+------------------------------------------------------------------+
//| Returns the value of the variable ncholesky |
//+------------------------------------------------------------------+
int CMLPReportShell::GetNCholesky(void)
{
return(m_innerobj.m_ncholesky);
}
//+------------------------------------------------------------------+
//| Changing the value of the variable ncholesky |
//+------------------------------------------------------------------+
void CMLPReportShell::SetNCholesky(const int i)
{
m_innerobj.m_ncholesky=i;
}
//+------------------------------------------------------------------+
//| Return object of class |
//+------------------------------------------------------------------+
CMLPReport *CMLPReportShell::GetInnerObj(void)
{
return(GetPointer(m_innerobj));
}
//+------------------------------------------------------------------+
//| Cross-validation estimates of generalization error |
//+------------------------------------------------------------------+
class CMLPCVReport
{
public:
//--- variables
double m_RelCLSError;
double m_AvgCE;
double m_RMSError;
double m_AvgError;
double m_AvgRelError;
//--- constructor, destructor
CMLPCVReport(void) { ZeroMemory(this); }
~CMLPCVReport(void) {}
//--- copy
void Copy(const CMLPCVReport &obj);
//--- overloading
void operator=(const CMLPCVReport &obj) { Copy(obj); }
};
//+------------------------------------------------------------------+
//| Copy |
//+------------------------------------------------------------------+
void CMLPCVReport::Copy(const CMLPCVReport &obj)
{
//--- copy variables
m_RelCLSError=obj.m_RelCLSError;
m_AvgCE=obj.m_AvgCE;
m_RMSError=obj.m_RMSError;
m_AvgError=obj.m_AvgError;
m_AvgRelError=obj.m_AvgRelError;
}
//+------------------------------------------------------------------+
//| Cross-validation estimates of generalization error |
//+------------------------------------------------------------------+
class CMLPCVReportShell
{
private:
CMLPCVReport m_innerobj;
public:
//--- constructors, destructor
CMLPCVReportShell(void) {}
CMLPCVReportShell(CMLPCVReport &obj) { m_innerobj.Copy(obj); }
~CMLPCVReportShell(void) {}
//--- methods
double GetRelClsError(void);
void SetRelClsError(const double d);
double GetAvgCE(void);
void SetAvgCE(const double d);
double GetRMSError(void);
void SetRMSError(const double d);
double GetAvgError(void);
void SetAvgError(const double d);
double GetAvgRelError(void);
void SetAvgRelError(const double d);
CMLPCVReport *GetInnerObj(void);
};
//+------------------------------------------------------------------+
//| Returns the value of the variable relclserror |
//+------------------------------------------------------------------+
double CMLPCVReportShell::GetRelClsError(void)
{
return(m_innerobj.m_RelCLSError);
}
//+------------------------------------------------------------------+
//| Changing the value of the variable relclserror |
//+------------------------------------------------------------------+
void CMLPCVReportShell::SetRelClsError(const double d)
{
m_innerobj.m_RelCLSError=d;
}
//+------------------------------------------------------------------+
//| Returns the value of the variable avgce |
//+------------------------------------------------------------------+
double CMLPCVReportShell::GetAvgCE(void)
{
return(m_innerobj.m_AvgCE);
}
//+------------------------------------------------------------------+
//| Changing the value of the variable avgce |
//+------------------------------------------------------------------+
void CMLPCVReportShell::SetAvgCE(const double d)
{
m_innerobj.m_AvgCE=d;
}
//+------------------------------------------------------------------+
//| Returns the value of the variable rmserror |
//+------------------------------------------------------------------+
double CMLPCVReportShell::GetRMSError(void)
{
return(m_innerobj.m_RMSError);
}
//+------------------------------------------------------------------+
//| Changing the value of the variable rmserror |
//+------------------------------------------------------------------+
void CMLPCVReportShell::SetRMSError(const double d)
{
m_innerobj.m_RMSError=d;
}
//+------------------------------------------------------------------+
//| Returns the value of the variable avgerror |
//+------------------------------------------------------------------+
double CMLPCVReportShell::GetAvgError(void)
{
return(m_innerobj.m_AvgError);
}
//+------------------------------------------------------------------+
//| Changing the value of the variable avgerror |
//+------------------------------------------------------------------+
void CMLPCVReportShell::SetAvgError(const double d)
{
m_innerobj.m_AvgError=d;
}
//+------------------------------------------------------------------+
//| Returns the value of the variable avgrelerror |
//+------------------------------------------------------------------+
double CMLPCVReportShell::GetAvgRelError(void)
{
return(m_innerobj.m_AvgRelError);
}
//+------------------------------------------------------------------+
//| Changing the value of the variable avgrelerror |
//+------------------------------------------------------------------+
void CMLPCVReportShell::SetAvgRelError(const double d)
{
m_innerobj.m_AvgRelError=d;
}
//+------------------------------------------------------------------+
//| Return object of class |
//+------------------------------------------------------------------+
CMLPCVReport *CMLPCVReportShell::GetInnerObj(void)
{
return(GetPointer(m_innerobj));
}
//+------------------------------------------------------------------+
//| Neural networks ensemble |
//+------------------------------------------------------------------+
class CMLPEnsemble
{
public:
//--- variables
int m_ensemblesize;
CMultilayerPerceptron m_network;
//--- arrays
CRowDouble m_weights;
CRowDouble m_columnmeans;
CRowDouble m_columnsigmas;
CRowDouble m_y;
//--- constructor, destructor
CMLPEnsemble(void) { m_ensemblesize=0; }
~CMLPEnsemble(void) {}
//--- copy
void Copy(const CMLPEnsemble &obj);
//--- overloading
void operator=(const CMLPEnsemble &obj) { Copy(obj); }
};
//+------------------------------------------------------------------+
//| Copy |
//+------------------------------------------------------------------+
void CMLPEnsemble::Copy(const CMLPEnsemble &obj)
{
//--- copy variables
m_ensemblesize=obj.m_ensemblesize;
CMLPBase::MLPCopy(obj.m_network,m_network);
//--- copy arrays
m_weights=obj.m_weights;
m_columnmeans=obj.m_columnmeans;
m_columnsigmas=obj.m_columnsigmas;
m_y=obj.m_y;
}
//+------------------------------------------------------------------+
//| Neural networks ensemble |
//+------------------------------------------------------------------+
class CMLPEnsembleShell
{
private:
CMLPEnsemble m_innerobj;
public:
//--- constructors, destructor
CMLPEnsembleShell(void) {}
CMLPEnsembleShell(CMLPEnsemble &obj) { m_innerobj.Copy(obj); }
~CMLPEnsembleShell(void) {}
//--- method
CMLPEnsemble *GetInnerObj(void) { return(GetPointer(m_innerobj)); }
};
//+------------------------------------------------------------------+
//| Temporary data structures used by following functions: |
//| * TrainNetworkX |
//| * StartTrainingX |
//| * ContinueTrainingX |
//| This structure contains: |
//| * network being trained |
//| * fully initialized LBFGS optimizer (we have to call |
//| MinLBFGSRestartFrom() before using it; usually it is done |
//| by StartTrainingX() function). |
//| * additional temporary arrays |
//| This structure should be initialized with InitMLPTrnSession() |
//| call. |
//+------------------------------------------------------------------+
struct CSMLPTrnSession
{
CRowDouble m_bestparameters;
double m_bestrmserror;
bool m_randomizenetwork;
CMultilayerPerceptron m_network;
CMinLBFGSState m_optimizer;
CMinLBFGSReport m_optimizerrep;
CRowDouble m_wbuf0;
CRowDouble m_wbuf1;
CRowInt m_allminibatches;
CRowInt m_currentminibatch;
RCommState m_rstate;
int m_algoused;
int m_minibatchsize;
CMLPReport m_mlprep;
CRowInt m_trnsubset;
CRowInt m_valsubset;
CHighQualityRandState m_generator;
//--- constructor / destructor
CSMLPTrnSession(void);
~CSMLPTrnSession(void) {}
//---
void Copy(const CSMLPTrnSession &obj);
//--- overloading
void operator=(const CSMLPTrnSession &obj) { Copy(obj); }
};
//+------------------------------------------------------------------+
//| Coonstructor |
//+------------------------------------------------------------------+
CSMLPTrnSession::CSMLPTrnSession(void)
{
m_bestrmserror=0;
m_randomizenetwork=false;
m_algoused=0;
m_minibatchsize=0;
}
//+------------------------------------------------------------------+
//| Copy |
//+------------------------------------------------------------------+
void CSMLPTrnSession::Copy(const CSMLPTrnSession &obj)
{
m_bestparameters=obj.m_bestparameters;
m_bestrmserror=obj.m_bestrmserror;
m_randomizenetwork=obj.m_randomizenetwork;
m_network=obj.m_network;
m_optimizer=obj.m_optimizer;
m_optimizerrep=obj.m_optimizerrep;
m_wbuf0=obj.m_wbuf0;
m_wbuf1=obj.m_wbuf1;
m_allminibatches=obj.m_allminibatches;
m_currentminibatch=obj.m_currentminibatch;
m_rstate=obj.m_rstate;
m_algoused=obj.m_algoused;
m_minibatchsize=obj.m_minibatchsize;
m_mlprep=obj.m_mlprep;
m_trnsubset=obj.m_trnsubset;
m_valsubset=obj.m_valsubset;
m_generator=obj.m_generator;
}
//+------------------------------------------------------------------+
//| Trainer object for neural network. |
//| You should not try to access fields of this object directly - use|
//| ALGLIB functions to work with this object. |
//+------------------------------------------------------------------+
struct CMLPTrainer
{
int m_nin;
int m_nout;
bool m_rcpar;
int m_lbfgsfactor;
double m_decay;
double m_wstep;
int m_maxits;
int m_datatype;
int m_npoints;
CMatrixDouble m_densexy;
CSparseMatrix m_sparsexy;
CSMLPTrnSession m_session;
int m_ngradbatch;
CRowInt m_subset;
int m_subsetsize;
CRowInt m_valsubset;
int m_valsubsetsize;
int m_algokind;
int m_minibatchsize;
//--- constructor / destructor
CMLPTrainer(void);
~CMLPTrainer(void) {}
void Copy(const CMLPTrainer &obj);
//--- overloading
void operator=(const CMLPTrainer &obj) { Copy(obj); }
};
//+------------------------------------------------------------------+
//| Constructor |
//+------------------------------------------------------------------+
CMLPTrainer::CMLPTrainer(void)
{
m_nin=0;
m_nout=0;
m_rcpar=false;
m_lbfgsfactor=0;
m_decay=0;
m_wstep=0;
m_maxits=0;
m_datatype=0;
m_npoints=0;
m_ngradbatch=0;
m_subsetsize=0;
m_valsubsetsize=0;
m_algokind=0;
m_minibatchsize=0;
}
//+------------------------------------------------------------------+
//| Copy |
//+------------------------------------------------------------------+
void CMLPTrainer::Copy(const CMLPTrainer &obj)
{
m_nin=obj.m_nin;
m_nout=obj.m_nout;
m_rcpar=obj.m_rcpar;
m_lbfgsfactor=obj.m_lbfgsfactor;
m_decay=obj.m_decay;
m_wstep=obj.m_wstep;
m_maxits=obj.m_maxits;
m_datatype=obj.m_datatype;
m_npoints=obj.m_npoints;
m_densexy=obj.m_densexy;
m_sparsexy=obj.m_sparsexy;
m_session=obj.m_session;
m_ngradbatch=obj.m_ngradbatch;
m_subset=obj.m_subset;
m_subsetsize=obj.m_subsetsize;
m_valsubset=obj.m_valsubset;
m_valsubsetsize=obj.m_valsubsetsize;
m_algokind=obj.m_algokind;
m_minibatchsize=obj.m_minibatchsize;
}
//+------------------------------------------------------------------+
//| Training neural networks |
//+------------------------------------------------------------------+
class CMLPTrain
{
public:
//--- constant
static const double m_mindecay;
static const int m_defaultlbfgsfactor;
//--- public methods
static void MLPTrainLM(CMultilayerPerceptron &network,CMatrixDouble &xy,const int npoints,double decay,const int restarts,int &info,CMLPReport &rep);
static void MLPTrainLBFGS(CMultilayerPerceptron &network,CMatrixDouble &xy,const int npoints,double decay,const int restarts,const double wstep,int maxits,int &info,CMLPReport &rep);
static void MLPTrainES(CMultilayerPerceptron &network,CMatrixDouble &trnxy,const int trnsize,CMatrixDouble &valxy,const int valsize,const double decay,int restarts,int &info,CMLPReport &rep);
static void MLPKFoldCVLBFGS(CMultilayerPerceptron &network,CMatrixDouble &xy,const int npoints,const double decay,const int restarts,const double wstep,const int maxits,const int foldscount,int &info,CMLPReport &rep,CMLPCVReport &cvrep);
static void MLPKFoldCVLM(CMultilayerPerceptron &network,CMatrixDouble &xy,const int npoints,const double decay,const int restarts,int foldscount,int &info,CMLPReport &rep,CMLPCVReport &cvrep);
static void MLPCreateTrainer(int nin,int nout,CMLPTrainer &s);
static void MLPCreateTrainerCls(int nin,int nclasses,CMLPTrainer &s);
static void MLPSetDataset(CMLPTrainer &s,CMatrixDouble &xy,int npoints);
static void MLPSetSparseDataset(CMLPTrainer &s,CSparseMatrix &xy,int npoints);
static void MLPSetDecay(CMLPTrainer &s,double decay);
static void MLPSetCond(CMLPTrainer &s,double wstep,int maxits);
static void MLPSetAlgoBatch(CMLPTrainer &s);
static void MLPTrainNetwork(CMLPTrainer &s,CMultilayerPerceptron &network,int nrestarts,CMLPReport &rep);
static void MLPStartTraining(CMLPTrainer &s,CMultilayerPerceptron &network,bool randomstart);
static bool MLPContinueTraining(CMLPTrainer &s,CMultilayerPerceptron &network);
static void MLPEBaggingLM(CMLPEnsemble &ensemble,CMatrixDouble &xy,const int npoints,const double decay,const int restarts,int &info,CMLPReport &rep,CMLPCVReport &ooberrors);
static void MLPEBaggingLBFGS(CMLPEnsemble &ensemble,CMatrixDouble &xy,const int npoints,const double decay,const int restarts,const double wstep,const int maxits,int &info,CMLPReport &rep,CMLPCVReport &ooberrors);
static void MLPETrainES(CMLPEnsemble &ensemble,CMatrixDouble &xy,const int npoints,const double decay,const int restarts,int &info,CMLPReport &rep);
static void MLPTrainEnsembleES(CMLPTrainer &s,CMLPEnsemble &ensemble,int nrestarts,CMLPReport &rep);
private:
static void MLPKFoldCVGeneral(CMultilayerPerceptron &n,CMatrixDouble &xy,const int npoints,const double decay,const int restarts,const int foldscount,const bool lmalgorithm,const double wstep,const int maxits,int &info,CMLPReport &rep,CMLPCVReport &cvrep);
static void MLPKFoldSplit(CMatrixDouble &xy,const int npoints,const int nclasses,const int foldscount,const bool stratifiedsplits,CRowInt &folds);
static void MLPTrainNetworkX(CMLPTrainer &s,int nrestarts,int algokind,CRowInt &trnsubset,int trnsubsetsize,CRowInt &valsubset,int valsubsetsize,CMultilayerPerceptron &network,CMLPReport &rep);
static void MLPTrainEnsembleX(CMLPTrainer &s,CMLPEnsemble &ensemble,int idx0,int idx1,int nrestarts,int trainingmethod,int ngrad);
static void MLPStartTrainingX(CMLPTrainer &s,bool randomstart,int algokind,CRowInt &subset,int subsetsize,CSMLPTrnSession &session);
static bool MLPContinueTrainingX(CMLPTrainer &s,CRowInt &subset,int subsetsize,int &ngradbatch,CSMLPTrnSession &session);
static bool MLPContinueTrainingX_lbl1(CMLPTrainer &s,CRowInt &subset,int subsetsize,int &ngradbatch,CSMLPTrnSession &session,double &decay);
static bool MLPContinueTrainingX_lbl3(CMLPTrainer &s,CRowInt &subset,int subsetsize,int &ngradbatch,CSMLPTrnSession &session,double &decay);
static void MLPEBaggingInternal(CMLPEnsemble &ensemble,CMatrixDouble &xy,int npoints,double decay,int restarts,double wstep,int maxits,bool lmalgorithm,int &info,CMLPReport &rep,CMLPCVReport &ooberrors);
static void InitMLPTrnSession(CMultilayerPerceptron &networktrained,bool randomizenetwork,CMLPTrainer &trainer,CSMLPTrnSession &session);
};
//+------------------------------------------------------------------+
//| Initialize constant |
//+------------------------------------------------------------------+
const double CMLPTrain::m_mindecay=0.001;
const int CMLPTrain::m_defaultlbfgsfactor=6;
//+------------------------------------------------------------------+
//| Neural network training using modified Levenberg-Marquardt with |
//| exact Hessian calculation and regularization. Subroutine trains |
//| neural network with restarts from random positions. Algorithm is |
//| well suited for small |
//| and medium scale problems (hundreds of weights). |
//| INPUT PARAMETERS: |
//| Network - neural network with initialized geometry |
//| XY - training set |
//| NPoints - training set size |
//| Decay - weight decay constant, >=0.001 |
//| Decay term 'Decay*||Weights||^2' is added to |
//| error function. |
//| If you don't know what Decay to choose, use |
//| 0.001. |
//| Restarts - number of restarts from random position, >0. |
//| If you don't know what Restarts to choose, |
//| use 2. |
//| OUTPUT PARAMETERS: |
//| Network - trained neural network. |
//| Info - return code: |
//| * -9, if internal matrix inverse subroutine |
//| failed |
//| * -2, if there is a point with class number |
//| outside of [0..NOut-1]. |
//| * -1, if wrong parameters specified |
//| (NPoints<0, Restarts<1). |
//| * 2, if task has been solved. |
//| Rep - training report |
//+------------------------------------------------------------------+
void CMLPTrain::MLPTrainLM(CMultilayerPerceptron &network,CMatrixDouble &xy,
const int npoints,double decay,const int restarts,
int &info,CMLPReport &rep)
{
//--- create variables
int nin=0;
int nout=0;
int wcount=0;
double lmm_ftol=0;
double lmsteptol=0;
int i=0;
int k=0;
double v=0;
double e=0;
double enew=0;
double xnorm2=0;
double stepnorm=0;
bool spd;
double nu=0;
double lambdav=0;
double lambdaup=0;
double lambdadown=0;
int pass=0;
double ebest=0;
int invinfo=0;
int solverinfo=0;
int i_=0;
//--- creating arrays
CRowDouble g;
vector<double> d;
vector<double> x;
vector<double> y;
CRowDouble wbase;
CRowDouble wdir;
CRowDouble wt;
vector<double> wx;
CRowDouble wbest;
//--- create matrix
CMatrixDouble h;
CMatrixDouble hmod;
CMatrixDouble z;
//--- objects of classes
CMinLBFGSReport internalrep;
CMinLBFGSState state;
CMatInvReport invrep;
CDenseSolverReport solverrep;
//--- initialization
info=0;
//--- function call
CMLPBase::MLPProperties(network,nin,nout,wcount);
//--- initialization
lambdaup=10;
lambdadown=0.3;
lmm_ftol=0.001;
lmsteptol=0.001;
//--- Test for inputs
if(npoints<=0 || restarts<1)
{
info=-1;
return;
}
//--- check
if(CMLPBase::MLPIsSoftMax(network))
{
y=xy.Col(nin);
//--- check
if((int)MathRound(y.Min())<0 || (int)MathRound(y.Max())>=nout)
{
info=-2;
return;
}
}
//--- change values
decay=MathMax(decay,m_mindecay);
info=2;
//--- Initialize data
rep.m_ngrad=0;
rep.m_nhess=0;
rep.m_ncholesky=0;
//--- General case.
//--- Prepare task and network. Allocate space.
CMLPBase::MLPInitPreprocessor(network,xy,npoints);
//--- allocation
g.Resize(wcount);
h.Resize(wcount,wcount);
hmod.Resize(wcount,wcount);
wbase.Resize(wcount);
wdir.Resize(wcount);
wbest.Resize(wcount);
wt.Resize(wcount);
wx.Resize(wcount);
//--- initialization
ebest=CMath::m_maxrealnumber;
//--- Multiple passes
for(pass=1; pass<=restarts; pass++)
{
//--- Initialize weights
CMLPBase::MLPRandomize(network);
//--- First stage of the hybrid algorithm: LBFGS
wbase=network.m_weights;
//--- function calls
CMinLBFGS::MinLBFGSCreate(wcount,(int)(MathMin(wcount,5)),wbase,state);
CMinLBFGS::MinLBFGSSetCond(state,0,0,0,(int)(MathMax(25,wcount)));
while(CMinLBFGS::MinLBFGSIteration(state))
{
//--- gradient
network.m_weights=state.m_x;
//--- function call
CMLPBase::MLPGradBatch(network,xy,npoints,state.m_f,state.m_g);
//--- weight decay
v=network.m_weights.Dot(network.m_weights);
state.m_f+=0.5*decay*v;
state.m_g+=network.m_weights*decay+0;
//--- next iteration
rep.m_ngrad++;
}
//--- function call
CMinLBFGS::MinLBFGSResults(state,wbase,internalrep);
network.m_weights=wbase;
//--- Second stage of the hybrid algorithm: LM
//--- Initialize H with identity matrix,
//--- G with gradient,
//--- E with regularized error.
CMLPBase::MLPHessianBatch(network,xy,npoints,e,g,h);
v=network.m_weights.Dot(network.m_weights);
//--- change values
e=e+0.5*decay*v;
g+=network.m_weights*decay+0;
h.Diag(h.Diag()+decay);
//--- change values
rep.m_nhess++;
lambdav=0.001;
nu=2;
//--- cycle
while(true)
{
//--- 1. HMod=H+lambda*I
//--- 2. Try to solve (H+Lambda*I)*dx=-g.
//--- Increase lambda if left part is not positive definite.
hmod=h;
hmod.Diag(hmod.Diag(0)+lambdav);
//--- function call
spd=CTrFac::SPDMatrixCholesky(hmod,wcount,true);
rep.m_ncholesky++;
//--- check
if(!spd)
{
lambdav=lambdav*lambdaup*nu;
nu*=2;
continue;
}
//--- function call
CDenseSolver::SPDMatrixCholeskySolve(hmod,wcount,true,g,solverinfo,solverrep,wdir);
//--- check
if(solverinfo<0)
{
lambdav=lambdav*lambdaup*nu;
nu*=2;
continue;
}
wdir*=(-1);
//--- Lambda found.
//--- 1. Save old w in WBase
//--- 1. Test some stopping criterions
//--- 2. If error(w+wdir)>error(w),increase lambda
network.m_weights+=wdir;
xnorm2=network.m_weights.Dot(network.m_weights);
//--- change value
stepnorm=MathSqrt(wdir.Dot(wdir));
//--- function call
enew=CMLPBase::MLPError(network,xy,npoints)+0.5*decay*xnorm2;
//--- check
if(stepnorm<lmsteptol*(1+MathSqrt(xnorm2)))
break;
//--- check
if(enew>e)
{
lambdav*=lambdaup*nu;
nu*=2;
continue;
}
//--- Optimize using inv(cholesky(H)) as preconditioner
CMatInv::RMatrixTrInverse(hmod,wcount,true,false,invinfo,invrep);
//--- check
if(invinfo<=0)
{
//--- if matrix can't be inverted then exit with errors
//--- TODO: make WCount steps in direction suggested by HMod
info=-9;
return;
}
//--- calculation
wbase=network.m_weights;
wt=vector<double>::Zeros(wcount);
//--- function calls
CMinLBFGS::MinLBFGSCreateX(wcount,wcount,wt,1,0.0,state);
CMinLBFGS::MinLBFGSSetCond(state,0,0,0,5);
while(CMinLBFGS::MinLBFGSIteration(state))
{
//--- gradient
network.m_weights=wbase.ToVector()+state.m_x.MatMul(hmod.Transpose()+0);
//--- function call
CMLPBase::MLPGradBatch(network,xy,npoints,state.m_f,g);
state.m_g=g.MatMul(hmod)+0;
//--- weight decay
//--- grad(x'*x)=A'*(x0+A*t)
v=network.m_weights.Dot(network.m_weights);
state.m_f+=0.5*decay*v;
for(i=0; i<wcount; i++)
{
v=decay*network.m_weights[i];
state.m_g+=hmod[i]*v;
}
//--- next iteration
rep.m_ngrad++;
}
//--- function call
CMinLBFGS::MinLBFGSResults(state,wt,internalrep);
//--- Accept new position.
//--- Calculate Hessian
network.m_weights=wbase.ToVector()+wt.MatMul(hmod.Transpose()+0);
//--- function call
CMLPBase::MLPHessianBatch(network,xy,npoints,e,g,h);
v=network.m_weights.Dot(network.m_weights);
//--- change value
e+=0.5*decay*v;
g+=network.m_weights*decay+0;
h.Diag(h.Diag()+decay);
rep.m_nhess++;
//--- Update lambda
lambdav=lambdav*lambdadown;
nu=2;
}
//--- update WBest
v=network.m_weights.Dot(network.m_weights);
//--- change value
e=0.5*decay*v+CMLPBase::MLPError(network,xy,npoints);
//--- check
if(e<ebest)
{
ebest=e;
wbest=network.m_weights;
}
}
//--- copy WBest to output
network.m_weights=wbest;
}
//+------------------------------------------------------------------+
//| Neural network training using L-BFGS algorithm with |
//| regularization. Subroutine trains neural network with restarts |
//| from random positions. Algorithm is well suited for problems of |
//| any dimensionality (memory requirements and step complexity are |
//| linear by weights number). |
//| INPUT PARAMETERS: |
//| Network - neural network with initialized geometry |
//| XY - training set |
//| NPoints - training set size |
//| Decay - weight decay constant, >=0.001 |
//| Decay term 'Decay*||Weights||^2' is added to |
//| error function. |
//| If you don't know what Decay to choose, use |
//| 0.001. |
//| Restarts - number of restarts from random position, >0. |
//| If you don't know what Restarts to choose, |
//| use 2. |
//| WStep - stopping criterion. Algorithm stops if step |
//| size is less than WStep. Recommended |
//| value - 0.01. Zero step size means stopping |
//| after MaxIts iterations. |
//| MaxIts - stopping criterion. Algorithm stops after |
//| MaxIts iterations (NOT gradient calculations).|
//| Zero MaxIts means stopping when step is |
//| sufficiently small. |
//| OUTPUT PARAMETERS: |
//| Network - trained neural network. |
//| Info - return code: |
//| * -8, if both WStep=0 and MaxIts=0 |
//| * -2, if there is a point with class number |
//| outside of [0..NOut-1]. |
//| * -1, if wrong parameters specified |
//| (NPoints<0, Restarts<1). |
//| * 2, if task has been solved. |
//| Rep - training report |
//+------------------------------------------------------------------+
void CMLPTrain::MLPTrainLBFGS(CMultilayerPerceptron &network,
CMatrixDouble &xy,const int npoints,
double decay,const int restarts,
const double wstep,int maxits,
int &info,CMLPReport &rep)
{
//--- create variables
int nin=0;
int nout=0;
int wcount=0;
double e=0;
double v=0;
double ebest=0;
//--- creating arrays
CRowDouble w;
CRowDouble wbest;
//--- create objects of classes
CMinLBFGSReport internalrep;
CMinLBFGSState state;
//--- initialization
info=0;
//--- Test inputs,parse flags,read network geometry
if(wstep==0.0 && maxits==0)
{
info=-8;
return;
}
//--- check
if(npoints<=0 || restarts<1 || wstep<0.0 || maxits<0)
{
info=-1;
return;
}
//--- function call
CMLPBase::MLPProperties(network,nin,nout,wcount);
//--- check
if(CMLPBase::MLPIsSoftMax(network))
{
w=xy.Col(nin)+0;
//--- check
if((int)MathRound(w.Min())<0 || (int)MathRound(w.Max())>=nout)
{
info=-2;
return;
}
}
//--- change values
decay=MathMax(decay,m_mindecay);
info=2;
//--- Prepare
CMLPBase::MLPInitPreprocessor(network,xy,npoints);
//--- allocation
wbest.Resize(wcount);
//--- initialization
ebest=CMath::m_maxrealnumber;
//--- Multiple starts
rep.m_ncholesky=0;
rep.m_nhess=0;
rep.m_ngrad=0;
for(int pass=1; pass<=restarts; pass++)
{
//--- Process
CMLPBase::MLPRandomize(network);
w=network.m_weights;
//--- function calls
CMinLBFGS::MinLBFGSCreate(wcount,(int)(MathMin(wcount,10)),w,state);
CMinLBFGS::MinLBFGSSetCond(state,0.0,0.0,wstep,maxits);
while(CMinLBFGS::MinLBFGSIteration(state))
{
network.m_weights=state.m_x;
//--- function call
CMLPBase::MLPGradNBatch(network,xy,npoints,state.m_f,state.m_g);
v=network.m_weights.Dot(network.m_weights);
state.m_f=state.m_f+0.5*decay*v;
state.m_g+=network.m_weights*decay+0;
rep.m_ngrad++;
}
//--- function call
CMinLBFGS::MinLBFGSResults(state,w,internalrep);
network.m_weights=w;
//--- Compare with best
v=network.m_weights.Dot(network.m_weights);
//--- change value
e=CMLPBase::MLPErrorN(network,xy,npoints)+0.5*decay*v;
//--- check
if(e<ebest)
{
wbest=network.m_weights;
ebest=e;
}
}
//--- The best network
network.m_weights=wbest;
}
//+------------------------------------------------------------------+
//| Neural network training using early stopping (base algorithm - |
//| L-BFGS with regularization). |
//| INPUT PARAMETERS: |
//| Network - neural network with initialized geometry |
//| TrnXY - training set |
//| TrnSize - training set size |
//| ValXY - validation set |
//| ValSize - validation set size |
//| Decay - weight decay constant, >=0.001 |
//| Decay term 'Decay*||Weights||^2' is added to |
//| error function. |
//| If you don't know what Decay to choose, use |
//| 0.001. |
//| Restarts - number of restarts from random position, >0. |
//| If you don't know what Restarts to choose, |
//| use 2. |
//| OUTPUT PARAMETERS: |
//| Network - trained neural network. |
//| Info - return code: |
//| * -2, if there is a point with class number |
//| outside of [0..NOut-1]. |
//| * -1, if wrong parameters specified |
//| (NPoints<0, Restarts<1, ...). |
//| * 2, task has been solved, stopping |
//| criterion met - sufficiently small |
//| step size. Not expected (we use EARLY |
//| stopping) but possible and not an error|
//| * 6, task has been solved, stopping |
//| criterion met - increasing of |
//| validation set error. |
//| Rep - training report |
//| NOTE: |
//| Algorithm stops if validation set error increases for a long |
//| enough or step size is small enought (there are task where |
//| validation set may decrease for eternity). In any case solution |
//| returned corresponds to the minimum of validation set error. |
//+------------------------------------------------------------------+
void CMLPTrain::MLPTrainES(CMultilayerPerceptron &network,
CMatrixDouble &trnxy,const int trnsize,
CMatrixDouble &valxy,const int valsize,
const double decay,int restarts,
int &info,CMLPReport &rep)
{
//--- create variables
int nin=0;
int nout=0;
int wcount=0;
double e=0;
double v=0;
double ebest=0;
int itbest=0;
int itcnt=0;
double wstep=0;
bool needrandomization=true;
//--- creating arrays
CRowDouble w;
CRowDouble wbest;
CRowDouble wfinal;
double efinal=0;
//--- objects of classes
CMinLBFGSReport internalrep;
CMinLBFGSState state;
//--- initialization
info=0;
wstep=0.001;
//--- Test inputs,parse flags,read network geometry
if(trnsize<=0 || valsize<=0 || restarts<1 || decay<0.0)
{
info=-1;
return;
}
if(restarts==-1)
{
needrandomization=false;
restarts=1;
}
//--- function call
CMLPBase::MLPProperties(network,nin,nout,wcount);
//--- check
if(CMLPBase::MLPIsSoftMax(network))
{
w=trnxy.Col(nin)+0;
w.Resize(trnsize);
//--- check
if((int)MathRound(w.Min())<0 || (int)MathRound(w.Max())>=nout)
{
info=-2;
return;
}
w=valxy.Col(nin)+0;
w.Resize(valsize);
//--- check
if((int)MathRound(w.Min())<0 || (int)MathRound(w.Max())>=nout)
{
info=-2;
return;
}
}
//--- change value
info=2;
//--- Prepare
CMLPBase::MLPInitPreprocessor(network,trnxy,trnsize);
//--- initialization
wfinal=vector<double>::Zeros(wcount);
efinal=CMath::m_maxrealnumber;
//--- Multiple starts
rep.m_ncholesky=0;
rep.m_nhess=0;
rep.m_ngrad=0;
//--- calculation
for(int pass=1; pass<=restarts; pass++)
{
//--- Process
if(needrandomization)
CMLPBase::MLPRandomize(network);
//--- change values
ebest=CMLPBase::MLPError(network,valxy,valsize);
wbest=network.m_weights;
//--- change values
itbest=0;
w=network.m_weights;
//--- function calls
CMinLBFGS::MinLBFGSCreate(wcount,(int)(MathMin(wcount,10)),w,state);
CMinLBFGS::MinLBFGSSetCond(state,0.0,0.0,wstep,0);
CMinLBFGS::MinLBFGSSetXRep(state,true);
while(CMinLBFGS::MinLBFGSIteration(state))
{
//--- Calculate gradient
if(state.m_needfg)
{
network.m_weights=state.m_x;
//--- function call
CMLPBase::MLPGradNBatch(network,trnxy,trnsize,state.m_f,state.m_g);
v=network.m_weights.Dot(network.m_weights);
state.m_f+=0.5*decay*v;
state.m_g+=network.m_weights*decay+0;
rep.m_ngrad++;
}
//--- Validation set
if(state.m_xupdated)
{
network.m_weights=state.m_x;
//--- function call
e=CMLPBase::MLPError(network,valxy,valsize);
//--- check
if(e<ebest)
{
ebest=e;
wbest=network.m_weights;
itbest=internalrep.m_iterationscount;
}
//--- check
if(itcnt>30 && (double)(itcnt)>(double)(1.5*itbest))
{
info=6;
break;
}
itcnt++;
}
}
//--- function call
CMinLBFGS::MinLBFGSResults(state,w,internalrep);
//--- Compare with final answer
if(ebest<efinal)
{
wfinal=wbest;
efinal=ebest;
}
}
//--- The best network
network.m_weights=wfinal;
}
//+------------------------------------------------------------------+
//| Cross-validation estimate of generalization error. |
//| Base algorithm - L-BFGS. |
//| INPUT PARAMETERS: |
//| Network - neural network with initialized geometry. |
//| Network is not changed during |
//| cross-validation - it is used only as a |
//| representative of its architecture. |
//| XY - training set. |
//| SSize - training set size |
//| Decay - weight decay, same as in MLPTrainLBFGS |
//| Restarts - number of restarts, >0. |
//| restarts are counted for each partition |
//| separately, so total number of restarts will |
//| be Restarts*FoldsCount. |
//| WStep - stopping criterion, same as in MLPTrainLBFGS |
//| MaxIts - stopping criterion, same as in MLPTrainLBFGS |
//| FoldsCount - number of folds in k-fold cross-validation, |
//| 2<=FoldsCount<=SSize. |
//| recommended value: 10. |
//| OUTPUT PARAMETERS: |
//| Info - return code, same as in MLPTrainLBFGS |
//| Rep - report, same as in MLPTrainLM/MLPTrainLBFGS |
//| CVRep - generalization error estimates |
//+------------------------------------------------------------------+
void CMLPTrain::MLPKFoldCVLBFGS(CMultilayerPerceptron &network,
CMatrixDouble &xy,const int npoints,
const double decay,const int restarts,
const double wstep,const int maxits,
const int foldscount,int &info,
CMLPReport &rep,CMLPCVReport &cvrep)
{
//--- initialization
info=0;
//--- function call
MLPKFoldCVGeneral(network,xy,npoints,decay,restarts,foldscount,false,wstep,maxits,info,rep,cvrep);
}
//+------------------------------------------------------------------+
//| Cross-validation estimate of generalization error. |
//| Base algorithm - Levenberg-Marquardt. |
//| INPUT PARAMETERS: |
//| Network - neural network with initialized geometry. |
//| Network is not changed during |
//| cross-validation - it is used only as a |
//| representative of its architecture. |
//| XY - training set. |
//| SSize - training set size |
//| Decay - weight decay, same as in MLPTrainLBFGS |
//| Restarts - number of restarts, >0. |
//| restarts are counted for each partition |
//| separately, so total number of restarts will |
//| be Restarts*FoldsCount. |
//| FoldsCount - number of folds in k-fold cross-validation, |
//| 2<=FoldsCount<=SSize. |
//| recommended value: 10. |
//| OUTPUT PARAMETERS: |
//| Info - return code, same as in MLPTrainLBFGS |
//| Rep - report, same as in MLPTrainLM/MLPTrainLBFGS |
//| CVRep - generalization error estimates |
//+------------------------------------------------------------------+
void CMLPTrain::MLPKFoldCVLM(CMultilayerPerceptron &network,
CMatrixDouble &xy,const int npoints,
const double decay,const int restarts,
int foldscount,int &info,CMLPReport &rep,
CMLPCVReport &cvrep)
{
//--- initialization
info=0;
//--- function call
MLPKFoldCVGeneral(network,xy,npoints,decay,restarts,foldscount,true,0.0,0,info,rep,cvrep);
}
//+------------------------------------------------------------------+
//| Creation of the network trainer object for regression networks |
//| INPUT PARAMETERS: |
//| NIn - number of inputs, NIn>=1 |
//| NOut - number of outputs, NOut>=1 |
//| OUTPUT PARAMETERS: |
//| S - neural network trainer object. |
//| This structure can be used to train any regression network with |
//| NIn inputs and NOut outputs. |
//+------------------------------------------------------------------+
void CMLPTrain::MLPCreateTrainer(int nin,int nout,CMLPTrainer &s)
{
//--- check
if(!CAp::Assert(nin>=1,__FUNCTION__+": NIn<1."))
return;
if(!CAp::Assert(nout>=1,__FUNCTION__+": NOut<1."))
return;
//--- change value
s.m_nin=nin;
s.m_nout=nout;
s.m_rcpar=true;
s.m_lbfgsfactor=m_defaultlbfgsfactor;
s.m_decay=1.0E-6;
MLPSetCond(s,0,0);
s.m_datatype=0;
s.m_npoints=0;
MLPSetAlgoBatch(s);
}
//+------------------------------------------------------------------+
//| Creation of the network trainer object for classification |
//| networks |
//| INPUT PARAMETERS: |
//| NIn - number of inputs, NIn>=1 |
//| NClasses - number of classes, NClasses>=2 |
//| OUTPUT PARAMETERS: |
//| S - neural network trainer object. |
//| This structure can be used to train any classification network |
//| with NIn inputs and NOut outputs. |
//+------------------------------------------------------------------+
void CMLPTrain::MLPCreateTrainerCls(int nin,int nclasses,CMLPTrainer &s)
{
//--- check
if(!CAp::Assert(nin>=1,__FUNCTION__+": NIn<1."))
return;
if(!CAp::Assert(nclasses>=2,__FUNCTION__+": NClasses<2."))
return;
s.m_nin=nin;
s.m_nout=nclasses;
s.m_rcpar=false;
s.m_lbfgsfactor=m_defaultlbfgsfactor;
s.m_decay=1.0E-6;
MLPSetCond(s,0,0);
s.m_datatype=0;
s.m_npoints=0;
MLPSetAlgoBatch(s);
}
//+------------------------------------------------------------------+
//| This function sets "current dataset" of the trainer object to one|
//| passed by user. |
//| INPUT PARAMETERS: |
//| S - trainer object |
//| XY - training set, see below for information on the |
//| training set format. This function checks |
//| correctness of the dataset (no NANs/INFs, class |
//| numbers are correct) and throws exception when |
//| incorrect dataset is passed. |
//| NPoints - points count, >=0. |
//| DATASET FORMAT: |
//| This function uses two different dataset formats - one for |
//| regression networks, another one for classification networks. |
//| For regression networks with NIn inputs and NOut outputs |
//| following dataset format is used: |
//| * dataset is given by NPoints*(NIn+NOut) matrix |
//| * each row corresponds to one example |
//| * first NIn columns are inputs, next NOut columns are |
//| outputs |
//| For classification networks with NIn inputs and NClasses clases |
//| following datasetformat is used: |
//| * dataset is given by NPoints*(NIn+1) matrix |
//| * each row corresponds to one example |
//| * first NIn columns are inputs, last column stores class |
//| number (from 0 to NClasses-1). |
//+------------------------------------------------------------------+
void CMLPTrain::MLPSetDataset(CMLPTrainer &s,CMatrixDouble &xy,int npoints)
{
int ndim=0;
//--- check
if(!CAp::Assert(s.m_nin>=1,__FUNCTION__+": possible parameter S is not initialized or spoiled(S.NIn<=0)."))
return;
if(!CAp::Assert(npoints>=0,__FUNCTION__+": NPoint<0"))
return;
if(!CAp::Assert(npoints<=xy.Rows(),__FUNCTION__+": invalid size of matrix XY(NPoint more then rows of matrix XY)"))
return;
//--- change value
s.m_datatype=0;
s.m_npoints=npoints;
if(npoints==0)
return;
if(s.m_rcpar)
{
//--- check
if(!CAp::Assert(s.m_nout>=1,__FUNCTION__+": possible parameter S is not initialized or is spoiled(NOut<1 for regression)."))
return;
ndim=s.m_nin+s.m_nout;
//--- check
if(!CAp::Assert(ndim<=xy.Cols(),__FUNCTION__+": invalid size of matrix XY(too few columns in matrix XY)."))
return;
if(!CAp::Assert(CApServ::IsFiniteMatrix(xy,npoints,ndim),__FUNCTION__+": parameter XY contains Infinite or NaN."))
return;
}
else
{
//--- check
if(!CAp::Assert(s.m_nout>=2,__FUNCTION__+": possible parameter S is not initialized or is spoiled(NClasses<2 for classifier)."))
return;
ndim=s.m_nin+1;
//--- check
if(!CAp::Assert(ndim<=xy.Cols(),__FUNCTION__+": invalid size of matrix XY(too few columns in matrix XY)."))
return;
if(!CAp::Assert(CApServ::IsFiniteMatrix(xy,npoints,ndim),__FUNCTION__+": parameter XY contains Infinite or NaN."))
return;
vector<double> temp=xy.Col(s.m_nin);
temp.Resize(npoints);
//--- check
if(!CAp::Assert((int)MathRound(temp.Min())>=0 && (int)MathRound(temp.Max())<s.m_nout,__FUNCTION__+": invalid parameter XY(in classifier used nonexistent class number: either XY[.,NIn]<0 or XY[.,NIn]>=NClasses)."))
return;
}
s.m_densexy=xy;
s.m_densexy.Resize(npoints,ndim);
}
//+------------------------------------------------------------------+
//| This function sets "current dataset" of the trainer object to one|
//| passed by user (sparse matrix is used to store dataset). |
//| INPUT PARAMETERS: |
//| S - trainer object |
//| XY - training set, see below for information on the |
//| training set format. This function checks |
//| correctness of the dataset (no NANs/INFs, class |
//| numbers are correct) and throws exception when |
//| incorrect dataset is passed. Any sparse storage |
//| format can be used: Hash-table, CRS... |
//| NPoints - points count, >=0 |
//| DATASET FORMAT: |
//| This function uses two different dataset formats - one for |
//| regression networks, another one for classification networks. |
//| For regression networks with NIn inputs and NOut outputs |
//| following dataset format is used: |
//| * dataset is given by NPoints*(NIn+NOut) matrix |
//| * each row corresponds to one example |
//| * first NIn columns are inputs, next NOut columns are outputs |
//| For classification networks with NIn inputs and NClasses clases |
//| following datasetformat is used: |
//| * dataset is given by NPoints*(NIn+1) matrix |
//| * each row corresponds to one example |
//| * first NIn columns are inputs, last column stores class number|
//| (from 0 to NClasses-1). |
//+------------------------------------------------------------------+
void CMLPTrain::MLPSetSparseDataset(CMLPTrainer &s,CSparseMatrix &xy,int npoints)
{
//--- create variables
double v=0;
int t0=0;
int t1=0;
int i=0;
int j=0;
//--- Check correctness of the data
if(!CAp::Assert(s.m_nin>0,__FUNCTION__+": possible parameter S is not initialized or spoiled(S.NIn<=0)."))
return;
if(!CAp::Assert(npoints>=0,__FUNCTION__+": NPoint<0"))
return;
if(!CAp::Assert(npoints<=CSparse::SparseGetNRows(xy),__FUNCTION__+": invalid size of sparse matrix XY(NPoint more then rows of matrix XY)"))
return;
if(npoints>0)
{
t0=0;
t1=0;
if(s.m_rcpar)
{
if(!CAp::Assert(s.m_nout>=1,__FUNCTION__+": possible parameter S is not initialized or is spoiled(NOut<1 for regression)."))
return;
if(!CAp::Assert(s.m_nin+s.m_nout<=CSparse::SparseGetNCols(xy),__FUNCTION__+": invalid size of sparse matrix XY(too few columns in sparse matrix XY)."))
return;
//---
while(CSparse::SparseEnumerate(xy,t0,t1,i,j,v))
{
if(i<npoints && j<s.m_nin+s.m_nout)
if(!CAp::Assert(MathIsValidNumber(v),__FUNCTION__+": sparse matrix XY contains Infinite or NaN."))
return;
}
}
else
{
if(!CAp::Assert(s.m_nout>=2,__FUNCTION__+": possible parameter S is not initialized or is spoiled(NClasses<2 for classifier)."))
return;
if(!CAp::Assert(s.m_nin+1<=CSparse::SparseGetNCols(xy),__FUNCTION__+": invalid size of sparse matrix XY(too few columns in sparse matrix XY)."))
return;
//---
while(CSparse::SparseEnumerate(xy,t0,t1,i,j,v))
{
if(i<npoints && j<=s.m_nin)
{
if(j!=s.m_nin)
{
if(!CAp::Assert(MathIsValidNumber(v),__FUNCTION__+": sparse matrix XY contains Infinite or NaN."))
return;
}
else
{
if(!CAp::Assert((MathIsValidNumber(v) && (int)MathRound(v)>=0) && (int)MathRound(v)<s.m_nout,__FUNCTION__+": invalid sparse matrix XY(in classifier used nonexistent class number: either XY[.,NIn]<0 or XY[.,NIn]>=NClasses)."))
return;
}
}
}
}
}
//--- Set dataset
s.m_datatype=1;
s.m_npoints=npoints;
CSparse::SparseCopyToCRS(xy,s.m_sparsexy);
}
//+------------------------------------------------------------------+
//| This function sets weight decay coefficient which is used for |
//| training. |
//| INPUT PARAMETERS: |
//| S - trainer object |
//| Decay - weight decay coefficient, >=0. Weight decay term |
//| 'Decay*||Weights||^2' is added to error function. |
//| If you don't know what Decay to choose, use 1.0E-3.|
//| Weight decay can be set to zero, in this case |
//| network is trained without weight decay. |
//| NOTE: by default network uses some small nonzero value for weight|
//| decay. |
//+------------------------------------------------------------------+
void CMLPTrain::MLPSetDecay(CMLPTrainer &s,double decay)
{
//--- check
if(!CAp::Assert(MathIsValidNumber(decay),__FUNCTION__+": parameter Decay contains Infinite or NaN."))
return;
if(!CAp::Assert(decay>=0.0,__FUNCTION__+": Decay<0."))
return;
s.m_decay=decay;
}
//+------------------------------------------------------------------+
//| This function sets stopping criteria for the optimizer. |
//| INPUT PARAMETERS: |
//| S - trainer object |
//| WStep - stopping criterion. Algorithm stops if step size is|
//| less than WStep. Recommended value - 0.01. Zero |
//| step size means stopping after MaxIts iterations. |
//| WStep>=0. |
//| MaxIts - stopping criterion. Algorithm stops after MaxIts |
//| epochs (full passes over entire dataset). Zero |
//| MaxIts means stopping when step is sufficiently |
//| small. MaxIts>=0. |
//| NOTE: by default, WStep=0.005 and MaxIts=0 are used. These values|
//| are also used when MLPSetCond() is called with WStep=0 and |
//| MaxIts=0. |
//| NOTE: these stopping criteria are used for all kinds of neural |
//| training-from "conventional" networks to early stopping |
//| ensembles. When used for "conventional" networks, they are |
//| used as the only stopping criteria. When combined with |
//| early stopping, they used as ADDITIONAL stopping criteria |
//| which can terminate early stopping algorithm. |
//+------------------------------------------------------------------+
void CMLPTrain::MLPSetCond(CMLPTrainer &s,double wstep,int maxits)
{
//--- check
if(!CAp::Assert(MathIsValidNumber(wstep),__FUNCTION__+": parameter WStep contains Infinite or NaN."))
return;
if(!CAp::Assert(wstep>=0.0,__FUNCTION__+": WStep<0."))
return;
if(!CAp::Assert(maxits>=0,__FUNCTION__+": MaxIts<0."))
return;
if(wstep!=0.0 || maxits!=0)
{
s.m_wstep=wstep;
s.m_maxits=maxits;
}
else
{
s.m_wstep=0.005;
s.m_maxits=0;
}
}
//+------------------------------------------------------------------+
//| This function sets training algorithm: batch training using |
//| L-BFGS will be used. |
//| This algorithm: |
//| * the most robust for small-scale problems, but may be too slow|
//| for large scale ones. |
//| * perfoms full pass through the dataset before performing step |
//| * uses conditions specified by MLPSetCond() for stopping |
//| * is default one used by trainer object |
//| INPUT PARAMETERS: |
//| S - trainer object |
//+------------------------------------------------------------------+
void CMLPTrain::MLPSetAlgoBatch(CMLPTrainer &s)
{
s.m_algokind=0;
}
//+------------------------------------------------------------------+
//| This function trains neural network passed to this function, |
//| using current dataset (one which was passed to MLPSetDataset() |
//| or MLPSetSparseDataset()) and current training settings. Training|
//| from NRestarts random starting positions is performed, best |
//| network is chosen. |
//| Training is performed using current training algorithm. |
//| INPUT PARAMETERS: |
//| S - trainer object |
//| Network - neural network. It must have same number of inputs |
//| and output/classes as was specified during creation|
//| of the trainer object. |
//| NRestarts- number of restarts, >=0: |
//| * NRestarts>0 means that specified number of random|
//| restarts are performed, best network is chosen |
//| after training |
//| * NRestarts=0 means that current state of the |
//| network is used for training. |
//| OUTPUT PARAMETERS: |
//| Network - trained network |
//| NOTE: when no dataset was specified with MLPSetDataset / |
//| SetSparseDataset(), network is filled by zero values. Same |
//| behavior for functions MLPStartTraining and |
//| MLPContinueTraining. |
//| NOTE: this method uses sum-of-squares error function for training|
//+------------------------------------------------------------------+
void CMLPTrain::MLPTrainNetwork(CMLPTrainer &s,CMultilayerPerceptron &network,
int nrestarts,CMLPReport &rep)
{
//--- create variables
int nin=0;
int nout=0;
int wcount=0;
int ntype=0;
int ttype=0;
//--- check
if(!CAp::Assert(s.m_npoints>=0,__FUNCTION__+": parameter S is not initialized or is spoiled(S.NPoints<0)"))
return;
if(!CMLPBase::MLPIsSoftMax(network))
ntype=0;
else
ntype=1;
if(s.m_rcpar)
ttype=0;
else
ttype=1;
//--- check
if(!CAp::Assert(ntype==ttype,__FUNCTION__+": type of input network is not similar to network type in trainer object"))
return;
CMLPBase::MLPProperties(network,nin,nout,wcount);
//--- check
if(!CAp::Assert(s.m_nin==nin,__FUNCTION__+": number of inputs in trainer is not equal to number of inputs in network"))
return;
if(!CAp::Assert(s.m_nout==nout,__FUNCTION__+": number of outputs in trainer is not equal to number of outputs in network"))
return;
if(!CAp::Assert(nrestarts>=0,__FUNCTION__+": NRestarts<0."))
return;
//--- Train
MLPTrainNetworkX(s,nrestarts,-1,s.m_subset,-1,s.m_subset,0,network,rep);
}
//+------------------------------------------------------------------+
//| IMPORTANT: this is an "expert" version of the MLPTrain() function|
//| We do not recommend you to use it unless you are |
//| pretty sure that you need ability to monitor training |
//| progress. |
//| This function performs step-by-step training of the neural |
//| network. Here "step-by-step" means that training starts with |
//| MLPStartTraining() call, and then user subsequently calls |
//| MLPContinueTraining() to perform one more iteration of the |
//| training. |
//| After call to this function trainer object remembers network and |
//| is ready to train it. However, no training is performed until |
//| first call to MLPContinueTraining() function. Subsequent calls |
//| to MLPContinueTraining() will advance training progress one |
//| iteration further. |
//| EXAMPLE: |
//| > |
//| > ...initialize network and trainer object.... |
//| > |
//| > MLPStartTraining(Trainer, Network, True) |
//| > while MLPContinueTraining(Trainer, Network) do |
//| > ...visualize training progress... |
//| > |
//| INPUT PARAMETERS: |
//| S - trainer object |
//| Network - neural network. It must have same number of inputs |
//| and output/classes as was specified during creation|
//| of the trainer object. |
//| RandomStart - randomize network before training or not: |
//| * True means that network is randomized and its |
//| initial state (one which was passed to the |
//| trainer object) is lost. |
//| * False means that training is started from the |
//| current state of the network |
//| OUTPUT PARAMETERS: |
//| Network - neural network which is ready to training (weights |
//| are initialized, preprocessor is initialized using |
//| current training set) |
//| NOTE: this method uses sum-of-squares error function for training|
//| NOTE: it is expected that trainer object settings are NOT changed|
//| during step-by-step training, i.e. no one changes stopping |
//| criteria or training set during training. It is possible |
//| and there is no defense against such actions, but algorithm|
//| behavior in such cases is undefined and can be |
//| unpredictable. |
//+------------------------------------------------------------------+
void CMLPTrain::MLPStartTraining(CMLPTrainer &s,CMultilayerPerceptron &network,
bool randomstart)
{
//--- create variables
int nin=0;
int nout=0;
int wcount=0;
int ntype=0;
int ttype=0;
//--- check
if(!CAp::Assert(s.m_npoints>=0,__FUNCTION__+": parameter S is not initialized or is spoiled(S.NPoints<0)"))
return;
if(!CMLPBase::MLPIsSoftMax(network))
ntype=0;
else
ntype=1;
if(s.m_rcpar)
ttype=0;
else
ttype=1;
//--- check
if(!CAp::Assert(ntype==ttype,__FUNCTION__+": type of input network is not similar to network type in trainer object"))
return;
CMLPBase::MLPProperties(network,nin,nout,wcount);
//--- check
if(!CAp::Assert(s.m_nin==nin,__FUNCTION__+": number of inputs in trainer is not equal to number of inputs in the network."))
return;
if(!CAp::Assert(s.m_nout==nout,__FUNCTION__+": number of outputs in trainer is not equal to number of outputs in the network."))
return;
//--- Initialize temporaries
InitMLPTrnSession(network,randomstart,s,s.m_session);
//--- Train network
MLPStartTrainingX(s,randomstart,-1,s.m_subset,-1,s.m_session);
//--- Update network
CMLPBase::MLPCopyTunableParameters(s.m_session.m_network,network);
}
//+------------------------------------------------------------------+
//| IMPORTANT: this is an "expert" version of the MLPTrain() function|
//| We do not recommend you to use it unless you are |
//| pretty sure that you need ability to monitor training |
//| progress. |
//| This function performs step-by-step training of the neural |
//| network. Here "step-by-step" means that training starts with |
//| MLPStartTraining() call, and then user subsequently calls |
//| MLPContinueTraining() to perform one more iteration of the |
//| training. |
//| This function performs one more iteration of the training and |
//| returns either True (training continues) or False (training |
//| stopped). In case True was returned, Network weights are updated |
//| according to the current state of the optimization progress. |
//| In case False was returned, no additional updates is performed |
//| (previous update of the network weights moved us to the final |
//| point, and no additional updates is needed). |
//| EXAMPLE: |
//| > |
//| > [initialize network and trainer object] |
//| > |
//| > MLPStartTraining(Trainer, Network, True) |
//| > while MLPContinueTraining(Trainer, Network) do |
//| > [visualize training progress] |
//| > |
//| INPUT PARAMETERS: |
//| S - trainer object |
//| Network - neural network structure, which is used to store |
//| current state of the training process. |
//| OUTPUT PARAMETERS: |
//| Network - weights of the neural network are rewritten by the |
//| current approximation. |
//| NOTE: this method uses sum-of-squares error function for training|
//| NOTE: it is expected that trainer object settings are NOT changed|
//| during step-by-step training, i.e. no one changes stopping |
//| criteria or training set during training. It is possible |
//| and there is no defense against such actions, but algorithm|
//| behavior in such cases is undefined and can be |
//| unpredictable. |
//| NOTE: It is expected that Network is the same one which was |
//| passed to MLPStartTraining() function. However, THIS |
//| function checks only following: |
//| * that number of network inputs is consistent with |
//| trainer object settings |
//| * that number of network outputs / classes is consistent |
//| with trainer object settings |
//| * that number of network weights is the same as number of|
//| weights in the network passed to MLPStartTraining() |
//| function Exception is thrown when these conditions are |
//| violated. |
//| It is also expected that you do not change state of the network |
//| on your own - the only party who has right to change network |
//| during its training is a trainer object. Any attempt to interfere|
//| with trainer may lead to unpredictable results. |
//+------------------------------------------------------------------+
bool CMLPTrain::MLPContinueTraining(CMLPTrainer &s,
CMultilayerPerceptron &network)
{
//--- create variables
bool result;
int nin=0;
int nout=0;
int wcount=0;
int ntype=0;
int ttype=0;
int i_=0;
//--- check
if(!CAp::Assert(s.m_npoints>=0,__FUNCTION__+": parameter S is not initialized or is spoiled(S.NPoints<0)"))
return(false);
if(s.m_rcpar)
ttype=0;
else
ttype=1;
if(!CMLPBase::MLPIsSoftMax(network))
ntype=0;
else
ntype=1;
//--- check
if(!CAp::Assert(ntype==ttype,__FUNCTION__+": type of input network is not similar to network type in trainer object."))
return(false);
CMLPBase::MLPProperties(network,nin,nout,wcount);
//--- check
if(!CAp::Assert(s.m_nin==nin,__FUNCTION__+": number of inputs in trainer is not equal to number of inputs in the network."))
return(false);
if(!CAp::Assert(s.m_nout==nout,__FUNCTION__+": number of outputs in trainer is not equal to number of outputs in the network."))
return(false);
result=MLPContinueTrainingX(s,s.m_subset,-1,s.m_ngradbatch,s.m_session);
if(result)
network.m_weights=s.m_session.m_network.m_weights;
//--- return result
return(result);
}
//+------------------------------------------------------------------+
//| Internal cross-validation subroutine |
//+------------------------------------------------------------------+
void CMLPTrain::MLPKFoldCVGeneral(CMultilayerPerceptron &n,
CMatrixDouble &xy,const int npoints,
const double decay,const int restarts,
const int foldscount,const bool lmalgorithm,
const double wstep,const int maxits,
int &info,CMLPReport &rep,
CMLPCVReport &cvrep)
{
//--- create variables
int k=0;
int nin=0;
int nout=0;
int rowlen=0;
int wcount=0;
int nclasses=0;
int tssize=0;
int cvssize=0;
int relcnt=0;
//--- creating arrays
CRowInt folds;
CRowDouble x;
CRowDouble y;
//--- create matrix
CMatrixDouble cvset;
CMatrixDouble testset;
//--- creating arrays
CMultilayerPerceptron network;
CMLPReport internalrep;
//--- initialization
info=0;
//--- Read network geometry,test parameters
CMLPBase::MLPProperties(n,nin,nout,wcount);
//--- check
if(CMLPBase::MLPIsSoftMax(n))
{
nclasses=nout;
rowlen=nin+1;
}
else
{
nclasses=-nout;
rowlen=nin+nout;
}
//--- check
if(npoints<=0 || foldscount<2 || foldscount>npoints)
{
info=-1;
return;
}
//--- function call
CMLPBase::MLPCopy(n,network);
//--- K-fold out cross-validation.
//--- First,estimate generalization error
testset.Resize(npoints,rowlen);
cvset.Resize(npoints,rowlen);
x.Resize(nin);
y.Resize(nout);
//--- function call
MLPKFoldSplit(xy,npoints,nclasses,foldscount,false,folds);
//--- change values
cvrep.m_RelCLSError=0;
cvrep.m_AvgCE=0;
cvrep.m_RMSError=0;
cvrep.m_AvgError=0;
cvrep.m_AvgRelError=0;
rep.m_ngrad=0;
rep.m_nhess=0;
rep.m_ncholesky=0;
relcnt=0;
//--- calculation
for(int fold=0; fold<=foldscount-1; fold++)
{
//--- Separate set
tssize=0;
cvssize=0;
for(int i=0; i<npoints; i++)
{
//--- check
if(folds[i]==fold)
{
testset.Row(tssize,xy[i]+0);
tssize++;
}
else
{
cvset.Row(cvssize,xy[i]+0);
cvssize++;
}
}
//--- Train on CV training set
if(lmalgorithm)
MLPTrainLM(network,cvset,cvssize,decay,restarts,info,internalrep);
else
MLPTrainLBFGS(network,cvset,cvssize,decay,restarts,wstep,maxits,info,internalrep);
//--- check
if(info<0)
{
//--- change values
cvrep.m_RelCLSError=0;
cvrep.m_AvgCE=0;
cvrep.m_RMSError=0;
cvrep.m_AvgError=0;
cvrep.m_AvgRelError=0;
//--- exit the function
return;
}
//--- change values
rep.m_ngrad+=internalrep.m_ngrad;
rep.m_nhess+=internalrep.m_nhess;
rep.m_ncholesky+=internalrep.m_ncholesky;
//--- Estimate error using CV test set
if(CMLPBase::MLPIsSoftMax(network))
{
//--- classification-only code
cvrep.m_RelCLSError+=CMLPBase::MLPClsError(network,testset,tssize);
cvrep.m_AvgCE+=CMLPBase::MLPErrorN(network,testset,tssize);
}
//--- calculation
for(int i=0; i<=tssize-1; i++)
{
x=testset[i]+0;
x.Resize(nin);
//--- function call
CMLPBase::MLPProcess(network,x,y);
//--- check
if(CMLPBase::MLPIsSoftMax(network))
{
//--- Classification-specific code
k=(int)MathRound(testset.Get(i,nin));
for(int j=0; j<nout; j++)
{
//--- check
if(j==k)
{
//--- change values
cvrep.m_RMSError+=CMath::Sqr(y[j]-1);
cvrep.m_AvgError+=MathAbs(y[j]-1);
cvrep.m_AvgRelError+=MathAbs(y[j]-1);
relcnt++;
}
else
{
//--- change values
cvrep.m_RMSError+=CMath::Sqr(y[j]);
cvrep.m_AvgError+=MathAbs(y[j]);
}
}
}
else
{
//--- Regression-specific code
for(int j=0; j<nout; j++)
{
cvrep.m_RMSError+=CMath::Sqr(y[j]-testset.Get(i,nin+j));
cvrep.m_AvgError=cvrep.m_AvgError+MathAbs(y[j]-testset.Get(i,nin+j));
//--- check
if(testset.Get(i,nin+j)!=0.0)
{
cvrep.m_AvgRelError+=MathAbs((y[j]-testset.Get(i,nin+j))/testset.Get(i,nin+j));
relcnt++;
}
}
}
}
}
//--- check
if(CMLPBase::MLPIsSoftMax(network))
{
cvrep.m_RelCLSError/=npoints;
cvrep.m_AvgCE/=(MathLog(2)*npoints);
}
//--- change values
cvrep.m_RMSError=MathSqrt(cvrep.m_RMSError/(npoints*nout));
cvrep.m_AvgError/=(npoints*nout);
cvrep.m_AvgRelError/=relcnt;
info=1;
}
//+------------------------------------------------------------------+
//| Subroutine prepares K-fold split of the training set. |
//| NOTES: |
//| "NClasses>0" means that we have classification task. |
//| "NClasses<0" means regression task with -NClasses real |
//| outputs. |
//+------------------------------------------------------------------+
void CMLPTrain::MLPKFoldSplit(CMatrixDouble &xy,const int npoints,
const int nclasses,const int foldscount,
const bool stratifiedsplits,CRowInt &folds)
{
//--- create variables
int j=0;
CHighQualityRandState rs;
//--- test parameters
if(!CAp::Assert(npoints>0,__FUNCTION__+": wrong NPoints!"))
return;
//--- check
if(!CAp::Assert(nclasses>1 || nclasses<0,__FUNCTION__+": wrong NClasses!"))
return;
//--- check
if(!CAp::Assert(foldscount>=2 && foldscount<=npoints,__FUNCTION__+" wrong FoldsCount!"))
return;
//--- check
if(!CAp::Assert(!stratifiedsplits,__FUNCTION__+": stratified splits are not supported!"))
return;
//--- Folds
folds.Resize(npoints);
CHighQualityRand::HQRndRandomize(rs);
for(int i=0; i<npoints; i++)
folds.Set(i,i*foldscount/npoints);
//--- calculation
for(int i=0; i<npoints-1; i++)
{
j=i+CHighQualityRand::HQRndUniformI(rs,npoints-i);
//--- check
if(j!=i)
folds.Swap(i,j);
}
}
//+------------------------------------------------------------------+
//| This function trains neural network passed to this function, |
//| using current dataset (one which was passed to MLPSetDataset() or|
//| MLPSetSparseDataset()) and current training settings. Training |
//| from NRestarts random starting positions is performed, best |
//| network is chosen. |
//| This function is inteded to be used internally. It may be used in|
//| several settings: |
//| *training with ValSubsetSize=0, corresponds to "normal" |
//| training with termination criteria based on S.MaxIts |
//| (steps count) and S.WStep (step size). Training sample is |
//| given by TrnSubset/TrnSubsetSize. |
//| * training with ValSubsetSize>0, corresponds to early stopping |
//| training with additional MaxIts/WStep stopping criteria. |
//| Training sample is given by TrnSubset/TrnSubsetSize, |
//| validation sample is given by ValSubset/ ValSubsetSize. |
//+------------------------------------------------------------------+
void CMLPTrain::MLPTrainNetworkX(CMLPTrainer &s,
int nrestarts,
int algokind,
CRowInt &trnsubset,
int trnsubsetsize,
CRowInt &valsubset,
int valsubsetsize,
CMultilayerPerceptron &network,
CMLPReport &rep)
{
//--- create variables
CModelErrors modrep;
double eval=0;
double ebest=0;
int ngradbatch=0;
int nin=0;
int nout=0;
int wcount=0;
int pcount=0;
int itbest=0;
int itcnt=0;
int ntype=0;
int ttype=0;
bool rndstart;
int nr0=0;
int nr1=0;
double bestrmserror=0;
CSMLPTrnSession psession;
//--- function call
CMLPBase::MLPProperties(network,nin,nout,wcount);
//--- Process root call
//--- Check correctness of parameters
if(!CAp::Assert(algokind==0 || algokind==-1,__FUNCTION__+": unexpected AlgoKind"))
return;
if(!CAp::Assert(s.m_npoints>=0,__FUNCTION__+": internal error - parameter S is not initialized or is spoiled(S.NPoints<0)"))
return;
if(s.m_rcpar)
ttype=0;
else
ttype=1;
if(!CMLPBase::MLPIsSoftMax(network))
ntype=0;
else
ntype=1;
//--- check
if(!CAp::Assert(ntype==ttype,__FUNCTION__+": internal error - type of the training network is not similar to network type in trainer object"))
return;
if(!CAp::Assert(s.m_nin==nin,__FUNCTION__+": internal error - number of inputs in trainer is not equal to number of inputs in the training network."))
return;
if(!CAp::Assert(s.m_nout==nout,__FUNCTION__+": internal error - number of outputs in trainer is not equal to number of outputs in the training network."))
return;
if(!CAp::Assert(nrestarts>=0,__FUNCTION__+": internal error - NRestarts<0."))
return;
if(!CAp::Assert(trnsubset.Size()>=trnsubsetsize,__FUNCTION__+": internal error - parameter TrnSubsetSize more than input subset size(Length(TrnSubset)<TrnSubsetSize)"))
return;
if(!CAp::Assert(trnsubsetsize<=0 || (trnsubset.Min()>=0 && trnsubset.Max()<s.m_npoints),__FUNCTION__+": internal error - parameter TrnSubset contains incorrect index(TrnSubset[I]<0 or TrnSubset[I]>S.NPoints-1)"))
return;
if(!CAp::Assert(valsubset.Size()>=valsubsetsize,__FUNCTION__+": internal error - parameter ValSubsetSize more than input subset size(Length(ValSubset)<ValSubsetSize)"))
return;
if(!CAp::Assert(valsubsetsize<=0 || (valsubset.Min()>=0 && valsubset.Max()<s.m_npoints),__FUNCTION__+": internal error - parameter ValSubset contains incorrect index(ValSubset[I]<0 or ValSubset[I]>S.NPoints-1)"))
return;
//--- Train
//--- * NRestarts>=1 means that network is restarted from random position
//--- * NRestarts=0 means that network is not randomized
rndstart=nrestarts>0;
bestrmserror=CMath::m_maxrealnumber;
for(int iter=0; iter<=nrestarts; iter++)
{
rep.m_ngrad=0;
rep.m_nhess=0;
rep.m_ncholesky=0;
InitMLPTrnSession(network,rndstart,s,psession);
if((s.m_datatype==0 || s.m_datatype==1) && s.m_npoints>0 && trnsubsetsize!=0)
{
//--- Train network using combination of early stopping and step-size
//--- and step-count based criteria. Network state with best value of
//--- validation set error is stored in WBuf0. When validation set is
//--- zero, most recent state of network is stored.
ngradbatch=0;
eval=0;
ebest=0;
itbest=0;
itcnt=0;
MLPStartTrainingX(s,rndstart,algokind,trnsubset,trnsubsetsize,psession);
switch(s.m_datatype)
{
case 0:
ebest=CMLPBase::MLPErrorSubset(psession.m_network,s.m_densexy,s.m_npoints,valsubset,valsubsetsize);
break;
case 1:
ebest=CMLPBase::MLPErrorSparseSubset(psession.m_network,s.m_sparsexy,s.m_npoints,valsubset,valsubsetsize);
break;
}
psession.m_wbuf0=psession.m_network.m_weights;
while(MLPContinueTrainingX(s,trnsubset,trnsubsetsize,ngradbatch,psession))
{
switch(s.m_datatype)
{
case 0:
eval=CMLPBase::MLPErrorSubset(psession.m_network,s.m_densexy,s.m_npoints,valsubset,valsubsetsize);
break;
case 1:
eval=CMLPBase::MLPErrorSparseSubset(psession.m_network,s.m_sparsexy,s.m_npoints,valsubset,valsubsetsize);
break;
}
if(eval<=ebest || valsubsetsize==0)
{
psession.m_wbuf0=psession.m_network.m_weights;
ebest=eval;
itbest=itcnt;
}
if(itcnt>30 && (double)(itcnt)>(double)(1.5*itbest))
break;
itcnt++;
}
psession.m_network.m_weights=psession.m_wbuf0;
rep.m_ngrad=ngradbatch;
}
else
psession.m_network.m_weights.Fill(0);
//--- Evaluate network performance and update PSession.BestParameters/BestRMSError
//--- (if needed).
switch(s.m_datatype)
{
case 0:
CMLPBase::MLPAllErrorsSubset(psession.m_network,s.m_densexy,s.m_npoints,trnsubset,trnsubsetsize,modrep);
break;
case 1:
CMLPBase::MLPAllErrorsSparseSubset(psession.m_network,s.m_sparsexy,s.m_npoints,trnsubset,trnsubsetsize,modrep);
break;
}
if(modrep.m_RMSError<psession.m_bestrmserror)
{
CMLPBase::MLPExportTunableParameters(psession.m_network,psession.m_bestparameters,pcount);
psession.m_bestrmserror=modrep.m_RMSError;
}
//--- Choose best network
if(psession.m_bestrmserror<bestrmserror)
{
CMLPBase::MLPImportTunableParameters(network,psession.m_bestparameters);
bestrmserror=psession.m_bestrmserror;
}
}
//--- Calculate errors
switch(s.m_datatype)
{
case 0:
CMLPBase::MLPAllErrorsSubset(network,s.m_densexy,s.m_npoints,trnsubset,trnsubsetsize,modrep);
break;
case 1:
CMLPBase::MLPAllErrorsSparseSubset(network,s.m_sparsexy,s.m_npoints,trnsubset,trnsubsetsize,modrep);
break;
}
rep.m_RelCLSError=modrep.m_RelCLSError;
rep.m_AvgCE=modrep.m_AvgCE;
rep.m_RMSError=modrep.m_RMSError;
rep.m_AvgError=modrep.m_AvgError;
rep.m_AvgRelError=modrep.m_AvgRelError;
}
//+------------------------------------------------------------------+
//| This function trains neural network ensemble passed to this |
//| function using current dataset and early stopping training |
//| algorithm. Each early stopping round performs NRestarts random |
//| restarts (thus, EnsembleSize*NRestarts training rounds is |
//| performed in total). |
//+------------------------------------------------------------------+
void CMLPTrain::MLPTrainEnsembleX(CMLPTrainer &s,CMLPEnsemble &ensemble,
int idx0,
int idx1,
int nrestarts,
int trainingmethod,
int ngrad)
{
//--- create variables
int nin=CMLPBase::MLPGetInputsCount(ensemble.m_network);
int nout=CMLPBase::MLPGetOutputsCount(ensemble.m_network);
int wcount=CMLPBase::MLPGetWeightsCount(ensemble.m_network);
int pcount=0;
int trnsubsetsize=0;
int valsubsetsize=0;
int k0=0;
int i1_=0;
CSMLPTrnSession psession;
CHighQualityRandState rs;
if(CMLPBase::MLPIsSoftMax(ensemble.m_network))
pcount=nin;
else
pcount=nin+nout;
if(nrestarts<=0)
nrestarts=1;
//--- Handle degenerate case
if(s.m_npoints<2)
{
ensemble.m_weights.Fill(0.0);
ensemble.m_columnmeans.Fill(0.0);
ensemble.m_columnsigmas.Fill(1.0);
return;
}
//--- Prepare:
//--- * prepare MLPETrnSessions
//--- * fill ensemble by zeros (helps to detect errors)
ensemble.m_weights.Fill(0.0);
ensemble.m_columnmeans.Fill(0.0);
ensemble.m_columnsigmas.Fill(0.0);
//--- Train
CHighQualityRand::HQRndRandomize(rs);
for(int k=idx0; k<idx1; k++)
{
InitMLPTrnSession(ensemble.m_network,true,s,psession);
//--- Split set
trnsubsetsize=0;
valsubsetsize=0;
if(trainingmethod==0)
{
psession.m_trnsubset.Resize(s.m_npoints);
psession.m_valsubset.Resize(s.m_npoints);
do
{
trnsubsetsize=0;
valsubsetsize=0;
for(int i=0; i<s.m_npoints; i++)
{
if(CMath::RandomReal()<0.66)
{
//--- Assign sample to training set
psession.m_trnsubset.Set(trnsubsetsize,i);
trnsubsetsize++;
}
else
{
//--- Assign sample to validation set
psession.m_valsubset.Set(valsubsetsize,i);
valsubsetsize++;
}
}
}
while(!(trnsubsetsize!=0 && valsubsetsize!=0));
psession.m_trnsubset.Resize(trnsubsetsize);
psession.m_valsubset.Resize(valsubsetsize);
}
if(trainingmethod==1)
{
valsubsetsize=0;
trnsubsetsize=s.m_npoints;
psession.m_trnsubset.Resize(trnsubsetsize);
psession.m_valsubset.Resize(valsubsetsize);
for(int i=0; i<s.m_npoints; i++)
psession.m_trnsubset.Set(i,CHighQualityRand::HQRndUniformI(rs,s.m_npoints));
}
//--- Train
MLPTrainNetworkX(s,nrestarts,-1,psession.m_trnsubset,trnsubsetsize,psession.m_valsubset,valsubsetsize,psession.m_network,psession.m_mlprep);
ngrad+=psession.m_mlprep.m_ngrad;
//--- Save results
i1_=-(k*wcount);
for(int i_=k*wcount; i_<(k+1)*wcount; i_++)
ensemble.m_weights.Set(i_,psession.m_network.m_weights[i_+i1_]);
i1_=-(k*pcount);
for(int i_=k*pcount; i_<(k+1)*pcount ; i_++)
{
ensemble.m_columnmeans.Set(i_,psession.m_network.m_columnmeans[i_+i1_]);
ensemble.m_columnsigmas.Set(i_,psession.m_network.m_columnsigmas[i_+i1_]);
}
}
}
//+------------------------------------------------------------------+
//| This function performs step-by-step training of the neural |
//| network. Here "step-by-step" means that training starts with |
//| MLPStartTrainingX call, and then user subsequently calls |
//| MLPContinueTrainingX to perform one more iteration of the |
//| training. |
//| After call to this function trainer object remembers network and |
//| is ready to train it. However, no training is performed until |
//| first call to MLPContinueTraining() function. Subsequent calls |
//| to MLPContinueTraining() will advance traing progress one |
//| iteration further. |
//+------------------------------------------------------------------+
void CMLPTrain::MLPStartTrainingX(CMLPTrainer &s,bool randomstart,
int algokind,CRowInt &subset,
int subsetsize,
CSMLPTrnSession &session)
{
//--- create variables
int nin=0;
int nout=0;
int wcount=0;
int ntype=0;
int ttype=0;
//--- Check parameters
if(!CAp::Assert(s.m_npoints>=0,__FUNCTION__+": internal error - parameter S is not initialized or is spoiled(S.NPoints<0)"))
return;
if(!CAp::Assert(algokind==0 || algokind==-1,__FUNCTION__+": unexpected AlgoKind"))
return;
if(s.m_rcpar)
ttype=0;
else
ttype=1;
if(!CMLPBase::MLPIsSoftMax(session.m_network))
ntype=0;
else
ntype=1;
if(!CAp::Assert(ntype==ttype,__FUNCTION__+": internal error - type of the resulting network is not similar to network type in trainer object"))
return;
CMLPBase::MLPProperties(session.m_network,nin,nout,wcount);
if(!CAp::Assert(s.m_nin==nin,__FUNCTION__+": number of inputs in trainer is not equal to number of inputs in the network."))
return;
if(!CAp::Assert(s.m_nout==nout,__FUNCTION__+": number of outputs in trainer is not equal to number of outputs in the network."))
return;
if(!CAp::Assert(subset.Size()>=subsetsize,__FUNCTION__+": internal error - parameter SubsetSize more than input subset size(Length(Subset)<SubsetSize)"))
return;
if(!CAp::Assert(subsetsize<=0 || (subset.Min()>=0 && subset.Max()<s.m_npoints),__FUNCTION__+": internal error - parameter Subset contains incorrect index(Subset[I]<0 or Subset[I]>S.NPoints-1)"))
return;
//--- Prepare session
CMinLBFGS::MinLBFGSSetCond(session.m_optimizer,0.0,0.0,s.m_wstep,s.m_maxits);
if(s.m_npoints>0 && subsetsize!=0)
{
if(randomstart)
CMLPBase::MLPRandomize(session.m_network);
CMinLBFGS::MinLBFGSRestartFrom(session.m_optimizer,session.m_network.m_weights);
}
else
session.m_network.m_weights.Fill(0);
//---
if(algokind==-1)
{
session.m_algoused=s.m_algokind;
if(s.m_algokind==1)
session.m_minibatchsize=s.m_minibatchsize;
}
else
session.m_algoused=0;
CHighQualityRand::HQRndRandomize(session.m_generator);
session.m_rstate.ia.Resize(16);
session.m_rstate.ra.Resize(2);
session.m_rstate.stage=-1;
}
//+------------------------------------------------------------------+
//| This function performs step-by-step training of the neural |
//| network. Here "step-by-step" means that training starts with |
//| MLPStartTrainingX call, and then user subsequently calls |
//| MLPContinueTrainingX to perform one more iteration of the |
//| training. |
//| This function performs one more iteration of the training and |
//| returns either True (training continues) or False (training |
//| stopped). In case True was returned, Network weights are updated |
//| according to the current state of the optimization progress. In |
//| case False was returned, no additional updates is performed |
//| (previous update of the network weights moved us to the final |
//| point, and no additional updates is needed). |
//| EXAMPLE: |
//| > |
//| > [initialize network and trainer object] |
//| > |
//| > MLPStartTraining(Trainer, Network, True) |
//| > while MLPContinueTraining(Trainer, Network) do |
//| > [visualize training progress] |
//| > |
//+------------------------------------------------------------------+
bool CMLPTrain::MLPContinueTrainingX(CMLPTrainer &s,CRowInt &subset,
int subsetsize,
int &ngradbatch,
CSMLPTrnSession &session)
{
//--- create variables
int nin=0;
int nout=0;
int wcount=0;
int twcount=0;
int ntype=0;
int ttype=0;
double decay=0;
double v=0;
int i=0;
int j=0;
int k=0;
int trnsetsize=0;
int epoch=0;
int minibatchcount=0;
int minibatchidx=0;
int cursize=0;
int idx0=0;
int idx1=0;
//--- Reverse communication preparations
//--- I know it looks ugly, but it works the same way
//--- anywhere from C++ to Python.
//--- This code initializes locals by:
//--- * random values determined during code
//--- generation - on first subroutine call
//--- * values from previous call - on subsequent calls
if(session.m_rstate.stage>=0)
{
nin=session.m_rstate.ia[0];
nout=session.m_rstate.ia[1];
wcount=session.m_rstate.ia[2];
twcount=session.m_rstate.ia[3];
ntype=session.m_rstate.ia[4];
ttype=session.m_rstate.ia[5];
i=session.m_rstate.ia[6];
j=session.m_rstate.ia[7];
k=session.m_rstate.ia[8];
trnsetsize=session.m_rstate.ia[9];
epoch=session.m_rstate.ia[10];
minibatchcount=session.m_rstate.ia[11];
minibatchidx=session.m_rstate.ia[12];
cursize=session.m_rstate.ia[13];
idx0=session.m_rstate.ia[14];
idx1=session.m_rstate.ia[15];
decay=session.m_rstate.ra[0];
v=session.m_rstate.ra[1];
}
else
{
nin=359;
nout=-58;
wcount=-919;
twcount=-909;
ntype=81;
ttype=255;
i=74;
j=-788;
k=809;
trnsetsize=205;
epoch=-838;
minibatchcount=939;
minibatchidx=-526;
cursize=763;
idx0=-541;
idx1=-698;
decay=-900;
v=-318;
}
if(session.m_rstate.stage==0)
{
if(!MLPContinueTrainingX_lbl3(s,subset,subsetsize,ngradbatch,session,decay))
return(false);
}
else
{
//--- Routine body
//--- Check correctness of inputs
if(!CAp::Assert(s.m_npoints>=0,__FUNCTION__+": internal error - parameter S is not initialized or is spoiled(S.NPoints<0)."))
return(false);
if(s.m_rcpar)
ttype=0;
else
ttype=1;
if(!CMLPBase::MLPIsSoftMax(session.m_network))
ntype=0;
else
ntype=1;
//--- check
if(!CAp::Assert(ntype==ttype,__FUNCTION__+": internal error - type of the resulting network is not similar to network type in trainer object."))
return(false);
CMLPBase::MLPProperties(session.m_network,nin,nout,wcount);
//--- check
if(!CAp::Assert(s.m_nin==nin,__FUNCTION__+": internal error - number of inputs in trainer is not equal to number of inputs in the network."))
return(false);
if(!CAp::Assert(s.m_nout==nout,__FUNCTION__+": internal error - number of outputs in trainer is not equal to number of outputs in the network."))
return(false);
if(!CAp::Assert(subset.Size()>=subsetsize,__FUNCTION__+": internal error - parameter SubsetSize more than input subset size(Length(Subset)<SubsetSize)."))
return(false);
if(!CAp::Assert(subsetsize<=0 || (subset.Min()>=0 && subset.Max()<s.m_npoints),__FUNCTION__+": internal error - parameter Subset contains incorrect index(Subset[I]<0 or Subset[I]>S.NPoints-1)."))
return(false);
//--- Quick exit on empty training set
if(s.m_npoints==0 || subsetsize==0)
return(false);
//--- Minibatch training
if(!CAp::Assert(!(session.m_algoused==1),"MINIBATCH TRAINING IS NOT IMPLEMENTED YET"))
return(false);
//--- Last option: full batch training
decay=s.m_decay;
//---
if(!MLPContinueTrainingX_lbl1(s,subset,subsetsize,ngradbatch,session,decay))
return(false);
}
//--- Saving state
session.m_rstate.ia.Set(0,nin);
session.m_rstate.ia.Set(1,nout);
session.m_rstate.ia.Set(2,wcount);
session.m_rstate.ia.Set(3,twcount);
session.m_rstate.ia.Set(4,ntype);
session.m_rstate.ia.Set(5,ttype);
session.m_rstate.ia.Set(6,i);
session.m_rstate.ia.Set(7,j);
session.m_rstate.ia.Set(8,k);
session.m_rstate.ia.Set(9,trnsetsize);
session.m_rstate.ia.Set(10,epoch);
session.m_rstate.ia.Set(11,minibatchcount);
session.m_rstate.ia.Set(12,minibatchidx);
session.m_rstate.ia.Set(13,cursize);
session.m_rstate.ia.Set(14,idx0);
session.m_rstate.ia.Set(15,idx1);
session.m_rstate.ra.Set(0,decay);
session.m_rstate.ra.Set(1,v);
//--- return result
return(true);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CMLPTrain::MLPContinueTrainingX_lbl1(CMLPTrainer &s,CRowInt &subset,
int subsetsize,
int &ngradbatch,
CSMLPTrnSession &session,double &decay)
{
if(!CMinLBFGS::MinLBFGSIteration(session.m_optimizer))
{
CMinLBFGS::MinLBFGSResultsBuf(session.m_optimizer,session.m_network.m_weights,session.m_optimizerrep);
return(false);
}
if(!session.m_optimizer.m_xupdated)
{
if(!MLPContinueTrainingX_lbl3(s,subset,subsetsize,ngradbatch,session,decay))
return(false);
}
session.m_network.m_weights=session.m_optimizer.m_x;
session.m_rstate.stage=0;
//--- return result
return(true);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CMLPTrain::MLPContinueTrainingX_lbl3(CMLPTrainer &s,CRowInt &subset,
int subsetsize,
int &ngradbatch,
CSMLPTrnSession &session,double &decay)
{
session.m_network.m_weights=session.m_optimizer.m_x;
switch(s.m_datatype)
{
case 0:
CMLPBase::MLPGradBatchSubset(session.m_network,s.m_densexy,s.m_npoints,subset,subsetsize,session.m_optimizer.m_f,session.m_optimizer.m_g);
break;
case 1:
CMLPBase::MLPGradBatchSparseSubset(session.m_network,s.m_sparsexy,s.m_npoints,subset,subsetsize,session.m_optimizer.m_f,session.m_optimizer.m_g);
break;
}
//--- Increment number of operations performed on batch gradient
ngradbatch++;
double v=session.m_network.m_weights.Dot(session.m_network.m_weights);
session.m_optimizer.m_f+=0.5*decay*v;
session.m_optimizer.m_g+=session.m_network.m_weights*decay+0;
//--- return result
return(MLPContinueTrainingX_lbl1(s,subset,subsetsize,ngradbatch,session,decay));
}
//+------------------------------------------------------------------+
//| Internal bagging subroutine. |
//+------------------------------------------------------------------+
void CMLPTrain::MLPEBaggingInternal(CMLPEnsemble &ensemble,
CMatrixDouble &xy,
int npoints,
double decay,
int restarts,
double wstep,
int maxits,
bool lmalgorithm,
int &info,
CMLPReport &rep,
CMLPCVReport &ooberrors)
{
//--- create variables
CMatrixDouble xys;
CMatrixDouble oobbuf;
CRowInt oobcntbuf;
CRowDouble x;
CRowDouble y;
CRowDouble dy;
CRowDouble dsbuf;
bool s[];
int ccnt=0;
int pcnt=0;
int j=0;
double v=0;
int nin=0;
int nout=0;
int wcount=0;
int i1_=0;
CMLPReport tmprep;
CHighQualityRandState rs;
info=0;
CMLPBase::MLPProperties(ensemble.m_network,nin,nout,wcount);
//--- Test for inputs
if(!lmalgorithm && wstep==0.0 && maxits==0)
{
info=-8;
return;
}
if(npoints<=0 || restarts<1 || wstep<0.0 || maxits<0)
{
info=-1;
return;
}
if(CMLPBase::MLPIsSoftMax(ensemble.m_network))
{
vector<double> temp=xy.Col(nin);
if((int)MathRound(temp.Min())<0 || (int)MathRound(temp.Max())>=nout)
{
info=-2;
return;
}
}
//--- allocate temporaries
info=2;
rep.m_ngrad=0;
rep.m_nhess=0;
rep.m_ncholesky=0;
ooberrors.m_RelCLSError=0;
ooberrors.m_AvgCE=0;
ooberrors.m_RMSError=0;
ooberrors.m_AvgError=0;
ooberrors.m_AvgRelError=0;
if(CMLPBase::MLPIsSoftMax(ensemble.m_network))
{
ccnt=nin+1;
pcnt=nin;
}
else
{
ccnt=nin+nout;
pcnt=nin+nout;
}
xys.Resize(npoints,ccnt);
ArrayResize(s,npoints);
oobbuf=matrix<double>::Zeros(npoints,nout);
oobcntbuf.Resize(npoints);
oobcntbuf.Fill(0);
x.Resize(nin);
y.Resize(nout);
if(CMLPBase::MLPIsSoftMax(ensemble.m_network))
dy.Resize(1);
else
dy.Resize(nout);
//--- main bagging cycle
CHighQualityRand::HQRndRandomize(rs);
for(int k=0; k<ensemble.m_ensemblesize; k++)
{
//--- prepare dataset
ArrayInitialize(s,false);
for(int i=0; i<npoints; i++)
{
j=CHighQualityRand::HQRndUniformI(rs,npoints);
s[j]=true;
xys.Row(i,xy[j]+0);
}
//--- train
if(lmalgorithm)
MLPTrainLM(ensemble.m_network,xys,npoints,decay,restarts,info,tmprep);
else
MLPTrainLBFGS(ensemble.m_network,xys,npoints,decay,restarts,wstep,maxits,info,tmprep);
if(info<0)
return;
//--- save results
rep.m_ngrad+=tmprep.m_ngrad;
rep.m_nhess+=tmprep.m_nhess;
rep.m_ncholesky+=tmprep.m_ncholesky;
i1_= -(k*wcount);
for(int i_=k*wcount; i_<(k+1)*wcount; i_++)
ensemble.m_weights.Set(i_,ensemble.m_network.m_weights[i_+i1_]);
i1_=-(k*pcnt);
for(int i_=k*pcnt; i_<(k+1)*pcnt; i_++)
{
ensemble.m_columnmeans.Set(i_,ensemble.m_network.m_columnmeans[i_+i1_]);
ensemble.m_columnsigmas.Set(i_,ensemble.m_network.m_columnsigmas[i_+i1_]);
}
//--- OOB estimates
for(int i=0; i<npoints; i++)
{
if(!s[i])
{
x=xy[i]+0;
CMLPBase::MLPProcess(ensemble.m_network,x,y);
oobbuf.Row(i,y.ToVector()+oobbuf[i]);
oobcntbuf.Add(i,1);
}
}
}
//--- OOB estimates
if(CMLPBase::MLPIsSoftMax(ensemble.m_network))
CBdSS::DSErrAllocate(nout,dsbuf);
else
CBdSS::DSErrAllocate(-nout,dsbuf);
for(int i=0; i<npoints; i++)
{
if(oobcntbuf[i]!=0)
{
v=1.0/(double)oobcntbuf[i];
y=oobbuf[i]*v;
if(CMLPBase::MLPIsSoftMax(ensemble.m_network))
dy.Set(0,xy.Get(i,nin));
else
{
i1_=(nin);
for(int i_=0; i_<nout; i_++)
dy.Set(i_,v*xy.Get(i,i_+i1_));
}
CBdSS::DSErrAccumulate(dsbuf,y,dy);
}
}
CBdSS::DSErrFinish(dsbuf);
ooberrors.m_RelCLSError=dsbuf[0];
ooberrors.m_AvgCE=dsbuf[1];
ooberrors.m_RMSError=dsbuf[2];
ooberrors.m_AvgError=dsbuf[3];
ooberrors.m_AvgRelError=dsbuf[4];
}
//+------------------------------------------------------------------+
//| This function initializes temporaries needed for training |
//| session. |
//+------------------------------------------------------------------+
void CMLPTrain::InitMLPTrnSession(CMultilayerPerceptron &networktrained,
bool randomizenetwork,
CMLPTrainer &trainer,
CSMLPTrnSession &session)
{
//--- create variables
int nin=0;
int nout=0;
int wcount=0;
int pcount=0;
CRowInt dummysubset;
//--- Prepare network:
//--- * copy input network to Session.Network
//--- * re-initialize preprocessor and weights if RandomizeNetwork=True
CMLPBase::MLPCopy(networktrained,session.m_network);
if(randomizenetwork)
{
//--- check
if(!CAp::Assert(trainer.m_datatype==0 || trainer.m_datatype==1,__FUNCTION__+": unexpected Trainer.DataType"))
return;
switch(trainer.m_datatype)
{
case 0:
CMLPBase::MLPInitPreprocessorSubset(session.m_network,trainer.m_densexy,trainer.m_npoints,dummysubset,-1);
break;
case 1:
CMLPBase::MLPInitPreprocessorSparseSubset(session.m_network,trainer.m_sparsexy,trainer.m_npoints,dummysubset,-1);
break;
}
CMLPBase::MLPRandomize(session.m_network);
session.m_randomizenetwork=true;
}
else
session.m_randomizenetwork=false;
//--- Determine network geometry and initialize optimizer
CMLPBase::MLPProperties(session.m_network,nin,nout,wcount);
CMinLBFGS::MinLBFGSCreate(wcount,MathMin(wcount,trainer.m_lbfgsfactor),session.m_network.m_weights,session.m_optimizer);
CMinLBFGS::MinLBFGSSetXRep(session.m_optimizer,true);
//--- Create buffers
session.m_wbuf0.Resize(wcount);
session.m_wbuf1.Resize(wcount);
//--- Initialize session result
CMLPBase::MLPExportTunableParameters(session.m_network,session.m_bestparameters,pcount);
session.m_bestrmserror=CMath::m_maxrealnumber;
}
//+------------------------------------------------------------------+
//| Neural networks ensemble |
//+------------------------------------------------------------------+
class CMLPE
{
public:
//--- class constants
static const int m_mlpefirstversion;
//--- public methods
static void MLPECreate0(const int nin,const int nout,const int ensemblesize,CMLPEnsemble &ensemble);
static void MLPECreate1(const int nin,const int nhid,const int nout,const int ensemblesize,CMLPEnsemble &ensemble);
static void MLPECreate2(const int nin,const int nhid1,const int nhid2,const int nout,const int ensemblesize,CMLPEnsemble &ensemble);
static void MLPECreateB0(const int nin,const int nout,const double b,const double d,const int ensemblesize,CMLPEnsemble &ensemble);
static void MLPECreateB1(const int nin,const int nhid,const int nout,const double b,const double d,const int ensemblesize,CMLPEnsemble &ensemble);
static void MLPECreateB2(const int nin,const int nhid1,const int nhid2,const int nout,const double b,const double d,const int ensemblesize,CMLPEnsemble &ensemble);
static void MLPECreateR0(const int nin,const int nout,const double a,const double b,const int ensemblesize,CMLPEnsemble &ensemble);
static void MLPECreateR1(const int nin,const int nhid,const int nout,const double a,const double b,const int ensemblesize,CMLPEnsemble &ensemble);
static void MLPECreateR2(const int nin,const int nhid1,const int nhid2,const int nout,const double a,const double b,const int ensemblesize,CMLPEnsemble &ensemble);
static void MLPECreateC0(const int nin,const int nout,const int ensemblesize,CMLPEnsemble &ensemble);
static void MLPECreateC1(const int nin,const int nhid,const int nout,const int ensemblesize,CMLPEnsemble &ensemble);
static void MLPECreateC2(const int nin,const int nhid1,const int nhid2,const int nout,const int ensemblesize,CMLPEnsemble &ensemble);
static void MLPECreateFromNetwork(CMultilayerPerceptron &network,const int ensemblesize,CMLPEnsemble &ensemble);
static void MLPECopy(CMLPEnsemble &ensemble1,CMLPEnsemble &ensemble2);
static void MLPEAlloc(CSerializer &s,CMLPEnsemble &ensemble);
static void MLPESerialize(CSerializer &s,CMLPEnsemble &ensemble);
static void MLPEUnserialize(CSerializer &s,CMLPEnsemble &ensemble);
static void MLPERandomize(CMLPEnsemble &ensemble);
static void MLPEProperties(CMLPEnsemble &ensemble,int &nin,int &nout);
static bool MLPEIsSoftMax(CMLPEnsemble &ensemble);
static void MLPEProcess(CMLPEnsemble &ensemble,double &x[],double &y[]);
static void MLPEProcess(CMLPEnsemble &ensemble,CRowDouble &x,CRowDouble &y);
static void MLPEProcessI(CMLPEnsemble &ensemble,double &x[],double &y[]);
static void MLPEProcessI(CMLPEnsemble &ensemble,CRowDouble &x,CRowDouble &y);
static void MLPEAllErrorsX(CMLPEnsemble &ensemble,CMatrixDouble &densexy,CSparseMatrix &sparsexy,int datasetsize,int datasettype,CRowInt &idx,int subset0,int subset1,int subsettype,CModelErrors &rep);
static void MLPEAllerrorsSparse(CMLPEnsemble &ensemble,CSparseMatrix &xy,int npoints,double &relcls,double &avgce,double &rms,double &avg,double &avgrel);
static double MLPERelClsError(CMLPEnsemble &ensemble,CMatrixDouble &xy,const int npoints);
static double MLPEAvgCE(CMLPEnsemble &ensemble,CMatrixDouble &xy,const int npoints);
static double MLPERMSError(CMLPEnsemble &ensemble,CMatrixDouble &xy,const int npoints);
static double MLPEAvgError(CMLPEnsemble &ensemble,CMatrixDouble &xy,const int npoints);
static double MLPEAvgRelError(CMLPEnsemble &ensemble,CMatrixDouble &xy,const int npoints);
private:
static void MLPEAllErrors(CMLPEnsemble &ensemble,CMatrixDouble &xy,const int npoints,double &relcls,double &avgce,double &rms,double &avg,double &avgrel);
static void MLPEBaggingInternal(CMLPEnsemble &ensemble,CMatrixDouble &xy,const int npoints,const double decay,const int restarts,const double wstep,const int maxits,const bool lmalgorithm,int &info,CMLPReport &rep,CMLPCVReport &ooberrors);
};
//+------------------------------------------------------------------+
//| Initialize constants |
//+------------------------------------------------------------------+
const int CMLPE::m_mlpefirstversion=1;
//+------------------------------------------------------------------+
//| Like MLPCreate0, but for ensembles. |
//+------------------------------------------------------------------+
void CMLPE::MLPECreate0(const int nin,const int nout,const int ensemblesize,
CMLPEnsemble &ensemble)
{
//--- object of class
CMultilayerPerceptron net;
//--- function call
CMLPBase::MLPCreate0(nin,nout,net);
//--- function call
MLPECreateFromNetwork(net,ensemblesize,ensemble);
}
//+------------------------------------------------------------------+
//| Like MLPCreate1, but for ensembles. |
//+------------------------------------------------------------------+
void CMLPE::MLPECreate1(const int nin,const int nhid,const int nout,
const int ensemblesize,CMLPEnsemble &ensemble)
{
//--- object of class
CMultilayerPerceptron net;
//--- function call
CMLPBase::MLPCreate1(nin,nhid,nout,net);
//--- function call
MLPECreateFromNetwork(net,ensemblesize,ensemble);
}
//+------------------------------------------------------------------+
//| Like MLPCreate2, but for ensembles. |
//+------------------------------------------------------------------+
void CMLPE::MLPECreate2(const int nin,const int nhid1,const int nhid2,
const int nout,const int ensemblesize,
CMLPEnsemble &ensemble)
{
//--- object of class
CMultilayerPerceptron net;
//--- function call
CMLPBase::MLPCreate2(nin,nhid1,nhid2,nout,net);
//--- function call
MLPECreateFromNetwork(net,ensemblesize,ensemble);
}
//+------------------------------------------------------------------+
//| Like MLPCreateB0, but for ensembles. |
//+------------------------------------------------------------------+
void CMLPE::MLPECreateB0(const int nin,const int nout,const double b,
const double d,const int ensemblesize,
CMLPEnsemble &ensemble)
{
//--- object of class
CMultilayerPerceptron net;
//--- function call
CMLPBase::MLPCreateB0(nin,nout,b,d,net);
//--- function call
MLPECreateFromNetwork(net,ensemblesize,ensemble);
}
//+------------------------------------------------------------------+
//| Like MLPCreateB1, but for ensembles. |
//+------------------------------------------------------------------+
void CMLPE::MLPECreateB1(const int nin,const int nhid,const int nout,
const double b,const double d,const int ensemblesize,
CMLPEnsemble &ensemble)
{
//--- object of class
CMultilayerPerceptron net;
//--- function call
CMLPBase::MLPCreateB1(nin,nhid,nout,b,d,net);
//--- function call
MLPECreateFromNetwork(net,ensemblesize,ensemble);
}
//+------------------------------------------------------------------+
//| Like MLPCreateB2, but for ensembles. |
//+------------------------------------------------------------------+
void CMLPE::MLPECreateB2(const int nin,const int nhid1,const int nhid2,
const int nout,const double b,const double d,
const int ensemblesize,CMLPEnsemble &ensemble)
{
//--- object of class
CMultilayerPerceptron net;
//--- function call
CMLPBase::MLPCreateB2(nin,nhid1,nhid2,nout,b,d,net);
//--- function call
MLPECreateFromNetwork(net,ensemblesize,ensemble);
}
//+------------------------------------------------------------------+
//| Like MLPCreateR0, but for ensembles. |
//+------------------------------------------------------------------+
void CMLPE::MLPECreateR0(const int nin,const int nout,const double a,
const double b,const int ensemblesize,
CMLPEnsemble &ensemble)
{
//--- object of class
CMultilayerPerceptron net;
//--- function call
CMLPBase::MLPCreateR0(nin,nout,a,b,net);
//--- function call
MLPECreateFromNetwork(net,ensemblesize,ensemble);
}
//+------------------------------------------------------------------+
//| Like MLPCreateR1, but for ensembles. |
//+------------------------------------------------------------------+
void CMLPE::MLPECreateR1(const int nin,const int nhid,const int nout,
const double a,const double b,
const int ensemblesize,CMLPEnsemble &ensemble)
{
//--- object of class
CMultilayerPerceptron net;
//--- function call
CMLPBase::MLPCreateR1(nin,nhid,nout,a,b,net);
//--- function call
MLPECreateFromNetwork(net,ensemblesize,ensemble);
}
//+------------------------------------------------------------------+
//| Like MLPCreateR2, but for ensembles. |
//+------------------------------------------------------------------+
void CMLPE::MLPECreateR2(const int nin,const int nhid1,const int nhid2,
const int nout,const double a,const double b,
const int ensemblesize,CMLPEnsemble &ensemble)
{
//--- object of class
CMultilayerPerceptron net;
//--- function call
CMLPBase::MLPCreateR2(nin,nhid1,nhid2,nout,a,b,net);
//--- function call
MLPECreateFromNetwork(net,ensemblesize,ensemble);
}
//+------------------------------------------------------------------+
//| Like MLPCreateC0, but for ensembles. |
//+------------------------------------------------------------------+
void CMLPE::MLPECreateC0(const int nin,const int nout,const int ensemblesize,
CMLPEnsemble &ensemble)
{
//--- object of class
CMultilayerPerceptron net;
//--- function call
CMLPBase::MLPCreateC0(nin,nout,net);
//--- function call
MLPECreateFromNetwork(net,ensemblesize,ensemble);
}
//+------------------------------------------------------------------+
//| Like MLPCreateC1, but for ensembles. |
//+------------------------------------------------------------------+
void CMLPE::MLPECreateC1(const int nin,const int nhid,const int nout,
const int ensemblesize,CMLPEnsemble &ensemble)
{
//--- object of class
CMultilayerPerceptron net;
//--- function call
CMLPBase::MLPCreateC1(nin,nhid,nout,net);
//--- function call
MLPECreateFromNetwork(net,ensemblesize,ensemble);
}
//+------------------------------------------------------------------+
//| Like MLPCreateC2, but for ensembles. |
//+------------------------------------------------------------------+
void CMLPE::MLPECreateC2(const int nin,const int nhid1,const int nhid2,
const int nout,const int ensemblesize,
CMLPEnsemble &ensemble)
{
//--- object of class
CMultilayerPerceptron net;
//--- function call
CMLPBase::MLPCreateC2(nin,nhid1,nhid2,nout,net);
//--- function call
MLPECreateFromNetwork(net,ensemblesize,ensemble);
}
//+------------------------------------------------------------------+
//| Creates ensemble from network. Only network geometry is copied. |
//+------------------------------------------------------------------+
void CMLPE::MLPECreateFromNetwork(CMultilayerPerceptron &network,
const int ensemblesize,
CMLPEnsemble &ensemble)
{
//--- create variables
int ccount=0;
int i1_=0;
//--- check
if(!CAp::Assert(ensemblesize>0,__FUNCTION__+": incorrect ensemble size!"))
return;
//--- Copy network
CMLPBase::MLPCopy(network,ensemble.m_network);
//--- network properties
if(CMLPBase::MLPIsSoftMax(network))
ccount=CMLPBase::MLPGetInputsCount(ensemble.m_network);
else
ccount=CMLPBase::MLPGetInputsCount(ensemble.m_network)+CMLPBase::MLPGetOutputsCount(ensemble.m_network);
int wcount=CMLPBase::MLPGetWeightsCount(ensemble.m_network);
ensemble.m_ensemblesize=ensemblesize;
//--- weights, means, sigmas
ensemble.m_weights=vector<double>::Zeros(ensemblesize*wcount);
ensemble.m_columnmeans=vector<double>::Zeros(ensemblesize*ccount);
ensemble.m_columnsigmas=vector<double>::Zeros(ensemblesize*ccount);
for(int i=0; i<ensemblesize*wcount; i++)
ensemble.m_weights.Set(i,CMath::RandomReal()-0.5);
for(int i=0; i<ensemblesize; i++)
{
i1_=-(i*ccount);
for(int i_=i*ccount; i_<(i+1)*ccount; i_++)
{
ensemble.m_columnmeans.Set(i_,network.m_columnmeans[i_+i1_]);
ensemble.m_columnsigmas.Set(i_,network.m_columnsigmas[i_+i1_]);
}
}
//--- temporaries, internal buffers
ensemble.m_y.Resize(CMLPBase::MLPGetOutputsCount(ensemble.m_network));
}
//+------------------------------------------------------------------+
//| Copying of MLPEnsemble strucure |
//| INPUT PARAMETERS: |
//| Ensemble1 - original |
//| OUTPUT PARAMETERS: |
//| Ensemble2 - copy |
//+------------------------------------------------------------------+
void CMLPE::MLPECopy(CMLPEnsemble &ensemble1,CMLPEnsemble &ensemble2)
{
ensemble2=ensemble1;
}
//+------------------------------------------------------------------+
//| Serializer: allocation |
//+------------------------------------------------------------------+
void CMLPE::MLPEAlloc(CSerializer &s,CMLPEnsemble &ensemble)
{
s.Alloc_Entry();
s.Alloc_Entry();
s.Alloc_Entry();
CApServ::AllocRealArray(s,ensemble.m_weights,-1);
CApServ::AllocRealArray(s,ensemble.m_columnmeans,-1);
CApServ::AllocRealArray(s,ensemble.m_columnsigmas,-1);
CMLPBase::MLPAlloc(s,ensemble.m_network);
}
//+------------------------------------------------------------------+
//| Serializer: serialization |
//+------------------------------------------------------------------+
void CMLPE::MLPESerialize(CSerializer &s,CMLPEnsemble &ensemble)
{
s.Serialize_Int(CSCodes::GetMLPESerializationCode());
s.Serialize_Int(m_mlpefirstversion);
s.Serialize_Int(ensemble.m_ensemblesize);
CApServ::SerializeRealArray(s,ensemble.m_weights,-1);
CApServ::SerializeRealArray(s,ensemble.m_columnmeans,-1);
CApServ::SerializeRealArray(s,ensemble.m_columnsigmas,-1);
CMLPBase::MLPSerialize(s,ensemble.m_network);
}
//+------------------------------------------------------------------+
//| Serializer: unserialization |
//+------------------------------------------------------------------+
void CMLPE::MLPEUnserialize(CSerializer &s,CMLPEnsemble &ensemble)
{
//--- check correctness of header
int i0=s.Unserialize_Int();
if(!CAp::Assert(i0==CSCodes::GetMLPESerializationCode(),__FUNCTION__": stream header corrupted"))
return;
int i1=s.Unserialize_Int();
if(!CAp::Assert(i1==m_mlpefirstversion,__FUNCTION__": stream header corrupted"))
return;
//--- Create network
ensemble.m_ensemblesize=s.Unserialize_Int();
CApServ::UnserializeRealArray(s,ensemble.m_weights);
CApServ::UnserializeRealArray(s,ensemble.m_columnmeans);
CApServ::UnserializeRealArray(s,ensemble.m_columnsigmas);
CMLPBase::MLPUnserialize(s,ensemble.m_network);
//--- Allocate termoraries
ensemble.m_y.Resize(CMLPBase::MLPGetOutputsCount(ensemble.m_network));
}
//+------------------------------------------------------------------+
//| Randomization of MLP ensemble |
//+------------------------------------------------------------------+
void CMLPE::MLPERandomize(CMLPEnsemble &ensemble)
{
//--- calculation
int wcount=CMLPBase::MLPGetWeightsCount(ensemble.m_network);
for(int i=0; i<=ensemble.m_ensemblesize*wcount-1; i++)
ensemble.m_weights.Set(i,CMath::RandomReal()-0.5);
}
//+------------------------------------------------------------------+
//| Return ensemble properties (number of inputs and outputs). |
//+------------------------------------------------------------------+
void CMLPE::MLPEProperties(CMLPEnsemble &ensemble,int &nin,int &nout)
{
//--- change values
nin=CMLPBase::MLPGetInputsCount(ensemble.m_network);
nout=CMLPBase::MLPGetOutputsCount(ensemble.m_network);
}
//+------------------------------------------------------------------+
//| Return normalization type (whether ensemble is SOFTMAX-normalized|
//| or not). |
//+------------------------------------------------------------------+
bool CMLPE::MLPEIsSoftMax(CMLPEnsemble &ensemble)
{
return(CMLPBase::MLPIsSoftMax(ensemble.m_network));
}
//+------------------------------------------------------------------+
//| Procesing |
//| INPUT PARAMETERS: |
//| Ensemble- neural networks ensemble |
//| X - input vector, array[0..NIn-1]. |
//| Y - (possibly) preallocated buffer; if size of Y is |
//| less than NOut, it will be reallocated. If it is |
//| large enough, it is NOT reallocated, so we can |
//| save some time on reallocation. |
//| OUTPUT PARAMETERS: |
//| Y - result. Regression estimate when solving |
//| regression task, vector of posterior |
//| probabilities for classification task. |
//+------------------------------------------------------------------+
void CMLPE::MLPEProcess(CMLPEnsemble &ensemble,double &x[],double &y[])
{
CRowDouble X=x;
CRowDouble Y;
MLPEProcess(ensemble,X,Y);
Y.ToArray(y);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CMLPE::MLPEProcess(CMLPEnsemble &ensemble,CRowDouble &x,CRowDouble &y)
{
//--- create variables
int es=0;
int wc=0;
int cc=0;
double v=0;
int nout=0;
int i1_=0;
//--- initialization
es=ensemble.m_ensemblesize;
wc=CMLPBase::MLPGetWeightsCount(ensemble.m_network);
if(CMLPBase::MLPIsSoftMax(ensemble.m_network))
cc=CMLPBase::MLPGetInputsCount(ensemble.m_network);
else
cc=CMLPBase::MLPGetInputsCount(ensemble.m_network)+CMLPBase::MLPGetOutputsCount(ensemble.m_network);
//--- calculate
v=1.0/(double)es;
nout=CMLPBase::MLPGetOutputsCount(ensemble.m_network);
y=vector<double>::Zeros(nout);
for(int i=0; i<es; i++)
{
i1_=(i*wc);
for(int i_=0; i_<wc; i_++)
ensemble.m_network.m_weights.Set(i_,ensemble.m_weights[i_+i1_]);
i1_=(i*cc);
for(int i_=0; i_<cc; i_++)
ensemble.m_network.m_columnmeans.Set(i_,ensemble.m_columnmeans[i_+i1_]);
i1_=(i*cc);
for(int i_=0; i_<cc; i_++)
ensemble.m_network.m_columnsigmas.Set(i_,ensemble.m_columnsigmas[i_+i1_]);
CMLPBase::MLPProcess(ensemble.m_network,x,ensemble.m_y);
for(int i_=0; i_<nout; i_++)
y.Add(i_,v*ensemble.m_y[i_]);
}
}
//+------------------------------------------------------------------+
//| 'interactive' variant of MLPEProcess for languages like Python |
//| which support constructs like "Y = MLPEProcess(LM,X)" and |
//| interactive mode of the interpreter |
//| This function allocates new array on each call, so it is |
//| significantly slower than its 'non-interactive' counterpart, but |
//| it is more convenient when you call it from command line. |
//+------------------------------------------------------------------+
void CMLPE::MLPEProcessI(CMLPEnsemble &ensemble,double &x[],double &y[])
{
MLPEProcess(ensemble,x,y);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CMLPE::MLPEProcessI(CMLPEnsemble &ensemble,CRowDouble &x,CRowDouble &y)
{
MLPEProcess(ensemble,x,y);
}
//+------------------------------------------------------------------+
//| Calculation of all types of errors |
//+------------------------------------------------------------------+
void CMLPE::MLPEAllErrorsX(CMLPEnsemble &ensemble,
CMatrixDouble &densexy,
CSparseMatrix &sparsexy,
int datasetsize,
int datasettype,
CRowInt &idx,
int subset0,
int subset1,
int subsettype,
CModelErrors &rep)
{
//--- create variables
int nin=0;
int nout=0;
bool iscls;
int srcidx=0;
int i1_=0;
CMLPBuffers pbuf;
CModelErrors rep0;
CModelErrors rep1;
//--- Get network information
nin=CMLPBase::MLPGetInputsCount(ensemble.m_network);
nout=CMLPBase::MLPGetOutputsCount(ensemble.m_network);
iscls=CMLPBase::MLPIsSoftMax(ensemble.m_network);
//--- Retrieve buffer, prepare, process data, recycle buffer
if(iscls)
CBdSS::DSErrAllocate(nout,pbuf.m_Tmp0);
else
CBdSS::DSErrAllocate(-nout,pbuf.m_Tmp0);
CApServ::RVectorSetLengthAtLeast(pbuf.m_X,nin);
CApServ::RVectorSetLengthAtLeast(pbuf.m_Y,nout);
CApServ::RVectorSetLengthAtLeast(pbuf.m_Desiredy,nout);
for(int i=subset0; i<subset1; i++)
{
srcidx=-1;
if(subsettype==0)
srcidx=i;
if(subsettype==1)
srcidx=idx[i];
if(!CAp::Assert(srcidx>=0,__FUNCTION__": internal error"))
return;
switch(datasettype)
{
case 0:
pbuf.m_X=densexy[srcidx]+0;
break;
case 1:
CSparse::SparseGetRow(sparsexy,srcidx,pbuf.m_X);
break;
default:
CAp::Assert(false,__FUNCTION__": wrong Datase Type");
return;
}
pbuf.m_X.Resize(nin);
MLPEProcess(ensemble,pbuf.m_X,pbuf.m_Y);
if(CMLPBase::MLPIsSoftMax(ensemble.m_network))
{
if(datasettype==0)
pbuf.m_Desiredy.Set(0,densexy.Get(srcidx,nin));
else
pbuf.m_Desiredy.Set(0,CSparse::SparseGet(sparsexy,srcidx,nin));
}
else
{
if(datasettype==0)
{
i1_=nin;
for(int i_=0; i_<nout; i_++)
pbuf.m_Desiredy.Set(i_,densexy.Get(srcidx,i_+i1_));
}
else
{
for(int j=0; j<nout; j++)
pbuf.m_Desiredy.Set(j,CSparse::SparseGet(sparsexy,srcidx,nin+j));
}
}
CBdSS::DSErrAccumulate(pbuf.m_Tmp0,pbuf.m_Y,pbuf.m_Desiredy);
}
CBdSS::DSErrFinish(pbuf.m_Tmp0);
rep.m_RelCLSError=pbuf.m_Tmp0[0];
rep.m_AvgCE=pbuf.m_Tmp0[1]/MathLog(2);
rep.m_RMSError=pbuf.m_Tmp0[2];
rep.m_AvgError=pbuf.m_Tmp0[3];
rep.m_AvgRelError=pbuf.m_Tmp0[4];
}
//+------------------------------------------------------------------+
//| Calculation of all types of errors on dataset given by sparse |
//| matrix |
//+------------------------------------------------------------------+
void CMLPE::MLPEAllerrorsSparse(CMLPEnsemble &ensemble,
CSparseMatrix &xy,
int npoints,double &relcls,
double &avgce,double &rms,
double &avg,double &avgrel)
{
//--- create variables
int nin=CMLPBase::MLPGetInputsCount(ensemble.m_network);
int nout=CMLPBase::MLPGetOutputsCount(ensemble.m_network);
int i1_=0;
CRowDouble buf;
CRowDouble workx;
CRowDouble y;
CRowDouble dy;
relcls=0;
avgce=0;
rms=0;
avg=0;
avgrel=0;
if(CMLPBase::MLPIsSoftMax(ensemble.m_network))
{
dy.Resize(1);
CBdSS::DSErrAllocate(nout,buf);
}
else
{
dy.Resize(nout);
CBdSS::DSErrAllocate(-nout,buf);
}
for(int i=0; i<npoints; i++)
{
CSparse::SparseGetRow(xy,i,workx);
MLPEProcess(ensemble,workx,y);
if(CMLPBase::MLPIsSoftMax(ensemble.m_network))
dy.Set(0,workx[nin]);
else
{
i1_=nin;
for(int i_=0; i_<=nout-1; i_++)
dy.Set(i_,workx[i_+i1_]);
}
CBdSS::DSErrAccumulate(buf,y,dy);
}
CBdSS::DSErrFinish(buf);
relcls=buf[0];
avgce=buf[1];
rms=buf[2];
avg=buf[3];
avgrel=buf[4];
}
//+------------------------------------------------------------------+
//| Relative classification error on the test set |
//| INPUT PARAMETERS: |
//| Ensemble- ensemble |
//| XY - test set |
//| NPoints - test set size |
//| RESULT: |
//| percent of incorrectly classified cases. |
//| Works both for classifier betwork and for regression networks|
//| which are used as classifiers. |
//+------------------------------------------------------------------+
double CMLPE::MLPERelClsError(CMLPEnsemble &ensemble,CMatrixDouble &xy,
const int npoints)
{
//--- create variables
CModelErrors rep;
//--- function call
MLPEAllErrorsX(ensemble,xy,ensemble.m_network.m_dummysxy,npoints,0,ensemble.m_network.m_dummyidx,0,npoints,0,rep);
//--- return result
return(rep.m_RelCLSError);
}
//+------------------------------------------------------------------+
//| Average cross-entropy (in bits per element) on the test set |
//| INPUT PARAMETERS: |
//| Ensemble- ensemble |
//| XY - test set |
//| NPoints - test set size |
//| RESULT: |
//| CrossEntropy/(NPoints*LN(2)). |
//| Zero if ensemble solves regression task. |
//+------------------------------------------------------------------+
double CMLPE::MLPEAvgCE(CMLPEnsemble &ensemble,CMatrixDouble &xy,
const int npoints)
{
//--- create variables
CModelErrors rep;
//--- function call
MLPEAllErrorsX(ensemble,xy,ensemble.m_network.m_dummysxy,npoints,0,ensemble.m_network.m_dummyidx,0,npoints,0,rep);
//--- return result
return(rep.m_AvgCE);
}
//+------------------------------------------------------------------+
//| RMS error on the test set |
//| INPUT PARAMETERS: |
//| Ensemble- ensemble |
//| XY - test set |
//| NPoints - test set size |
//| RESULT: |
//| root mean square error. |
//| Its meaning for regression task is obvious. As for |
//| classification task RMS error means error when estimating |
//| posterior probabilities. |
//+------------------------------------------------------------------+
double CMLPE::MLPERMSError(CMLPEnsemble &ensemble,CMatrixDouble &xy,
const int npoints)
{
//--- create variables
CModelErrors rep;
//--- function call
MLPEAllErrorsX(ensemble,xy,ensemble.m_network.m_dummysxy,npoints,0,ensemble.m_network.m_dummyidx,0,npoints,0,rep);
//--- return result
return(rep.m_RMSError);
}
//+------------------------------------------------------------------+
//| Average error on the test set |
//| INPUT PARAMETERS: |
//| Ensemble- ensemble |
//| XY - test set |
//| NPoints - test set size |
//| RESULT: |
//| Its meaning for regression task is obvious. As for |
//| classification task it means average error when estimating |
//| posterior probabilities. |
//+------------------------------------------------------------------+
double CMLPE::MLPEAvgError(CMLPEnsemble &ensemble,CMatrixDouble &xy,
const int npoints)
{
//--- create variables
CModelErrors rep;
//--- function call
MLPEAllErrorsX(ensemble,xy,ensemble.m_network.m_dummysxy,npoints,0,ensemble.m_network.m_dummyidx,0,npoints,0,rep);
//--- return result
return(rep.m_AvgError);
}
//+------------------------------------------------------------------+
//| Average relative error on the test set |
//| INPUT PARAMETERS: |
//| Ensemble- ensemble |
//| XY - test set |
//| NPoints - test set size |
//| RESULT: |
//| Its meaning for regression task is obvious. As for |
//| classification task it means average relative error when |
//| estimating posterior probabilities. |
//+------------------------------------------------------------------+
double CMLPE::MLPEAvgRelError(CMLPEnsemble &ensemble,CMatrixDouble &xy,
const int npoints)
{
//--- create variables
CModelErrors rep;
//--- function call
MLPEAllErrorsX(ensemble,xy,ensemble.m_network.m_dummysxy,npoints,0,ensemble.m_network.m_dummyidx,0,npoints,0,rep);
//--- return result
return(rep.m_AvgRelError);
}
//+------------------------------------------------------------------+
//| Training neural networks ensemble using bootstrap aggregating |
//| (bagging). |
//| Modified Levenberg-Marquardt algorithm is used as base training |
//| method. |
//| INPUT PARAMETERS: |
//| Ensemble - model with initialized geometry |
//| XY - training set |
//| NPoints - training set size |
//| Decay - weight decay coefficient, >=0.001 |
//| Restarts - restarts, >0. |
//| OUTPUT PARAMETERS: |
//| Ensemble - trained model |
//| Info - return code: |
//| * -2, if there is a point with class number |
//| outside of [0..NClasses-1]. |
//| * -1, if incorrect parameters was passed |
//| (NPoints<0, Restarts<1). |
//| * 2, if task has been solved. |
//| Rep - training report. |
//| OOBErrors - out-of-bag generalization error estimate |
//+------------------------------------------------------------------+
void CMLPTrain::MLPEBaggingLM(CMLPEnsemble &ensemble,CMatrixDouble &xy,
const int npoints,const double decay,
const int restarts,int &info,CMLPReport &rep,
CMLPCVReport &ooberrors)
{
//--- initialization
info=0;
//--- function call
MLPEBaggingInternal(ensemble,xy,npoints,decay,restarts,0.0,0,true,info,rep,ooberrors);
}
//+------------------------------------------------------------------+
//| Training neural networks ensemble using bootstrap aggregating |
//| (bagging). L-BFGS algorithm is used as base training method. |
//| INPUT PARAMETERS: |
//| Ensemble - model with initialized geometry |
//| XY - training set |
//| NPoints - training set size |
//| Decay - weight decay coefficient, >=0.001 |
//| Restarts - restarts, >0. |
//| WStep - stopping criterion, same as in MLPTrainLBFGS |
//| MaxIts - stopping criterion, same as in MLPTrainLBFGS |
//| OUTPUT PARAMETERS: |
//| Ensemble - trained model |
//| Info - return code: |
//| * -8, if both WStep=0 and MaxIts=0 |
//| * -2, if there is a point with class number |
//| outside of [0..NClasses-1]. |
//| * -1, if incorrect parameters was passed |
//| (NPoints<0, Restarts<1). |
//| * 2, if task has been solved. |
//| Rep - training report. |
//| OOBErrors - out-of-bag generalization error estimate |
//+------------------------------------------------------------------+
void CMLPTrain::MLPEBaggingLBFGS(CMLPEnsemble &ensemble,CMatrixDouble &xy,
const int npoints,const double decay,
const int restarts,const double wstep,
const int maxits,int &info,CMLPReport &rep,
CMLPCVReport &ooberrors)
{
//--- initialization
info=0;
//--- function call
MLPEBaggingInternal(ensemble,xy,npoints,decay,restarts,wstep,maxits,false,info,rep,ooberrors);
}
//+------------------------------------------------------------------+
//| Training neural networks ensemble using early stopping. |
//| INPUT PARAMETERS: |
//| Ensemble - model with initialized geometry |
//| XY - training set |
//| NPoints - training set size |
//| Decay - weight decay coefficient, >=0.001 |
//| Restarts - restarts, >0. |
//| OUTPUT PARAMETERS: |
//| Ensemble - trained model |
//| Info - return code: |
//| * -2, if there is a point with class number |
//| outside of [0..NClasses-1]. |
//| * -1, if incorrect parameters was passed |
//| (NPoints<0, Restarts<1). |
//| * 6, if task has been solved. |
//| Rep - training report. |
//| OOBErrors - out-of-bag generalization error estimate |
//+------------------------------------------------------------------+
void CMLPTrain::MLPETrainES(CMLPEnsemble &ensemble,CMatrixDouble &xy,
const int npoints,const double decay,
const int restarts,int &info,CMLPReport &rep)
{
//--- create variables
int ccount=0;
int pcount=0;
int trnsize=0;
int valsize=0;
int tmpinfo=0;
int nin=0;
int nout=0;
int wcount=0;
int i1_=0;
//--- create matrix
CMatrixDouble trnxy;
CMatrixDouble valxy;
//--- objects of classes
CMLPReport tmprep;
CModelErrors moderr;
//--- initialization
info=0;
nin=CMLPBase::MLPGetInputsCount(ensemble.m_network);
nout=CMLPBase::MLPGetOutputsCount(ensemble.m_network);
wcount=CMLPBase::MLPGetWeightsCount(ensemble.m_network);
//--- check
if((npoints<2 || restarts<1) || decay<0.0)
{
info=-1;
return;
}
//--- check
if(CMLPBase::MLPIsSoftMax(ensemble.m_network))
{
vector<double> temp=xy.Col(nin);
temp.Resize(npoints);
//--- check
if((int)MathRound(temp.Min())<0 || (int)MathRound(temp.Max())>=nout)
{
info=-2;
return;
}
ccount=nin+1;
pcount=nin;
}
else
{
ccount=nin+nout;
pcount=nin+nout;
}
//--- change value
info=6;
//--- allocation
trnxy.Resize(npoints,ccount);
valxy.Resize(npoints,ccount);
//--- change values
rep.m_ngrad=0;
rep.m_nhess=0;
rep.m_ncholesky=0;
//--- train networks
for(int k=0; k<=ensemble.m_ensemblesize-1; k++)
{
//--- Split set
do
{
trnsize=0;
valsize=0;
for(int i=0; i<npoints; i++)
{
//--- check
if(CMath::RandomReal()<0.66)
{
//--- Assign sample to training set
trnxy.Row(trnsize,xy[i]+0);
trnsize++;
}
else
{
//--- Assign sample to validation set
valxy.Row(valsize,xy[i]+0);
valsize++;
}
}
}
while(!(trnsize!=0 && valsize!=0));
//--- Train
MLPTrainES(ensemble.m_network,trnxy,trnsize,valxy,valsize,decay,restarts,tmpinfo,tmprep);
//--- check
if(tmpinfo<0)
{
info=tmpinfo;
return;
}
//--- save results
i1_=-(k*wcount);
for(int i_=k*wcount; i_<(k+1)*wcount; i_++)
ensemble.m_weights.Set(i_,ensemble.m_network.m_weights[i_+i1_]);
i1_=-(k*pcount);
for(int i_=k*pcount; i_<(k+1)*pcount; i_++)
ensemble.m_columnmeans.Set(i_,ensemble.m_network.m_columnmeans[i_+i1_]);
i1_=-(k*pcount);
for(int i_=k*pcount; i_<(k+1)*pcount; i_++)
ensemble.m_columnsigmas.Set(i_,ensemble.m_network.m_columnsigmas[i_+i1_]);
//--- change values
rep.m_ngrad+=tmprep.m_ngrad;
rep.m_nhess+=tmprep.m_nhess;
rep.m_ncholesky+=tmprep.m_ncholesky;
}
CMLPE::MLPEAllErrorsX(ensemble,xy,ensemble.m_network.m_dummysxy,npoints,0,ensemble.m_network.m_dummyidx,0,npoints,0,moderr);
rep.m_RelCLSError=moderr.m_RelCLSError;
rep.m_AvgCE=moderr.m_AvgCE;
rep.m_RMSError=moderr.m_RMSError;
rep.m_AvgError=moderr.m_AvgError;
rep.m_AvgRelError=moderr.m_AvgRelError;
}
//+------------------------------------------------------------------+
//| This function trains neural network ensemble passed to this |
//| function using current dataset and early stopping training |
//| algorithm. Each early stopping round performs NRestarts random |
//| restarts (thus, EnsembleSize*NRestarts training rounds is |
//| performed in total). |
//| INPUT PARAMETERS: |
//| S - trainer object; |
//| Ensemble - neural network ensemble. It must have same |
//| number of inputs and outputs/classes as was |
//| specified during creation of the trainer object.|
//| NRestarts - number of restarts, >=0: |
//| * NRestarts>0 means that specified number of |
//| random restarts are performed during each ES |
//| round; |
//| * NRestarts=0 is silently replaced by 1. |
//| OUTPUT PARAMETERS: |
//| Ensemble - trained ensemble; |
//| Rep - it contains all type of errors. |
//| NOTE: this training method uses BOTH early stopping and weight |
//| decay! So, you should select weight decay before starting |
//| training just as you select it before training |
//| "conventional" networks. |
//| NOTE: when no dataset was specified with MLPSetDataset / |
//| SetSparseDataset(), or single-point dataset was passed, |
//| ensemble is filled by zero values. |
//| NOTE: this method uses sum-of-squares error function for training|
//+------------------------------------------------------------------+
void CMLPTrain::MLPTrainEnsembleES(CMLPTrainer &s,
CMLPEnsemble &ensemble,
int nrestarts,
CMLPReport &rep)
{
//--- create variables
int nin=0;
int nout=0;
int ntype=0;
int ttype=0;
int sgrad=0;
CModelErrors tmprep;
//--- check
if(!CAp::Assert(s.m_npoints>=0,__FUNCTION__+": parameter S is not initialized or is spoiled(S.NPoints<0)"))
return;
if(!CMLPE::MLPEIsSoftMax(ensemble))
ntype=0;
else
ntype=1;
if(s.m_rcpar)
ttype=0;
else
ttype=1;
//--- check
if(!CAp::Assert(ntype==ttype,__FUNCTION__+": internal error - type of input network is not similar to network type in trainer object"))
return;
nin=CMLPBase::MLPGetInputsCount(ensemble.m_network);
//--- check
if(!CAp::Assert(s.m_nin==nin,__FUNCTION__+": number of inputs in trainer is not equal to number of inputs in ensemble network"))
return;
nout=CMLPBase::MLPGetOutputsCount(ensemble.m_network);
//--- check
if(!CAp::Assert(s.m_nout==nout,__FUNCTION__+": number of outputs in trainer is not equal to number of outputs in ensemble network"))
return;
if(!CAp::Assert(nrestarts>=0,__FUNCTION__+": NRestarts<0."))
return;
//--- Initialize parameter Rep
rep.m_RelCLSError=0;
rep.m_AvgCE=0;
rep.m_RMSError=0;
rep.m_AvgError=0;
rep.m_AvgRelError=0;
rep.m_ngrad=0;
rep.m_nhess=0;
rep.m_ncholesky=0;
//--- Allocate
s.m_subset.Resize(s.m_npoints);
s.m_valsubset.Resize(s.m_npoints);
//--- Start training
//--- NOTE: ESessions is not initialized because MLPTrainEnsembleX
//--- needs uninitialized pool.
sgrad=0;
MLPTrainEnsembleX(s,ensemble,0,ensemble.m_ensemblesize,nrestarts,0,sgrad);
rep.m_ngrad=sgrad;
//--- Calculate errors.
CMLPE::MLPEAllErrorsX(ensemble,s.m_densexy,s.m_sparsexy,s.m_npoints,s.m_datatype,ensemble.m_network.m_dummyidx,0,s.m_npoints,0,tmprep);
rep.m_RelCLSError=tmprep.m_RelCLSError;
rep.m_AvgCE=tmprep.m_AvgCE;
rep.m_RMSError=tmprep.m_RMSError;
rep.m_AvgError=tmprep.m_AvgError;
rep.m_AvgRelError=tmprep.m_AvgRelError;
}
//+------------------------------------------------------------------+
//| Calculation of all types of errors |
//+------------------------------------------------------------------+
void CMLPE::MLPEAllErrors(CMLPEnsemble &ensemble,CMatrixDouble &xy,
const int npoints,double &relcls,
double &avgce,double &rms,
double &avg,double &avgrel)
{
//--- create variables
int i1_=0;
int nin=0;
int nout=0;
int wcount=0;
//--- creating arrays
CRowDouble buf;
CRowDouble workx;
CRowDouble y;
CRowDouble dy;
//--- initialization
relcls=0;
avgce=0;
rms=0;
avg=0;
avgrel=0;
//--- allocation
CMLPBase::MLPProperties(ensemble.m_network,nin,nout,wcount);
workx.Resize(nin);
y.Resize(nout);
//--- check
if(CMLPBase::MLPIsSoftMax(ensemble.m_network))
{
//--- allocation
dy.Resize(1);
//--- function call
CBdSS::DSErrAllocate(nout,buf);
}
else
{
//--- allocation
dy.Resize(nout);
//--- function call
CBdSS::DSErrAllocate(-nout,buf);
}
//--- calculation
for(int i=0; i<npoints; i++)
{
workx=xy[i]+0;
//--- function call
MLPEProcess(ensemble,workx,y);
//--- check
if(CMLPBase::MLPIsSoftMax(ensemble.m_network))
dy.Set(0,xy.Get(i,nin));
else
{
i1_=nin;
for(int i_=0; i_<nout; i_++)
dy.Set(i_,xy.Get(i,i_+i1_));
}
//--- function call
CBdSS::DSErrAccumulate(buf,y,dy);
}
//--- function call
CBdSS::DSErrFinish(buf);
//--- change values
relcls=buf[0];
avgce=buf[1];
rms=buf[2];
avg=buf[3];
avgrel=buf[4];
}
//+------------------------------------------------------------------+
//| Internal bagging subroutine. |
//+------------------------------------------------------------------+
void CMLPE::MLPEBaggingInternal(CMLPEnsemble &ensemble,CMatrixDouble &xy,
const int npoints,const double decay,
const int restarts,const double wstep,
const int maxits,const bool lmalgorithm,
int &info,CMLPReport &rep,CMLPCVReport &ooberrors)
{
//--- create variables
int nin=0;
int nout=0;
int wcount=0;
int ccnt=0;
int pcnt=0;
double v=0;
int i1_=0;
//--- creating arrays
bool s[];
int oobcntbuf[];
double x[];
double y[];
double dy[];
double dsbuf[];
//--- create matrix
CMatrixDouble xys;
CMatrixDouble oobbuf;
//--- objects of classes
CMLPReport tmprep;
CMultilayerPerceptron network;
//--- initialization
info=0;
//--- Test for inputs
if((!lmalgorithm && wstep==0.0) && maxits==0)
{
info=-8;
return;
}
//--- check
if(((npoints<=0 || restarts<1) || wstep<0.0) || maxits<0)
{
info=-1;
return;
}
//--- check
if(CMLPBase::MLPIsSoftMax(ensemble.m_network))
{
for(int i=0; i<npoints; i++)
{
//--- check
if((int)MathRound(xy.Get(i,nin))<0 || (int)MathRound(xy.Get(i,nin))>=nout)
{
info=-2;
return;
}
}
}
//--- allocate temporaries
info=2;
rep.m_ngrad=0;
rep.m_nhess=0;
rep.m_ncholesky=0;
ooberrors.m_RelCLSError=0;
ooberrors.m_AvgCE=0;
ooberrors.m_RMSError=0;
ooberrors.m_AvgError=0;
ooberrors.m_AvgRelError=0;
CMLPBase::MLPProperties(ensemble.m_network,nin,nout,wcount);
//--- check
if(CMLPBase::MLPIsSoftMax(ensemble.m_network))
{
ccnt=nin+1;
pcnt=nin;
}
else
{
ccnt=nin+nout;
pcnt=nin+nout;
}
//--- allocation
xys.Resize(npoints,ccnt);
ArrayResizeAL(s,npoints);
oobbuf.Resize(npoints,nout);
ArrayResizeAL(oobcntbuf,npoints);
ArrayResize(x,nin);
ArrayResize(y,nout);
//--- check
if(CMLPBase::MLPIsSoftMax(ensemble.m_network))
ArrayResize(dy,1);
else
ArrayResize(dy,nout);
//--- initialization
for(int i=0; i<npoints; i++)
{
for(int j=0; j<=nout-1; j++)
oobbuf.Set(i,j,0);
}
for(int i=0; i<npoints; i++)
oobcntbuf[i]=0;
//--- main bagging cycle
for(int k=0; k<=ensemble.m_ensemblesize-1; k++)
{
//--- prepare dataset
for(int i=0; i<npoints; i++)
s[i]=false;
for(int i=0; i<npoints; i++)
{
int j=CMath::RandomInteger(npoints);
s[j]=true;
for(int i_=0; i_<=ccnt-1; i_++)
xys.Set(i,i_,xy[j][i_]);
}
//--- train
if(lmalgorithm)
CMLPTrain::MLPTrainLM(network,xys,npoints,decay,restarts,info,tmprep);
else
CMLPTrain::MLPTrainLBFGS(network,xys,npoints,decay,restarts,wstep,maxits,info,tmprep);
//--- check
if(info<0)
return;
//--- save results
rep.m_ngrad=rep.m_ngrad+tmprep.m_ngrad;
rep.m_nhess=rep.m_nhess+tmprep.m_nhess;
rep.m_ncholesky=rep.m_ncholesky+tmprep.m_ncholesky;
//--- copy
i1_=-(k*wcount);
for(int i_=k*wcount; i_<=(k+1)*wcount-1; i_++)
ensemble.m_weights.Set(i_,network.m_weights[i_+i1_]);
//--- copy
i1_=-(k*pcnt);
for(int i_=k*pcnt; i_<=(k+1)*pcnt-1; i_++)
ensemble.m_columnmeans.Set(i_,network.m_columnmeans[i_+i1_]);
//--- copy
i1_=-(k*pcnt);
for(int i_=k*pcnt; i_<=(k+1)*pcnt-1; i_++)
ensemble.m_columnsigmas.Set(i_,network.m_columnsigmas[i_+i1_]);
//--- OOB estimates
for(int i=0; i<npoints; i++)
{
//--- check
if(!s[i])
{
for(int i_=0; i_<=nin-1; i_++)
x[i_]=xy.Get(i,i_);
//--- function call
CMLPBase::MLPProcess(network,x,y);
//--- change value
for(int i_=0; i_<=nout-1; i_++)
oobbuf.Set(i,i_,oobbuf[i][i_]+y[i_]);
oobcntbuf[i]=oobcntbuf[i]+1;
}
}
}
//--- OOB estimates
if(CMLPBase::MLPIsSoftMax(ensemble.m_network))
{
//--- function call
CBdSS::DSErrAllocate(nout,dsbuf);
}
else
{
//--- function call
CBdSS::DSErrAllocate(-nout,dsbuf);
}
for(int i=0; i<npoints; i++)
{
//--- check
if(oobcntbuf[i]!=0)
{
v=1.0/(double)oobcntbuf[i];
for(int i_=0; i_<=nout-1; i_++)
y[i_]=v*oobbuf[i][i_];
//--- check
if(CMLPBase::MLPIsSoftMax(ensemble.m_network))
dy[0]=xy[i][nin];
else
{
i1_=nin;
for(int i_=0; i_<=nout-1; i_++)
dy[i_]=v*xy[i][i_+i1_];
}
//--- function call
CBdSS::DSErrAccumulate(dsbuf,y,dy);
}
}
//--- function call
CBdSS::DSErrFinish(dsbuf);
//--- change values
ooberrors.m_RelCLSError=dsbuf[0];
ooberrors.m_AvgCE=dsbuf[1];
ooberrors.m_RMSError=dsbuf[2];
ooberrors.m_AvgError=dsbuf[3];
ooberrors.m_AvgRelError=dsbuf[4];
}
//+------------------------------------------------------------------+
//| Principal components analysis |
//+------------------------------------------------------------------+
class CPCAnalysis
{
public:
static void PCABuildBasis(CMatrixDouble &x,const int npoints,const int nvars,int &info,double &s2[],CMatrixDouble &v);
static void PCABuildBasis(CMatrixDouble &x,const int npoints,const int nvars,int &info,CRowDouble &s2,CMatrixDouble &v);
static void PCATruncatedSubSpace(CMatrixDouble &x,int npoints,int nvars,int nneeded,double eps,int maxits,CRowDouble &s2,CMatrixDouble &v);
static void PCATruncatedSubSpaceSparse(CSparseMatrix &x,int npoints,int nvars,int nneeded,double eps,int maxits,CRowDouble &s2,CMatrixDouble &v);
};
//+------------------------------------------------------------------+
//| Principal components analysis |
//| Subroutine builds orthogonal basis where first axis corresponds |
//| to direction with maximum variance, second axis maximizes |
//| variance in subspace orthogonal to first axis and so on. |
//| It should be noted that, unlike LDA, PCA does not use class |
//| labels. |
//| INPUT PARAMETERS: |
//| X - dataset, array[0..NPoints-1,0..NVars-1]. |
//| matrix contains ONLY INDEPENDENT VARIABLES. |
//| NPoints - dataset size, NPoints>=0 |
//| NVars - number of independent variables, NVars>=1 |
//| OUTPUT PARAMETERS: |
//| Info - return code: |
//| * -4, if SVD subroutine haven't converged |
//| * -1, if wrong parameters has been passed |
//| (NPoints<0, NVars<1) |
//| * 1, if task is solved |
//| S2 - array[0..NVars-1]. variance values |
//| corresponding to basis vectors. |
//| V - array[0..NVars-1,0..NVars-1] |
//| matrix, whose columns store basis vectors. |
//+------------------------------------------------------------------+
void CPCAnalysis::PCABuildBasis(CMatrixDouble &x,const int npoints,
const int nvars,int &info,
double &s2[],
CMatrixDouble &v)
{
CRowDouble S2=s2;
PCABuildBasis(x,npoints,nvars,info,S2,v);
S2.ToArray(s2);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CPCAnalysis::PCABuildBasis(CMatrixDouble &x,const int npoints,
const int nvars,int &info,
CRowDouble &s2,
CMatrixDouble &v)
{
//--- creating arrays
vector<double> m;
//--- create matrix
CMatrixDouble a;
CMatrixDouble u;
CMatrixDouble vt;
//--- initialization
info=0;
//--- Check input data
if(npoints<0 || nvars<1)
{
info=-1;
return;
}
//--- change value
info=1;
//--- Special case: NPoints=0
if(npoints==0)
{
//--- initialization
s2=vector<double>::Zeros(nvars);
v=matrix<double>::Eye(nvars,nvars);
//--- exit the function
return;
}
//--- Calculate means
CMatrixDouble mx=x;
mx.Resize(npoints,nvars);
m=mx.Mean(0);
//--- Center,apply SVD,prepare output
a=matrix<double>::Zeros(MathMax(npoints,nvars),nvars);
//--- calculation
for(int i=0; i<npoints; i++)
a.Row(i,mx[i]-m);
//--- check
if(!CSingValueDecompose::RMatrixSVD(a,MathMax(npoints,nvars),nvars,0,1,2,s2,u,vt))
{
info=-4;
return;
}
//--- check
if(npoints!=1)
s2=s2.Pow(2.0)/(npoints-1);
//--- function call
v=vt.Transpose()+0;
}
//+------------------------------------------------------------------+
//| Principal components analysis |
//| This function performs truncated PCA, i.e. returns just a few |
//| most important directions. |
//| Internally it uses iterative eigensolver which is very efficient |
//| when only a minor fraction of full basis is required. Thus, if |
//| you need full basis, it is better to use pcabuildbasis() function|
//| It should be noted that, unlike LDA, PCA does not use class |
//| labels. |
//| INPUT PARAMETERS: |
//| X - dataset, array[0..NPoints-1,0..NVars-1] matrix |
//| contains ONLY INDEPENDENT VARIABLES. |
//| NPoints - dataset size, NPoints>=0 |
//| NVars - number of independent variables, NVars>=1 |
//| NNeeded - number of requested components, in [1,NVars] range;|
//| this function is efficient only for NNeeded<<NVars.|
//| Eps - desired precision of vectors returned; underlying |
//| solver will stop iterations as soon as absolute |
//| error in corresponding singular values reduces to |
//| roughly eps*MAX(lambda[]), with lambda[] being |
//| array of eigen values. |
//| Zero value means that algorithm performs number of |
//| iterations specified by maxits parameter, without |
//| paying attention to precision. |
//| MaxIts - number of iterations performed by subspace |
//| iteration method. Zero value means that no limit on|
//| iteration count is placed (eps-based stopping |
//| condition is used). |
//| OUTPUT PARAMETERS: |
//| S2 - array[NNeeded]. Variance values corresponding to |
//| basis vectors. |
//| V - array[NVars,NNeeded] matrix, whose columns store |
//| basis vectors. |
//| NOTE: passing eps=0 and maxits=0 results in small eps being |
//| selected as stopping condition. Exact value of automatically|
//| selected eps is version-dependent. |
//+------------------------------------------------------------------+
void CPCAnalysis::PCATruncatedSubSpace(CMatrixDouble &x,
int npoints,
int nvars,
int nneeded,
double eps,
int maxits,
CRowDouble &s2,
CMatrixDouble &v)
{
//--- check
if(!CAp::Assert(npoints>=0,__FUNCTION__+": npoints<0"))
return;
if(!CAp::Assert(nvars>=1,__FUNCTION__+": nvars<1"))
return;
if(!CAp::Assert(nneeded>0,__FUNCTION__+": nneeded<1"))
return;
if(!CAp::Assert(nneeded<=nvars,__FUNCTION__+": nneeded>nvars"))
return;
if(!CAp::Assert(maxits>=0,__FUNCTION__+": maxits<0"))
return;
if(!CAp::Assert(MathIsValidNumber(eps) && (double)(eps)>=0.0,__FUNCTION__+": eps<0 or is not finite"))
return;
if(!CAp::Assert((int)CAp::Rows(x)>=npoints,__FUNCTION__+": rows(x)<npoints"))
return;
if(!CAp::Assert((int)CAp::Cols(x)>=nvars || npoints==0,__FUNCTION__+": cols(x)<nvars"))
return;
//--- create variables
CMatrixDouble a,b;
vector<double> means;
int k=0;
double vv=0;
CEigSubSpaceState solver;
CEigSubSpaceReport rep;
//--- Special case: NPoints=0
if(npoints==0)
{
s2=vector<double>::Zeros(nneeded);
v=matrix<double>::Eye(nvars,nneeded);
return;
}
//--- Center matrix
a=x;
a.Resize(npoints,nvars);
means=a.Mean(0);
for(int i=0; i<npoints; i++)
{
//--- check
if(!CAp::Assert(a.Row(i,x[i]-means),__FUNCTION__+":error update row " + IntegerToString(i)))
return;
}
//--- Find eigenvalues with subspace iteration solver
CEigenVDetect::EigSubSpaceCreate(nvars,nneeded,solver);
CEigenVDetect::EigSubSpaceSetCond(solver,eps,maxits);
CEigenVDetect::EigSubSpaceOOCStart(solver,0);
while(CEigenVDetect::EigSubSpaceOOCContinue(solver))
{
//--- check
if(!CAp::Assert(solver.m_RequestType==0,__FUNCTION__+": integrity check failed"))
return;
k=solver.m_RequestSize;
CApServ::RMatrixSetLengthAtLeast(b,npoints,k);
CAblas::RMatrixGemm(npoints,k,nvars,1.0,a,0,0,0,solver.m_X,0,0,0,0.0,b,0,0);
CAblas::RMatrixGemm(nvars,k,npoints,1.0,a,0,0,1,b,0,0,0,0.0,solver.m_AX,0,0);
}
CEigenVDetect::EigSubSpaceOOCStop(solver,s2,v,rep);
if(npoints!=1)
s2/=(npoints-1);
}
//+------------------------------------------------------------------+
//| Sparse truncated principal components analysis |
//| This function performs sparse truncated PCA, i.e. returns just a |
//| few most important principal components for a sparse input X. |
//| Internally it uses iterative eigensolver which is very efficient |
//| when only a minor fraction of full basis is required. |
//| It should be noted that, unlike LDA, PCA does not use class |
//| labels. |
//| INPUT PARAMETERS: |
//| X - sparse dataset, sparse npoints*nvars matrix. It is |
//| recommended to use CRS sparse storage format; |
//| non-CRS input will be internally converted to CRS. |
//| Matrix contains ONLY INDEPENDENT VARIABLES, and |
//| must be EXACTLY npoints*nvars. |
//| NPoints - dataset size, NPoints>=0 |
//| NVars - number of independent variables, NVars>=1 |
//| NNeeded - number of requested components, in [1,NVars] range;|
//| this function is efficient only for NNeeded<<NVars.|
//| Eps - desired precision of vectors returned; underlying |
//| solver will stop iterations as soon as absolute |
//| error in corresponding singular values reduces to |
//| roughly eps*MAX(lambda[]), with lambda[] being |
//| array of eigen values. |
//| Zero value means that algorithm performs number of |
//| iterations specified by maxits parameter, without |
//| paying attention to precision. |
//| MaxIts - number of iterations performed by subspace |
//| iteration method. Zero value means that no limit on|
//| iteration count is placed (eps-based stopping |
//| condition is used). |
//| OUTPUT PARAMETERS: |
//| S2 - array[NNeeded]. Variance values corresponding to |
//| basis vectors. |
//| V - array[NVars,NNeeded] matrix, whose columns store |
//| basis vectors. |
//| NOTE: passing eps=0 and maxits=0 results in small eps being |
//| selected as a stopping condition. Exact value of |
//| automatically selected eps is version-dependent. |
//| NOTE: zero MaxIts is silently replaced by some reasonable value |
//| which prevents eternal loops (possible when inputs are |
//| degenerate and too stringent stopping criteria are |
//| specified). In current version it is 50+2*NVars. |
//+------------------------------------------------------------------+
void CPCAnalysis::PCATruncatedSubSpaceSparse(CSparseMatrix &x,
int npoints,
int nvars,
int nneeded,
double eps,
int maxits,
CRowDouble &s2,
CMatrixDouble &v)
{
//--- check
if(!CAp::Assert(npoints>=0,__FUNCTION__+": npoints<0"))
return;
if(!CAp::Assert(nvars>=1,__FUNCTION__+": nvars<1"))
return;
if(!CAp::Assert(nneeded>0,__FUNCTION__+": nneeded<1"))
return;
if(!CAp::Assert(nneeded<=nvars,__FUNCTION__+": nneeded>nvars"))
return;
if(!CAp::Assert(maxits>=0,__FUNCTION__+": maxits<0"))
return;
if(!CAp::Assert(MathIsValidNumber(eps) && eps>=0.0,__FUNCTION__+": eps<0 or is not finite"))
return;
//--- create variables
double vv=0;
//--- create arrays
CSparseMatrix xcrs;
CRowDouble b1;
CRowDouble c1;
CRowDouble z1;
CRowDouble means;
CEigSubSpaceState solver;
CEigSubSpaceReport rep;
//--- Special case: NPoints=0
if(npoints==0)
{
s2=vector<double>::Zeros(nneeded);
v=matrix<double>::Identity(nvars,nneeded);
return;
}
//--- check
if(!CAp::Assert(CSparse::SparseGetNRows(x)==npoints,__FUNCTION__+": rows(x)!=npoints"))
return;
if(!CAp::Assert(CSparse::SparseGetNCols(x)==nvars,__FUNCTION__+": cols(x)!=nvars"))
return;
//--- If input data are not in CRS format, perform conversion to CRS
if(!CSparse::SparseIsCRS(x))
{
CSparse::SparseCopyToCRS(x,xcrs);
PCATruncatedSubSpaceSparse(xcrs,npoints,nvars,nneeded,eps,maxits,s2,v);
return;
}
//--- Initialize parameters, prepare buffers
if(eps==0.0 && maxits==0)
eps=1.0E-6;
if(maxits==0)
maxits=50+2*nvars;
//--- Calculate mean values
vv=1.0/(double)npoints;
b1=vector<double>::Full(npoints,vv);
CSparse::SparseMTV(x,b1,means);
//--- Find eigenvalues with subspace iteration solver
CEigenVDetect::EigSubSpaceCreate(nvars,nneeded,solver);
CEigenVDetect::EigSubSpaceSetCond(solver,eps,maxits);
CEigenVDetect::EigSubSpaceOOCStart(solver,0);
while(CEigenVDetect::EigSubSpaceOOCContinue(solver))
{
if(!CAp::Assert(solver.m_RequestType==0,__FUNCTION__+": integrity check failed"))
return;
for(int k=0; k<solver.m_RequestSize; k++)
{
//--- Calculate B1=(X-meansX)*Zk
z1=solver.m_X.Col(k)+0;
CSparse::SparseMV(x,z1,b1);
vv=means.Dot(solver.m_X.Col(k)+0);
b1-=vv;
//--- Calculate (X-meansX)^T*B1
CSparse::SparseMTV(x,b1,c1);
vv=b1.Sum();
solver.m_AX.Col(k,(c1.ToVector()-means*vv));
}
}
CEigenVDetect::EigSubSpaceOOCStop(solver,s2,v,rep);
if(npoints!=1)
s2/=(npoints-1);
}
//+------------------------------------------------------------------+
//| This structure is used to store temporaries for |
//| KMeansGenerateInternal function, so we will be able to reuse them|
//| during multiple subsequent calls. |
//+------------------------------------------------------------------+
struct CKmeansBuffers
{
CMatrixDouble m_ct;
CMatrixDouble m_ctbest;
CRowInt m_xycbest;
CRowInt m_xycprev;
CRowDouble m_d2;
CRowInt m_csizes;
CApBuff m_initbuf;
//---
CKmeansBuffers(void) {}
~CKmeansBuffers(void) {}
//---
void Copy(const CKmeansBuffers &obj);
//--- overloading
void operator=(const CKmeansBuffers &obj) { Copy(obj); }
};
//+------------------------------------------------------------------+
//| Copy |
//+------------------------------------------------------------------+
void CKmeansBuffers::Copy(const CKmeansBuffers &obj)
{
m_ct=obj.m_ct;
m_ctbest=obj.m_ctbest;
m_xycbest=obj.m_xycbest;
m_xycprev=obj.m_xycprev;
m_d2=obj.m_d2;
m_csizes=obj.m_csizes;
m_initbuf=obj.m_initbuf;
}
//+------------------------------------------------------------------+
//| This structure is a clusterization engine. |
//| You should not try to access its fields directly. |
//| Use ALGLIB functions in order to work with this object. |
//+------------------------------------------------------------------+
struct CClusterizerState
{
int m_NPoints;
int m_nfeatures;
int m_disttype;
CMatrixDouble m_xy;
CMatrixDouble m_d;
int m_ahcalgo;
int m_kmeansrestarts;
int m_kmeansmaxits;
int m_kmeansinitalgo;
bool m_kmeansdbgnoits;
int m_seed;
CMatrixDouble m_tmpd;
CApBuff m_distbuf;
CKmeansBuffers m_kmeanstmp;
//---
CClusterizerState(void);
~CClusterizerState(void) {}
//---
void Copy(const CClusterizerState &obj);
//--- overloading
void operator=(const CClusterizerState &obj) { Copy(obj); }
};
//+------------------------------------------------------------------+
//| Constructor |
//+------------------------------------------------------------------+
CClusterizerState::CClusterizerState(void)
{
m_NPoints=0;
m_nfeatures=0;
m_disttype=0;
m_ahcalgo=0;
m_kmeansrestarts=0;
m_kmeansmaxits=0;
m_kmeansinitalgo=0;
m_kmeansdbgnoits=0;
m_seed=0;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CClusterizerState::Copy(const CClusterizerState &obj)
{
m_NPoints=obj.m_NPoints;
m_nfeatures=obj.m_nfeatures;
m_disttype=obj.m_disttype;
m_xy=obj.m_xy;
m_d=obj.m_d;
m_ahcalgo=obj.m_ahcalgo;
m_kmeansrestarts=obj.m_kmeansrestarts;
m_kmeansmaxits=obj.m_kmeansmaxits;
m_kmeansinitalgo=obj.m_kmeansinitalgo;
m_kmeansdbgnoits=obj.m_kmeansdbgnoits;
m_seed=obj.m_seed;
m_tmpd=obj.m_tmpd;
m_distbuf=obj.m_distbuf;
m_kmeanstmp=obj.m_kmeanstmp;
}
//+------------------------------------------------------------------+
//| This structure is used to store results of the agglomerative |
//| hierarchical clustering (AHC). |
//| Following information is returned: |
//| * TerminationType - completion code: |
//| * 1 for successful completion of algorithm |
//| * -5 inappropriate combination of clustering algorithm and |
//| distance function was used. As for now, it is possible |
//| only when Ward's method is called for dataset with |
//| non-Euclidean distance function. |
//| In case negative completion code is returned, other fields of |
//| report structure are invalid and should not be used. |
//| * NPoints contains number of points in the original dataset |
//| * Z contains information about merges performed (see below). |
//| Z contains indexes from the original (unsorted) dataset and |
//| it can be used when you need to know what points were merged.|
//| However, it is not convenient when you want to build a |
//| dendrograd (see below). |
//| * if you want to build dendrogram, you can use Z, but it is not|
//| good option, because Z contains indexes from unsorted |
//| dataset. Dendrogram built from such dataset is likely to |
//| have intersections. So, you have to reorder you points before|
//| building dendrogram. |
//| Permutation which reorders point is returned in P. Another |
//| representation of merges, which is more convenient for dendorgram|
//| construction, is returned in PM. |
//| * more information on format of Z, P and PM can be found below |
//| and in the examples from ALGLIB Reference Manual. |
//| FORMAL DESCRIPTION OF FIELDS: |
//| NPoints - number of points |
//| Z - array[NPoints-1,2], contains indexes of clusters|
//| linked in pairs to form clustering tree. I-th |
//| row corresponds to I-th merge: |
//| * Z[I,0] - index of the first cluster to merge |
//| * Z[I,1] - index of the second cluster to merge |
//| * Z[I,0]<Z[I,1] |
//| * clusters are numbered from 0 to 2*NPoints-2, with|
//| indexes from 0 to NPoints-1 corresponding to |
//| points of the original dataset, and indexes from |
//| NPoints to 2*NPoints-2 correspond to clusters |
//| generated by subsequent merges (I-th row of Z |
//| creates cluster with index NPoints+I). |
//| IMPORTANT: indexes in Z[] are indexes in the ORIGINAL, unsorted |
//| dataset. In addition to Z algorithm outputs |
//| permutation which rearranges points in such way that |
//| subsequent merges are performed on adjacent points |
//| (such order is needed if you want to build |
//| dendrogram). However, indexes in Z are related to |
//| original, unrearranged sequence of points. |
//| P - array[NPoints], permutation which reorders points |
//| for dendrogram construction. P[i] contains index |
//| of the position where we should move I-th point |
//| of the original dataset in order to apply merges |
//| PZ/PM. |
//| PZ - same as Z, but for permutation of points given by P|
//| The only thing which changed are indexes of the |
//| original points; indexes of clusters remained same.|
//| MergeDist- array[NPoints-1], contains distances between |
//| clusters being merged (MergeDist[i] correspond to |
//| merge stored in Z[i,...]): |
//| * CLINK, SLINK and average linkage algorithms |
//| report "raw", unmodified distance metric. |
//| * Ward's method reports weighted intra-cluster |
//| variance, which is equal to |
//| ||Ca-Cb||^2 * Sa*Sb/(Sa+Sb). |
//| Here A and B are clusters being merged, Ca is a |
//| center of A, Cb is a center of B, Sa is a size |
//| of A, Sb is a size of B. |
//| PM - array[NPoints-1,6], another representation of |
//| merges, which is suited for dendrogram construction|
//| It deals with rearranged points (permutation P is |
//| applied) and represents merges in a form which |
//| different from one used by Z. For each I from 0 to |
//| NPoints-2, I-th row of PM represents merge |
//| performed on two clusters C0 and C1. Here: |
//| * C0 contains points with indexes PM[I,0]...PM[I,1]|
//| * C1 contains points with indexes PM[I,2]...PM[I,3]|
//| * indexes stored in PM are given for dataset sorted|
//| according to permutation P |
//| * PM[I,1]=PM[I,2]-1 (only adjacent clusters are |
//| merged) |
//| * PM[I,0]<=PM[I,1], PM[I,2]<=PM[I,3], i.e. both |
//| clusters contain at least one point |
//| *heights of "subdendrograms" corresponding to |
//| C0/C1 are stored in PM[I,4] and PM[I,5]. |
//| Subdendrograms corresponding to single-point |
//| clusters have height=0. Dendrogram of the merge |
//| result has height H=max(H0,H1)+1. |
//| NOTE: there is one-to-one correspondence between merges described|
//| by Z and PM. I-th row of Z describes same merge of clusters|
//| as I-th row of PM, with "left" cluster from Z corresponding|
//| to the "left" one from PM. |
//+------------------------------------------------------------------+
struct CAHCReport
{
int m_terminationtype;
int m_NPoints;
CRowInt m_p;
CMatrixInt m_z;
CMatrixInt m_pz;
CMatrixInt m_pm;
CRowDouble m_mergedist;
//---
CAHCReport(void) { m_terminationtype=0; m_NPoints=0; }
~CAHCReport(void) {}
//---
void Copy(const CAHCReport &obj);
//--- overloading
void operator=(const CAHCReport &obj) { Copy(obj); }
};
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CAHCReport::Copy(const CAHCReport &obj)
{
m_terminationtype=obj.m_terminationtype;
m_NPoints=obj.m_NPoints;
m_p=obj.m_p;
m_z=obj.m_z;
m_pz=obj.m_pz;
m_pm=obj.m_pm;
m_mergedist=obj.m_mergedist;
}
//+------------------------------------------------------------------+
//| This structure is used to store results of the k-means clustering|
//| algorithm. |
//| Following information is always returned: |
//| * NPoints contains number of points in the original dataset |
//| * TerminationType contains completion code, negative on |
//| failure, positive on success |
//| * K contains number of clusters |
//| For positive TerminationType we return: |
//| * NFeatures contains number of variables in the original |
//| dataset |
//| * C, which contains centers found by algorithm |
//| * CIdx, which maps points of the original dataset to clusters |
//| FORMAL DESCRIPTION OF FIELDS: |
//| NPoints - number of points, >=0 |
//| NFeatures - number of variables, >=1 |
//| TerminationType completion code: |
//| * -5 if distance type is anything different from |
//| Euclidean metric |
//| * -3 for degenerate dataset: a) less than K distinct |
//| points, b) K=0 for non-empty dataset. |
//| * +1 for successful completion |
//| K - number of clusters |
//| C - array[K,NFeatures], rows of the array store centers|
//| CIdx - array[NPoints], which contains cluster indexes |
//| IterationsCount actual number of iterations performed by |
//| clusterizer. If algorithm performed more than one |
//| random restart, total number of iterations is |
//| returned. |
//| Energy - merit function, "energy", sum of squared deviations|
//| from cluster centers |
//+------------------------------------------------------------------+
struct CKmeansReport
{
int m_NPoints;
int m_nfeatures;
int m_terminationtype;
int m_iterationscount;
double m_energy;
int m_k;
CMatrixDouble m_c;
CRowInt m_cidx;
//---
CKmeansReport(void);
~CKmeansReport(void) {}
//---
void Copy(const CKmeansReport &obj);
//--- overloading
void operator=(const CKmeansReport &obj) { Copy(obj); }
};
//+------------------------------------------------------------------+
//| Constructor |
//+------------------------------------------------------------------+
CKmeansReport::CKmeansReport(void)
{
m_NPoints=0;
m_nfeatures=0;
m_terminationtype=0;
m_iterationscount=0;
m_energy=0;
m_k=0;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CKmeansReport::Copy(const CKmeansReport &obj)
{
m_NPoints=obj.m_NPoints;
m_nfeatures=obj.m_nfeatures;
m_terminationtype=obj.m_terminationtype;
m_iterationscount=obj.m_iterationscount;
m_energy=obj.m_energy;
m_k=obj.m_k;
m_c=obj.m_c;
m_cidx=obj.m_cidx;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
class CClustering
{
public:
//--- constants
static const int m_kmeansblocksize;
static const int m_kmeansparalleldim;
static const int m_kmeansparallelk;
static const double m_complexitymultiplier;
static void ClusterizerCreate(CClusterizerState &s);
static void ClusterizerSetPoints(CClusterizerState &s,CMatrixDouble &xy,int npoints,int nfeatures,int disttype);
static void ClusterizerSetDistances(CClusterizerState &s,CMatrixDouble &d,int npoints,bool IsUpper);
static void ClusterizerSetAHCAlgo(CClusterizerState &s,int algo);
static void ClusterizerSetKMeansLimits(CClusterizerState &s,int restarts,int maxits);
static void ClusterizerSetKMeansInit(CClusterizerState &s,int initalgo);
static void ClusterizerSetSeed(CClusterizerState &s,int seed);
static void ClusterizerRunAHC(CClusterizerState &s,CAHCReport &rep);
static void ClusterizerRunKMeans(CClusterizerState &s,int k,CKmeansReport &rep);
static void ClusterizerGetDistances(CMatrixDouble &xy,int npoints,int nfeatures,int disttype,CMatrixDouble &d);
static void ClusterizerGetDistancesBuf(CApBuff &buf,CMatrixDouble &xy,int npoints,int nfeatures,int disttype,CMatrixDouble &d);
static void ClusterizerGetKClusters(CAHCReport &rep,int k,CRowInt &cidx,CRowInt &cz);
static void ClusterizerSeparatedByDist(CAHCReport &rep,double r,int &k,CRowInt &cidx,CRowInt &cz);
static void ClusterizerSeparatedByCorr(CAHCReport &rep,double r,int &k,CRowInt &cidx,CRowInt &cz);
static void KMeansGenerateInternal(CMatrixDouble &xy,int npoints,int nvars,int k,int initalgo,int seed,int maxits,
int restarts,bool kmeansdbgnoits,int &info,int &iterationscount,CMatrixDouble &ccol,
bool needccol,CMatrixDouble &crow,bool needcrow,CRowInt &xyc,double &energy,CKmeansBuffers &buf);
static void KMeansUpdateDistances(CMatrixDouble &xy,int idx0,int idx1,int nvars,CMatrixDouble &ct,int cidx0,int cidx1,CRowInt &xyc,CRowDouble &xydist2);
private:
static void SelectInitialCenters(CMatrixDouble &xy,int npoints,int nvars,int initalgo,CHighQualityRandState &rs,int k,CMatrixDouble &ct,CApBuff &initbuf);
static bool FixCenters(CMatrixDouble &xy,int npoints,int nvars,CMatrixDouble &ct,int k,CApBuff &initbuf);
static void ClusterizerRunAHCInternal(CClusterizerState &s,CMatrixDouble &d,CAHCReport &rep);
static void EvaluateDistanceMatrixRec(CMatrixDouble &xy,int nfeatures,int disttype,CMatrixDouble &d,int i0,int i1,int j0,int j1);
};
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
const int CClustering::m_kmeansblocksize=32;
const int CClustering::m_kmeansparalleldim=8;
const int CClustering::m_kmeansparallelk=4;
const double CClustering::m_complexitymultiplier=1.0;
//+------------------------------------------------------------------+
//| This function initializes clusterizer object. Newly initialized |
//| object is empty, i.e. it does not contain dataset. You should |
//| use it as follows: |
//| 1. creation |
//| 2. dataset is added with ClusterizerSetPoints() |
//| 3. additional parameters are set |
//| 3. clusterization is performed with one of the clustering |
//| functions |
//+------------------------------------------------------------------+
void CClustering::ClusterizerCreate(CClusterizerState &s)
{
s.m_NPoints=0;
s.m_nfeatures=0;
s.m_disttype=2;
s.m_ahcalgo=0;
s.m_kmeansrestarts=1;
s.m_kmeansmaxits=0;
s.m_kmeansinitalgo=0;
s.m_kmeansdbgnoits=false;
s.m_seed=1;
}
//+------------------------------------------------------------------+
//| This function adds dataset to the clusterizer structure. |
//| This function overrides all previous calls of |
//| ClusterizerSetPoints() or ClusterizerSetDistances(). |
//| INPUT PARAMETERS: |
//| S - clusterizer state, initialized by |
//| ClusterizerCreate() |
//| XY - array[NPoints,NFeatures], dataset |
//| NPoints - number of points, >=0 |
//| NFeatures- number of features, >=1 |
//| DistType - distance function: |
//| * 0 Chebyshev distance (L-inf norm) |
//| * 1 city block distance (L1 norm) |
//| * 2 Euclidean distance (L2 norm), non-squared |
//| * 10 Pearson correlation: |
//| dist(a,b) = 1-corr(a,b) |
//| * 11 Absolute Pearson correlation: |
//| dist(a,b) = 1-|corr(a,b)| |
//| * 12 Uncentered Pearson correlation (cosine of |
//| the angle): dist(a,b) = a'*b/(|a|*|b|) |
//| * 13 Absolute uncentered Pearson correlation |
//| dist(a,b) = |a'*b|/(|a|*|b|) |
//| * 20 Spearman rank correlation: |
//| dist(a,b) = 1-rankcorr(a,b) |
//| * 21 Absolute Spearman rank correlation |
//| dist(a,b) = 1-|rankcorr(a,b)| |
//| NOTE 1: different distance functions have different performance |
//| penalty: |
//| * Euclidean or Pearson correlation distances are |
//| the fastest ones |
//| * Spearman correlation distance function is a bit slower |
//| * city block and Chebyshev distances are order |
//| of magnitude slower |
//| The reason behing difference in performance is that |
//| correlation-based distance functions are computed using |
//| optimized linear algebra kernels, while Chebyshev and |
//| city block distance functions are computed using simple |
//| nested loops with two branches at each iteration. |
//| NOTE 2: different clustering algorithms have different |
//| limitations: |
//| * agglomerative hierarchical clustering algorithms may |
//| be used with any kind of distance metric |
//| * k-means++ clustering algorithm may be used only with |
//| Euclidean distance function |
//| Thus, list of specific clustering algorithms you may use |
//| depends on distance function you specify when you set |
//| your dataset. |
//+------------------------------------------------------------------+
void CClustering::ClusterizerSetPoints(CClusterizerState &s,
CMatrixDouble &xy,
int npoints,
int nfeatures,
int disttype)
{
//--- check
if(!CAp::Assert(disttype==0 || disttype==1 || disttype==2 || disttype==10 || disttype==11 ||
disttype==12 || disttype==13 || disttype==20 || disttype==21,__FUNCTION__": incorrect DistType"))
return;
if(!CAp::Assert(npoints>=0,__FUNCTION__": NPoints<0"))
return;
if(!CAp::Assert(nfeatures>=1,__FUNCTION__": NFeatures<1"))
return;
if(!CAp::Assert(xy.Rows()>=npoints,__FUNCTION__": Rows(XY)<NPoints"))
return;
if(!CAp::Assert(xy.Cols()>=nfeatures,__FUNCTION__": Cols(XY)<NFeatures"))
return;
if(!CAp::Assert(CApServ::IsFiniteMatrix(xy,npoints,nfeatures),__FUNCTION__": XY contains NAN/INF"))
return;
s.m_NPoints=npoints;
s.m_nfeatures=nfeatures;
s.m_disttype=disttype;
s.m_xy=xy;
s.m_xy.Resize(npoints,nfeatures);
}
//+------------------------------------------------------------------+
//| This function adds dataset given by distance matrix to the |
//| clusterizer structure. It is important that dataset is not given |
//| explicitly - only distance matrix is given. |
//| This function overrides all previous calls of |
//| ClusterizerSetPoints() or ClusterizerSetDistances(). |
//| INPUT PARAMETERS: |
//| S - clusterizer state, initialized by |
//| ClusterizerCreate() |
//| D - array[NPoints,NPoints], distance matrix given by |
//| its upper or lower triangle (main diagonal is |
//| ignored because its entries are expected to |
//| be zero). |
//| NPoints - number of points |
//| IsUpper - whether upper or lower triangle of D is given. |
//| NOTE 1: different clustering algorithms have different |
//| limitations: |
//| * agglomerative hierarchical clustering algorithms may |
//| be used with any kind of distance metric, including |
//| one which is given by distance matrix |
//| * k-means++ clustering algorithm may be used only with |
//| Euclidean distance function and explicitly given |
//| points - it can not be used with dataset given by |
//| distance matrix. Thus, if you call this function, you |
//| will be unable to use k-means clustering algorithm |
//| to process your problem. |
//+------------------------------------------------------------------+
void CClustering::ClusterizerSetDistances(CClusterizerState &s,
CMatrixDouble &d,
int npoints,
bool IsUpper)
{
//--- check
if(!CAp::Assert(npoints>=0,__FUNCTION__": NPoints<0"))
return;
if(!CAp::Assert(d.Rows()>=npoints,__FUNCTION__": Rows(D)<NPoints"))
return;
if(!CAp::Assert(d.Cols()>=npoints,__FUNCTION__": Cols(D)<NPoints"))
return;
//--- initialization
s.m_NPoints=npoints;
s.m_nfeatures=0;
s.m_disttype=-1;
s.m_d=d;
s.m_d.Resize(npoints,npoints);
s.m_d.Diag(vector<double>::Zeros(npoints));
if(IsUpper)
s.m_d=s.m_d.TriU()+0;
else
s.m_d=s.m_d.TriL()+0;
s.m_d+=s.m_d.Transpose()+0;
if(!CAp::Assert(CApServ::IsFiniteMatrix(s.m_d,npoints,npoints) || s.m_d.Min()>=0,__FUNCTION__": D contains infinite,NAN or negative elements"))
return;
}
//+------------------------------------------------------------------+
//| This function sets agglomerative hierarchical clustering |
//| algorithm |
//| INPUT PARAMETERS: |
//| S - clusterizer state, initialized by ClusterizerCreate() |
//| Algo - algorithm type: |
//| * 0 complete linkage(default algorithm) |
//| * 1 single linkage |
//| * 2 unweighted average linkage |
//| * 3 weighted average linkage |
//| * 4 Ward's method |
//| NOTE: Ward's method works correctly only with Euclidean distance,|
//| that's why algorithm will return negative termination |
//| code(failure) for any other distance type. |
//| It is possible, however, to use this method with user - supplied |
//| distance matrix. It is your responsibility to pass one which was |
//| calculated with Euclidean distance function. |
//+------------------------------------------------------------------+
void CClustering::ClusterizerSetAHCAlgo(CClusterizerState &s,int algo)
{
if(!CAp::Assert(algo==0 || algo==1 || algo==2 || algo==3 || algo==4,__FUNCTION__": incorrect algorithm type"))
return;
s.m_ahcalgo=algo;
}
//+------------------------------------------------------------------+
//| This function sets k-means properties: |
//| number of restarts and maximum |
//| number of iterations per one run. |
//| INPUT PARAMETERS: |
//| S - clusterizer state, initialized by |
//| ClusterizerCreate() |
//| Restarts - restarts count, >= 1. |
//| k-means++ algorithm performs several restarts |
//| and chooses best set of centers(one with minimum |
//| squared distance). |
//| MaxIts - maximum number of k-means iterations performed |
//| during one run. >= 0, zero value means that |
//| algorithm performs unlimited number of iterations. |
//+------------------------------------------------------------------+
void CClustering::ClusterizerSetKMeansLimits(CClusterizerState &s,
int restarts,
int maxits)
{
if(!CAp::Assert(restarts>=1,__FUNCTION__": Restarts<=0"))
return;
if(!CAp::Assert(maxits>=0,__FUNCTION__": MaxIts<0"))
return;
s.m_kmeansrestarts=restarts;
s.m_kmeansmaxits=maxits;
}
//+------------------------------------------------------------------+
//| This function sets k-means initialization algorithm. Several |
//| different algorithms can be chosen, including k-means++. |
//| INPUT PARAMETERS: |
//| S - clusterizer state, initialized by |
//| ClusterizerCreate() |
//| InitAlgo - initialization algorithm: |
//| * 0 automatic selection(different versions of ALGLIB |
//| may select different algorithms) |
//| * 1 random initialization |
//| * 2 k-means++ initialization(best quality of initial |
//| centers, but long non-parallelizable initialization |
//| phase with bad cache locality) |
//| *3 "fast-greedy" algorithm with efficient, easy to |
//| parallelize initialization. Quality of initial centers |
//| is somewhat worse than that of k-means++. This |
//| algorithm is a default one in the current version of |
//| ALGLIB. |
//| *-1 "debug" algorithm which always selects first K rows |
//| of dataset; this algorithm is used for debug purposes |
//| only. Do not use it in the industrial code! |
//+------------------------------------------------------------------+
void CClustering::ClusterizerSetKMeansInit(CClusterizerState &s,
int initalgo)
{
if(!CAp::Assert(initalgo>=-1 && initalgo<=3,__FUNCTION__": InitAlgo is incorrect"))
return;
s.m_kmeansinitalgo=initalgo;
}
//+------------------------------------------------------------------+
//| This function sets seed which is used to initialize internal RNG.|
//| By default, deterministic seed is used - same for each run of |
//| clusterizer. If you specify non-deterministic seed value, then |
//| some algorithms which depend on random initialization(in current |
//| version : k-means) may return slightly different results after |
//| each run. |
//| INPUT PARAMETERS: |
//| S - clusterizer state, initialized by |
//| ClusterizerCreate() |
//| Seed - seed: |
//| * positive values = use deterministic seed for each|
//| run of algorithms which depend on random |
//| initialization |
//| * zero or negative values = use non-deterministic |
//| seed |
//+------------------------------------------------------------------+
void CClustering::ClusterizerSetSeed(CClusterizerState &s,int seed)
{
s.m_seed=seed;
}
//+------------------------------------------------------------------+
//| This function performs agglomerative hierarchical clustering |
//| NOTE: Agglomerative hierarchical clustering algorithm has two |
//| phases: distance matrix calculation and clustering itself. |
//| INPUT PARAMETERS: |
//| S - clusterizer state, initialized by |
//| ClusterizerCreate() |
//| OUTPUT PARAMETERS: |
//| Rep - clustering results; see description of AHCReport |
//| structure for more information. |
//| NOTE 1: hierarchical clustering algorithms require large amounts |
//| of memory. In particular, this implementation needs |
//| sizeof(double) *NPoints^2 bytes, which are used to store |
//| distance matrix. In case we work with user - supplied |
//| matrix, this amount is multiplied by 2 (we have to store |
//| original matrix and to work with its copy). |
//| For example, problem with 10000 points would require 800M|
//| of RAM, even when working in a 1-dimensional space. |
//+------------------------------------------------------------------+
void CClustering::ClusterizerRunAHC(CClusterizerState &s,
CAHCReport &rep)
{
//--- create variables
int npoints=s.m_NPoints;
int nfeatures=s.m_nfeatures;
//--- Fill Rep.NPoints, quick exit when NPoints<=1
rep.m_NPoints=npoints;
if(npoints==0)
{
rep.m_p.Resize(0);
rep.m_z.Resize(0,0);
rep.m_pz.Resize(0,0);
rep.m_pm.Resize(0,0);
rep.m_mergedist.Resize(0);
rep.m_terminationtype=1;
return;
}
if(npoints==1)
{
rep.m_p.Resize(1);
rep.m_z.Resize(0,0);
rep.m_pz.Resize(0,0);
rep.m_pm.Resize(0,0);
rep.m_mergedist.Resize(0);
rep.m_p.Set(0,0);
rep.m_terminationtype=1;
return;
}
//--- More than one point
if(s.m_disttype==-1)
{
//--- Run clusterizer with user-supplied distance matrix
ClusterizerRunAHCInternal(s,s.m_d,rep);
return;
}
else
{
//--- Check combination of AHC algo and distance type
if(s.m_ahcalgo==4 && s.m_disttype!=2)
{
rep.m_terminationtype=-5;
return;
}
//--- Build distance matrix D.
ClusterizerGetDistancesBuf(s.m_distbuf,s.m_xy,npoints,nfeatures,s.m_disttype,s.m_tmpd);
//--- Run clusterizer
ClusterizerRunAHCInternal(s,s.m_tmpd,rep);
return;
}
}
//+------------------------------------------------------------------+
//| This function performs clustering by k-means++ algorithm. |
//| You may change algorithm properties by calling: |
//| * ClusterizerSetKMeansLimits() to change number of restarts |
//| or iterations |
//| * ClusterizerSetKMeansInit() to change initialization |
//| algorithm |
//| By default, one restart and unlimited number of iterations are |
//| used. Initialization algorithm is chosen automatically. |
//| NOTE: k-means clustering algorithm has two phases: selection of |
//| initial centers and clustering itself. |
//| INPUT PARAMETERS: |
//| S - clusterizer state, initialized by |
//| ClusterizerCreate() |
//| K - number of clusters, K >= 0. |
//| K can be zero only when algorithm is called for |
//| empty dataset, in this case completion code is set |
//| to success(+1). |
//| If K = 0 and dataset size is non-zero, we can not |
//| meaningfully assign points to some center(there are|
//| no centers because K = 0) and return -3 as |
//| completion code (failure). |
//| OUTPUT PARAMETERS: |
//| Rep - clustering results; see description of KMeansReport|
//| structure for more information. |
//| NOTE 1: k-means clustering can be performed only for datasets |
//| with Euclidean distance function. Algorithm will return |
//| negative completion code in Rep.TerminationType in case |
//| dataset was added to clusterizer with DistType other |
//| than Euclidean (or dataset was specified by distance |
//| matrix instead of explicitly given points). |
//| NOTE 2: by default, k-means uses non-deterministic seed to |
//| initialize RNG which is used to select initial centers. |
//| As result, each run of algorithm may return different |
//| values. If you need deterministic behavior, use |
//| ClusterizerSetSeed() function. |
//+------------------------------------------------------------------+
void CClustering::ClusterizerRunKMeans(CClusterizerState &s,
int k,
CKmeansReport &rep)
{
//--- create variables
CMatrixDouble dummy;
//--- check
if(!CAp::Assert(k>=0,__FUNCTION__": K<0"))
return;
//--- Incorrect distance type
if(s.m_disttype!=2)
{
rep.m_NPoints=s.m_NPoints;
rep.m_terminationtype=-5;
rep.m_k=k;
rep.m_iterationscount=0;
rep.m_energy=0.0;
return;
}
//--- K>NPoints or (K=0 and NPoints>0)
if(k>s.m_NPoints || (k==0 && s.m_NPoints>0))
{
rep.m_NPoints=s.m_NPoints;
rep.m_terminationtype=-3;
rep.m_k=k;
rep.m_iterationscount=0;
rep.m_energy=0.0;
return;
}
//--- No points
if(s.m_NPoints==0)
{
rep.m_NPoints=0;
rep.m_terminationtype=1;
rep.m_k=k;
rep.m_iterationscount=0;
rep.m_energy=0.0;
return;
}
//--- Normal case:
//--- 1<=K<=NPoints, Euclidean distance
rep.m_NPoints=s.m_NPoints;
rep.m_nfeatures=s.m_nfeatures;
rep.m_k=k;
rep.m_NPoints=s.m_NPoints;
rep.m_nfeatures=s.m_nfeatures;
KMeansGenerateInternal(s.m_xy,s.m_NPoints,s.m_nfeatures,k,s.m_kmeansinitalgo,s.m_seed,s.m_kmeansmaxits,
s.m_kmeansrestarts,s.m_kmeansdbgnoits,rep.m_terminationtype,rep.m_iterationscount,dummy,
false,rep.m_c,true,rep.m_cidx,rep.m_energy,s.m_kmeanstmp);
}
//+------------------------------------------------------------------+
//| This function returns distance matrix for dataset |
//| INPUT PARAMETERS: |
//| XY - array[NPoints, NFeatures], dataset |
//| NPoints - number of points, >= 0 |
//| NFeatures- number of features, >= 1 |
//| DistType - distance function: |
//| * 0 Chebyshev distance(L - inf norm) |
//| * 1 city block distance(L1 norm) |
//| * 2 Euclidean distance(L2 norm, non - squared) |
//| * 10 Pearson correlation: |
//| dist(a, b) = 1 - corr(a, b) |
//| * 11 Absolute Pearson correlation: |
//| dist(a, b) = 1 - |corr(a, b)| |
//| * 12 Uncentered Pearson correlation(cosine of |
//| the angle): dist(a, b) = a'*b/(|a|*|b|) |
//| * 13 Absolute uncentered Pearson correlation |
//| dist(a, b) = |a'*b|/(|a|*|b|) |
//| * 20 Spearman rank correlation: |
//| dist(a, b) = 1 - rankcorr(a, b) |
//| * 21 Absolute Spearman rank correlation |
//| dist(a, b) = 1 - |rankcorr(a, b)| |
//| OUTPUT PARAMETERS: |
//| D - array[NPoints, NPoints], distance matrix (full |
//| matrix is returned, with lower and upper triangles)|
//| NOTE: different distance functions have different performance |
//| penalty: |
//| * Euclidean or Pearson correlation distances are the fastest|
//| ones |
//| * Spearman correlation distance function is a bit slower |
//| * city block and Chebyshev distances are order of magnitude |
//| slower |
//| The reason behing difference in performance is that correlation -|
//| based distance functions are computed using optimized linear |
//| algebra kernels, while Chebyshev and city block distance |
//| functions are computed using simple nested loops with two |
//| branches at each iteration. |
//+------------------------------------------------------------------+
void CClustering::ClusterizerGetDistances(CMatrixDouble &xy,
int npoints,
int nfeatures,
int disttype,
CMatrixDouble &d)
{
CApBuff buf;
d.Resize(0,0);
//--- check
if(!CAp::Assert(nfeatures>=1,__FUNCTION__": NFeatures<1"))
return;
if(!CAp::Assert(npoints>=0,__FUNCTION__": NPoints<1"))
return;
if(!CAp::Assert(disttype==0 || disttype==1 || disttype==2 || disttype==10 || disttype==11 ||
disttype==12 || disttype==13 || disttype==20 || disttype==21,__FUNCTION__": incorrect DistType"))
return;
if(!CAp::Assert(xy.Rows()>=npoints,__FUNCTION__": Rows(XY)<NPoints"))
return;
if(!CAp::Assert(xy.Cols()>=nfeatures,__FUNCTION__": Cols(XY)<NFeatures"))
return;
if(!CAp::Assert(CApServ::IsFiniteMatrix(xy,npoints,nfeatures),__FUNCTION__": XY contains NAN/INF"))
return;
ClusterizerGetDistancesBuf(buf,xy,npoints,nfeatures,disttype,d);
}
//+------------------------------------------------------------------+
//| Buffered version of ClusterizerGetDistances() which reuses |
//| previously allocated space. |
//+------------------------------------------------------------------+
void CClustering::ClusterizerGetDistancesBuf(CApBuff &buf,
CMatrixDouble &xy,
int npoints,
int nfeatures,
int disttype,
CMatrixDouble &d)
{
//--- create variables
double v=0;
double vv=0;
double vr=0;
//--- check
if(!CAp::Assert(nfeatures>=1,__FUNCTION__": NFeatures<1"))
return;
if(!CAp::Assert(npoints>=0,__FUNCTION__": NPoints<1"))
return;
if(!CAp::Assert(disttype==0 || disttype==1 || disttype==2 || disttype==10 || disttype==11 || disttype==12 ||
disttype==13 || disttype==20 || disttype==21,__FUNCTION__": incorrect DistType"))
return;
if(!CAp::Assert(xy.Rows()>=npoints,__FUNCTION__": Rows(XY)<NPoints"))
return;
if(!CAp::Assert(xy.Cols()>=nfeatures,__FUNCTION__": Cols(XY)<NFeatures"))
return;
if(!CAp::Assert(CApServ::IsFiniteMatrix(xy,npoints,nfeatures),__FUNCTION__": XY contains NAN/INF"))
return;
//--- Quick exit
if(npoints==0)
return;
if(npoints==1)
{
CApServ::RMatrixSetLengthAtLeast(d,1,1);
d.Set(0,0,0);
return;
}
//--- Build distance matrix D.
if(disttype==0 || disttype==1)
{
//--- Chebyshev or city-block distances:
//--- * recursively calculate upper triangle (with main diagonal)
//--- * copy it to the bottom part of the matrix
CApServ::RMatrixSetLengthAtLeast(d,npoints,npoints);
EvaluateDistanceMatrixRec(xy,nfeatures,disttype,d,0,npoints,0,npoints);
CAblas::RMatrixEnforceSymmetricity(d,npoints,true);
return;
}
if(disttype==2)
{
//--- Euclidean distance
//--- NOTE: parallelization is done within RMatrixSYRK
d.Resize(npoints,npoints);
buf.m_rm0=xy;
buf.m_rm0.Resize(npoints,nfeatures);
buf.m_ra1=buf.m_rm0.Mean(0)+0;
for(int i=0; i<npoints; i++)
buf.m_rm0.Row(i,buf.m_rm0[i]-(buf.m_ra1+0));
CAblas::RMatrixSyrk(npoints,nfeatures,1.0,buf.m_rm0,0,0,0,0.0,d,0,0,true);
buf.m_ra0=d.Diag(0)+0;
d.Diag(vector<double>::Zeros(npoints));
for(int i=0; i<npoints; i++)
{
for(int j=i+1; j<npoints; j++)
{
v=MathSqrt(MathMax(buf.m_ra0[i]+buf.m_ra0[j]-2*d.Get(i,j),0.0));
d.Set(i,j,v);
}
}
CAblas::RMatrixEnforceSymmetricity(d,npoints,true);
return;
}
if(disttype==10 || disttype==11)
{
//--- Absolute/nonabsolute Pearson correlation distance
//--- NOTE: parallelization is done within PearsonCorrM, which calls RMatrixSYRK internally
d.Resize(npoints,npoints);
buf.m_rm0=xy;
buf.m_rm0.Resize(npoints,nfeatures);
buf.m_ra0=buf.m_rm0.Mean(1)+0;
for(int i=0; i<npoints; i++)
buf.m_rm0.Row(i,buf.m_rm0[i]-buf.m_ra0[i]);
CAblas::RMatrixSyrk(npoints,nfeatures,1.0,buf.m_rm0,0,0,0,0.0,d,0,0,true);
buf.m_ra0=d.Diag()+0;
d.Diag(vector<double>::Zeros(npoints));
for(int i=0; i<npoints; i++)
{
for(int j=i+1; j<npoints; j++)
{
v=d.Get(i,j)/MathSqrt(buf.m_ra0[i]*buf.m_ra0[j]);
if(disttype==10)
v=1-v;
else
v=1-MathAbs(v);
v=MathMax(v,0.0);
d.Set(i,j,v);
}
}
CAblas::RMatrixEnforceSymmetricity(d,npoints,true);
return;
}
if(disttype==12 || disttype==13)
{
//--- Absolute/nonabsolute uncentered Pearson correlation distance
//--- NOTE: parallelization is done within RMatrixSYRK
d.Resize(npoints,npoints);
CAblas::RMatrixSyrk(npoints,nfeatures,1.0,xy,0,0,0,0.0,d,0,0,true);
buf.m_ra0=d.Diag()+0;
d.Diag(vector<double>::Zeros(npoints));
for(int i=0; i<npoints; i++)
{
for(int j=i+1; j<npoints; j++)
{
v=d.Get(i,j)/MathSqrt(buf.m_ra0[i]*buf.m_ra0[j]);
if(disttype==13)
v=MathAbs(v);
v=MathMin(v,1.0);
d.Set(i,j,1-v);
}
}
CAblas::RMatrixEnforceSymmetricity(d,npoints,true);
return;
}
if(disttype==20 || disttype==21)
{
//--- Spearman rank correlation
//--- NOTE: parallelization of correlation matrix is done within
//--- PearsonCorrM, which calls RMatrixSYRK internally
d.Resize(npoints,npoints);
buf.m_rm0=xy;
buf.m_rm0.Resize(npoints,nfeatures);
CBaseStat::RankDataCentered(buf.m_rm0,npoints,nfeatures);
CAblas::RMatrixSyrk(npoints,nfeatures,1.0,buf.m_rm0,0,0,0,0.0,d,0,0,true);
buf.m_ra0=vector<double>::Zeros(npoints);
for(int i=0; i<npoints; i++)
{
if(d.Get(i,i)>0.0)
buf.m_ra0.Set(i,1.0/MathSqrt(d.Get(i,i)));
}
d.Diag(vector<double>::Zeros(npoints));
for(int i=0; i<npoints; i++)
{
v=buf.m_ra0[i];
for(int j=i+1; j<npoints; j++)
{
vv=d.Get(i,j)*v*buf.m_ra0[j];
if(disttype==20)
vr=1-vv;
else
vr=1-MathAbs(vv);
if(vr<0.0)
vr=0.0;
d.Set(i,j,vr);
}
}
CAblas::RMatrixEnforceSymmetricity(d,npoints,true);
return;
}
if(!CAp::Assert(false))
return;
}
//+------------------------------------------------------------------+
//| This function takes as input clusterization report Rep, desired |
//| clusters count K, and builds top K clusters from hierarchical |
//| clusterization tree. |
//| It returns assignment of points to clusters(array of cluster |
//| indexes). |
//| INPUT PARAMETERS: |
//| Rep - report from ClusterizerRunAHC() performed on XY |
//| K - desired number of clusters, 1 <= K <= NPoints. |
//| K can be zero only when NPoints = 0. |
//| OUTPUT PARAMETERS: |
//| CIdx - array[NPoints], I-th element contains cluster |
//| index(from 0 to K-1) for I-th point of the dataset.|
//| CZ - array[K]. This array allows to convert cluster |
//| indexes returned by this function to indexes used |
//| by Rep.Z. J-th cluster returned by this function |
//| corresponds to CZ[J]-th cluster stored in |
//| Rep.Z/PZ/PM. It is guaranteed that CZ[I] < CZ[I+1].|
//| NOTE: K clusters built by this subroutine are assumed to have no |
//| hierarchy. Although they were obtained by manipulation with|
//| top K nodes of dendrogram(i.e. hierarchical decomposition |
//| of dataset), this function does not return information |
//| about hierarchy. Each of the clusters stand on its own. |
//| NOTE: Cluster indexes returned by this function does not |
//| correspond to indexes returned in Rep.Z/PZ/PM. Either you |
//| work with hierarchical representation of the dataset |
//| (dendrogram), or you work with "flat" representation |
//| returned by this function. Each of representations has its |
//| own clusters indexing system(former uses [0,2*NPoints-2]), |
//| while latter uses [0..K-1]), although it is possible to |
//| perform conversion from one system to another by means of |
//| CZ array, returned by this function, which allows you to |
//| convert indexes stored in CIdx to the numeration system |
//| used by Rep.Z. |
//| NOTE: this subroutine is optimized for moderate values of K. |
//| Say, for K=5 it will perform many times faster than for |
//| K=100. Its worst - case performance is O(N*K), although in |
//| average case it perform better (up to O(N*log(K))). |
//+------------------------------------------------------------------+
void CClustering::ClusterizerGetKClusters(CAHCReport &rep,
int k,
CRowInt &cidx,
CRowInt &cz)
{
//--- create variables
int npoints=rep.m_NPoints;
int i0=0;
int i1=0;
int t=0;
bool presentclusters[];
CRowInt clusterindexes;
CRowInt clustersizes;
CRowInt tmpidx;
cidx.Resize(0);
cz.Resize(0);
//--- check
if(!CAp::Assert(npoints>=0,__FUNCTION__": internal error in Rep integrity"))
return;
if(!CAp::Assert(k>0,__FUNCTION__": K<=0"))
return;
if(!CAp::Assert(k<=npoints,__FUNCTION__": K>NPoints"))
return;
if(!CAp::Assert(npoints==rep.m_NPoints,__FUNCTION__": NPoints<>Rep.NPoints"))
return;
//--- Quick exit
if(npoints==0)
return;
if(npoints==1)
{
int temp[]={0};
cz=temp;
cidx=temp;
return;
}
//--- Replay merges, from top to bottom,
//--- keep track of clusters being present at the moment
ArrayResize(presentclusters,2*npoints-1);
tmpidx.Resize(npoints);
ArrayInitialize(presentclusters,false);
presentclusters[2*npoints-2]=true;
tmpidx.Fill(2*npoints-2);
for(int mergeidx=npoints-2; mergeidx>=npoints-k; mergeidx--)
{
//--- Update information about clusters being present at the moment
presentclusters[npoints+mergeidx]=false;
presentclusters[rep.m_z.Get(mergeidx,0)]=true;
presentclusters[rep.m_z.Get(mergeidx,1)]=true;
//--- Update TmpIdx according to the current state of the dataset
//--- NOTE: TmpIdx contains cluster indexes from [0..2*NPoints-2];
//--- we will convert them to [0..K-1] later.
i0=rep.m_pm.Get(mergeidx,0);
i1=rep.m_pm.Get(mergeidx,1);
t=rep.m_z.Get(mergeidx,0);
tmpidx.Fill(t,i0,i1-i0+1);
i0=rep.m_pm.Get(mergeidx,2);
i1=rep.m_pm.Get(mergeidx,3);
t=rep.m_z.Get(mergeidx,1);
tmpidx.Fill(t,i0,i1-i0+1);
}
//--- Fill CZ - array which allows us to convert cluster indexes
//--- from one system to another.
cz.Resize(k);
clusterindexes.Resize(2*npoints-1);
t=0;
for(int i=0; i<2*npoints-1; i++)
{
if(presentclusters[i])
{
cz.Set(t,i);
clusterindexes.Set(i,t);
t++;
}
}
if(!CAp::Assert(t==k,__FUNCTION__": internal error"))
return;
//--- Convert indexes stored in CIdx
cidx.Resize(npoints);
for(int i=0; i<npoints; i++)
cidx.Set(i,clusterindexes[tmpidx[rep.m_p[i]]]);
}
//+------------------------------------------------------------------+
//| This function accepts AHC report Rep, desired minimum |
//| intercluster distance and returns top clusters from hierarchical |
//| clusterization tree which are separated by distance R or HIGHER. |
//| It returns assignment of points to clusters (array of cluster |
//| indexes). |
//| There is one more function with similar name - |
//| ClusterizerSeparatedByCorr, which returns clusters with |
//| intercluster correlation equal to R or LOWER (note: higher for |
//| distance, lower for correlation). |
//| INPUT PARAMETERS: |
//| Rep - report from ClusterizerRunAHC() performed on XY |
//| R - desired minimum intercluster distance, R >= 0 |
//| OUTPUT PARAMETERS: |
//| K - number of clusters, 1 <= K <= NPoints |
//| CIdx - array[NPoints], I-th element contains cluster |
//| index (from 0 to K-1) for I-th point of the dataset|
//| CZ - array[K]. This array allows to convert cluster |
//| indexes returned by this function to indexes used |
//| by Rep.Z. J-th cluster returned by this function |
//| corresponds to CZ[J]-th cluster stored in |
//| Rep.Z/PZ/PM. It is guaranteed that CZ[I] < CZ[I+1].|
//| NOTE: K clusters built by this subroutine are assumed to have no |
//| hierarchy. Although they were obtained by manipulation with|
//| top K nodes of dendrogram (i.e. hierarchical decomposition |
//| of dataset), this function does not return information |
//| about hierarchy. Each of the clusters stand on its own. |
//| NOTE: Cluster indexes returned by this function does not |
//| correspond to indexes returned in Rep.Z/PZ/PM. Either you |
//| work with hierarchical representation of the dataset |
//| (dendrogram), or you work with "flat" representation |
//| returned by this function. Each of representations has its |
//| own clusters indexing system (former uses [0,2*NPoints-2]),|
//| while latter uses [0..K-1]), although it is possible to |
//| perform conversion from one system to another by means of |
//| CZ array, returned by this function, which allows you to |
//| convert indexes stored in CIdx to the numeration system |
//| used by Rep.Z. |
//| NOTE: this subroutine is optimized for moderate values of K. Say,|
//| for K=5 it will perform many times faster than for K=100. |
//| Its worst - case performance is O(N*K), although in average|
//| case it perform better (up to O(N*log(K))). |
//+------------------------------------------------------------------+
void CClustering::ClusterizerSeparatedByDist(CAHCReport &rep,
double r,
int &k,
CRowInt &cidx,
CRowInt &cz)
{
//--- initialization
k=0;
cidx.Resize(0);
cz.Resize(0);
//--- check
if(!CAp::Assert(MathIsValidNumber(r) && r>=0.0,__FUNCTION__": R is infinite or less than 0"))
return;
k=1;
while(k<rep.m_NPoints && rep.m_mergedist[rep.m_NPoints-1-k]>=r)
k++;
ClusterizerGetKClusters(rep,k,cidx,cz);
}
//+------------------------------------------------------------------+
//| This function accepts AHC report Rep, desired maximum |
//| intercluster correlation and returns top clusters from |
//| hierarchical clusterization tree which are separated by |
//| correlation R or LOWER. |
//| It returns assignment of points to clusters(array of cluster |
//| indexes). |
//| There is one more function with similar name - |
//| ClusterizerSeparatedByDist, which returns clusters with |
//| intercluster distance equal to R or HIGHER (note: higher for |
//| distance, lower for correlation). |
//| INPUT PARAMETERS: |
//| Rep - report from ClusterizerRunAHC() performed on XY |
//| R - desired maximum intercluster correlation, -1<=R<=+1|
//| OUTPUT PARAMETERS: |
//| K - number of clusters, 1 <= K <= NPoints |
//| CIdx - array[NPoints], I-th element contains cluster index|
//| (from 0 to K-1) for I-th point of the dataset. |
//| CZ - array[K]. This array allows to convert cluster |
//| indexes returned by this function to indexes used |
//| by Rep.Z. J-th cluster returned by this function |
//| corresponds to CZ[J]-th cluster stored in |
//| Rep.Z/PZ/PM. It is guaranteed that CZ[I] < CZ[I+1].|
//| NOTE: K clusters built by this subroutine are assumed to have no |
//| hierarchy. Although they were obtained by manipulation with|
//| top K nodes of dendrogram (i.e. hierarchical decomposition |
//| of dataset), this function does not return information |
//| about hierarchy. Each of the clusters stand on its own. |
//| NOTE: Cluster indexes returned by this function does not |
//| correspond to indexes returned in Rep.Z/PZ/PM. Either you |
//| work with hierarchical representation of the dataset |
//| (dendrogram), or you work with "flat" representation |
//| returned by this function. Each of representations has its |
//| own clusters indexing system (former uses [0,2*NPoints-2]),|
//| while latter uses [0..K-1]), although it is possible to |
//| perform conversion from one system to another by means of |
//| CZ array, returned by this function, which allows you to |
//| convert indexes stored in CIdx to the numeration system |
//| used by Rep.Z. |
//| NOTE: this subroutine is optimized for moderate values of K. Say,|
//| for K=5 it will perform many times faster than for K=100. |
//| Its worst - case performance is O(N*K), although in average|
//| case it perform better (up to O(N*log(K))). |
//+------------------------------------------------------------------+
void CClustering::ClusterizerSeparatedByCorr(CAHCReport &rep,
double r,
int &k,
CRowInt &cidx,
CRowInt &cz)
{
k=0;
cidx.Resize(0);
cz.Resize(0);
if(!CAp::Assert(MathIsValidNumber(r) && r>=-1.0 && r<=1.0,__FUNCTION__": R is infinite or less than 0"))
return;
k=1;
while(k<rep.m_NPoints && rep.m_mergedist[rep.m_NPoints-1-k]>=(1.0-r))
k++;
ClusterizerGetKClusters(rep,k,cidx,cz);
}
//+------------------------------------------------------------------+
//| K-means++ clusterization |
//| INPUT PARAMETERS: |
//| XY - dataset, array [0..NPoints-1, 0..NVars-1]. |
//| NPoints - dataset size, NPoints >= K |
//| NVars - number of variables, NVars >= 1 |
//| K - desired number of clusters, K >= 1 |
//| InitAlgo - initialization algorithm: |
//| * 0 - automatic selection of best algorithm |
//| * 1 - random selection of centers |
//| * 2 - k-means++ |
//| * 3 - fast-greedy init |
//| * -1 - first K rows of dataset are used (special |
//| debug algorithm) |
//| Seed - seed value for internal RNG : |
//| * positive value is used to initialize RNG in order|
//| to induce deterministic behavior of algorithm |
//| * zero or negative value means that random seed is |
//| generated |
//| MaxIts - iterations limit or zero for no limit |
//| Restarts - number of restarts, Restarts >= 1 |
//| KMeansDbgNoIts - debug flag; if set, Lloyd's iteration is not |
//| performed, only initialization phase. |
//| Buf - special reusable structure which stores previously |
//| allocated memory, intended to avoid memory |
//| fragmentation when solving multiple subsequent |
//| problems: |
//| * MUST BE INITIALIZED WITH KMeansInitBuffers() |
//| CALL BEFORE FIRST PASS TO THIS FUNCTION! |
//| * subsequent passes must be made without |
//| re-initialization |
//| OUTPUT PARAMETERS: |
//| Info - return code: |
//| * -3, if task is degenerate(number of distinct |
//| points is less than K) |
//| * -1, if incorrect NPoints/NFeatures/K/Restarts was|
//| passed |
//| * 1, if subroutine finished successfully |
//| IterationsCount - actual number of iterations performed by |
//| clusterizer |
//| CCol - array[0..NVars-1, 0..K-1]. matrix whose columns |
//| store cluster's centers |
//| NeedCCol - True in case caller requires to store result in |
//| CCol |
//| CRow - array[0..K-1, 0..NVars-1], same as CCol, but |
//| centers are stored in rows |
//| NeedCRow - True in case caller requires to store result in |
//| CCol |
//| XYC - array[NPoints], which contains cluster indexes |
//| Energy - merit function of clusterization |
//+------------------------------------------------------------------+
void CClustering::KMeansGenerateInternal(CMatrixDouble &xy,
int npoints,
int nvars,
int k,
int initalgo,
int seed,
int maxits,
int restarts,
bool kmeansdbgnoits,
int &info,
int &iterationscount,
CMatrixDouble &ccol,
bool needccol,
CMatrixDouble &crow,
bool needcrow,
CRowInt &xyc,
double &energy,
CKmeansBuffers &buf)
{
//--- create variables
int i1=0;
double e=0;
double eprev=0;
double v=0;
double vv=0;
bool waschanges;
bool zerosizeclusters;
int pass=0;
int itcnt=0;
CHighQualityRandState rs;
info=0;
iterationscount=0;
ccol.Resize(0,0);
crow.Resize(0,0);
xyc.Resize(0);
energy=0;
//--- Test parameters
if(npoints<k || nvars<1 || k<1 || restarts<1)
{
info=-1;
iterationscount=0;
return;
}
//--- TODO: special case K=1
//--- TODO: special case K=NPoints
info=1;
iterationscount=0;
//--- Multiple passes of k-means++ algorithm
if(seed<=0)
CHighQualityRand::HQRndRandomize(rs);
else
CHighQualityRand::HQRndSeed(325355,seed,rs);
xyc.Resize(npoints);
CApServ::RMatrixSetLengthAtLeast(buf.m_ctbest,k,nvars);
CApServ::IVectorSetLengthAtLeast(buf.m_xycbest,npoints);
buf.m_d2.Resize(npoints);
CApServ::IVectorSetLengthAtLeast(buf.m_csizes,k);
energy=CMath::m_maxrealnumber;
for(pass=1; pass<=restarts; pass++)
{
//--- Select initial centers.
//--- Note that for performance reasons centers are stored in ROWS of CT, not
//--- in columns. We'll transpose CT in the end and store it in the C.
//--- Also note that SelectInitialCenters() may return degenerate set of centers
//--- (some of them have no corresponding points in dataset, some are non-distinct).
//--- Algorithm below is robust enough to deal with such set.
SelectInitialCenters(xy,npoints,nvars,initalgo,rs,k,buf.m_ct,buf.m_initbuf);
//--- Lloyd's iteration
if(!kmeansdbgnoits)
{
//--- Perform iteration as usual, in normal mode
xyc.Fill(-1,0,npoints);
eprev=CMath::m_maxrealnumber;
e=CMath::m_maxrealnumber;
itcnt=0;
while(maxits==0 || itcnt<maxits)
{
//--- Update iteration counter
itcnt++;
iterationscount++;
//--- Call KMeansUpdateDistances(), fill XYC with center numbers,
//--- D2 with center distances.
buf.m_xycprev=xyc;
buf.m_xycprev.Resize(npoints);
KMeansUpdateDistances(xy,0,npoints,nvars,buf.m_ct,0,k,xyc,buf.m_d2);
waschanges=false;
for(int i=0; i<npoints; i++)
waschanges=waschanges || xyc[i]!=buf.m_xycprev[i];
//--- Update centers
buf.m_csizes.Fill(0,0,k);
buf.m_ct=matrix<double>::Zeros(k,nvars);
for(int i=0; i<npoints; i++)
{
buf.m_csizes.Add(xyc[i],1);
for(int i_=0; i_<nvars; i_++)
{
buf.m_ct.Add(xyc[i],i_,xy.Get(i,i_));
}
}
zerosizeclusters=false;
for(int j=0; j<k; j++)
{
if(buf.m_csizes[j]!=0)
{
v=(double)1/(double)buf.m_csizes[j];
buf.m_ct.Row(j,buf.m_ct.Row(j)*v);
}
zerosizeclusters=zerosizeclusters || buf.m_csizes[j]==0;
}
if(zerosizeclusters)
{
//--- Some clusters have zero size - rare, but possible.
//--- We'll choose new centers for such clusters using k-means++ rule
//--- and restart algorithm, decrementing iteration counter
//--- in order to allow one more iteration (this one was useless
//--- and should not be counted).
if(!FixCenters(xy,npoints,nvars,buf.m_ct,k,buf.m_initbuf))
{
info=-3;
return;
}
itcnt--;
continue;
}
//--- Stop if one of two conditions is met:
//--- 1. nothing has changed during iteration
//--- 2. energy function increased after recalculation on new centers
e=0;
for(int i=0; i<npoints; i++)
{
v=0.0;
i1=xyc[i];
for(int j=0; j<nvars; j++)
{
vv=xy.Get(i,j)-buf.m_ct.Get(i1,j);
v+=vv*vv;
}
e+=v;
}
if(!waschanges || e>=eprev)
break;
//--- Update EPrev
eprev=e;
}
}
else
{
//--- Debug mode: no Lloyd's iteration.
//--- We just calculate potential E.
KMeansUpdateDistances(xy,0,npoints,nvars,buf.m_ct,0,k,xyc,buf.m_d2);
e=buf.m_d2.Sum();
}
//--- Compare E with best centers found so far
if(e<energy)
{
//--- store partition.
energy=e;
CBlas::CopyMatrix(buf.m_ct,0,k-1,0,nvars-1,buf.m_ctbest,0,k-1,0,nvars-1);
buf.m_xycbest=xyc;
}
}
//--- Copy and transpose
if(needccol)
{
ccol.Resize(nvars,k);
CBlas::CopyAndTranspose(buf.m_ctbest,0,k-1,0,nvars-1,ccol,0,nvars-1,0,k-1);
}
if(needcrow)
{
crow.Resize(k,nvars);
CAblas::RMatrixCopy(k,nvars,buf.m_ctbest,0,0,crow,0,0);
}
xyc=buf.m_xycbest;
}
//+------------------------------------------------------------------+
//| This procedure recalculates distances from points to centers and |
//| assigns each point to closest center. |
//| INPUT PARAMETERS: |
//| XY - dataset, array [0..NPoints-1, 0..NVars-1]. |
//| Idx0, Idx1 - define range of dataset [Idx0, Idx1) to process;|
//| right boundary is not included. |
//| NVars - number of variables, NVars >= 1 |
//| CT - matrix of centers, centers are stored in rows |
//| CIdx0, CIdx1 - define range of centers [CIdx0, CIdx1) to |
//| process; right boundary is not included. |
//| XYC - preallocated output buffer, |
//| XYDist2 - preallocated output buffer |
//| Tmp - temporary buffer, automatically reallocated if |
//| needed |
//| OUTPUT PARAMETERS: |
//| XYC - new assignment of points to centers are stored |
//| in [Idx0, Idx1) |
//| XYDist2 - squared distances from points to their centers are |
//| stored in [Idx0, Idx1) |
//+------------------------------------------------------------------+
void CClustering::KMeansUpdateDistances(CMatrixDouble &xy,
int idx0,int idx1,int nvars,
CMatrixDouble &ct,int cidx0,
int cidx1,CRowInt &xyc,
CRowDouble &xydist2)
{
//--- create variables
int i0=0;
int i1=0;
int cclosest=0;
double dclosest=0;
double vv=0;
CApBuff buf;
double rcomplexity=0;
int task0=0;
int task1=0;
int pblkcnt=0;
int cblkcnt=0;
int vblkcnt=0;
int p0=0;
int p1=0;
int c0=0;
int c1=0;
int v0=0;
int v1=0;
double v00=0;
double v01=0;
double v10=0;
double v11=0;
double vp0=0;
double vp1=0;
double vc0=0;
double vc1=0;
int pcnt=0;
int pcntpadded=0;
int ccnt=0;
int ccntpadded=0;
int offs0=0;
int offs00=0;
int offs01=0;
int offs10=0;
int offs11=0;
int vcnt=0;
int stride=0;
//--- Quick exit for special cases
if(idx1<=idx0)
return;
if(cidx1<=cidx0)
return;
if(nvars<=0)
return;
//--- Dataset chunk is selected.
//--- Process it with blocked algorithm:
//--- * iterate over points, process them in KMeansBlockSize-ed chunks
//--- * for each chunk of dataset, iterate over centers, process them in KMeansBlockSize-ed chunks
//--- * for each chunk of dataset/centerset, iterate over variables, process them in KMeansBlockSize-ed chunks
if(!CAp::Assert(m_kmeansblocksize%2==0,__FUNCTION__": internal error"))
return;
CApServ::RVectorSetLengthAtLeast(buf.m_ra0,m_kmeansblocksize*m_kmeansblocksize);
CApServ::RVectorSetLengthAtLeast(buf.m_ra1,m_kmeansblocksize*m_kmeansblocksize);
CApServ::RVectorSetLengthAtLeast(buf.m_ra2,m_kmeansblocksize*m_kmeansblocksize);
CApServ::RVectorSetLengthAtLeast(buf.m_ra3,m_kmeansblocksize);
CApServ::IVectorSetLengthAtLeast(buf.m_ia3,m_kmeansblocksize);
pblkcnt=CApServ::ChunksCount(idx1-idx0,m_kmeansblocksize);
cblkcnt=CApServ::ChunksCount(cidx1-cidx0,m_kmeansblocksize);
vblkcnt=CApServ::ChunksCount(nvars,m_kmeansblocksize);
for(int pblk=0; pblk<pblkcnt; pblk++)
{
//--- Process PBlk-th chunk of dataset.
p0=idx0+pblk*m_kmeansblocksize;
p1=MathMin(p0+m_kmeansblocksize,idx1);
//--- Prepare RA3[]/IA3[] for storage of best distances and best cluster numbers.
buf.m_ra3=vector<double>::Full(m_kmeansblocksize,CMath::m_maxrealnumber);
buf.m_ia3.Fill(-1,0,m_kmeansblocksize);
//--- Iterare over chunks of centerset.
for(int cblk=0; cblk<cblkcnt; cblk++)
{
//--- Process CBlk-th chunk of centerset
c0=cidx0+cblk*m_kmeansblocksize;
c1=MathMin(c0+m_kmeansblocksize,cidx1);
//--- At this point we have to calculate a set of pairwise distances
//--- between points [P0,P1) and centers [C0,C1) and select best center
//--- for each point. It can also be done with blocked algorithm
//--- (blocking for variables).
//--- Following arrays are used:
//--- * RA0[] - matrix of distances, padded by zeros for even size,
//--- rows are stored with stride KMeansBlockSize.
//--- * RA1[] - matrix of points (variables corresponding to current
//--- block are extracted), padded by zeros for even size,
//--- rows are stored with stride KMeansBlockSize.
//--- * RA2[] - matrix of centers (variables corresponding to current
//--- block are extracted), padded by zeros for even size,
//--- rows are stored with stride KMeansBlockSize.
pcnt=p1-p0;
pcntpadded=pcnt+pcnt%2;
ccnt=c1-c0;
ccntpadded=ccnt+ccnt%2;
stride=m_kmeansblocksize;
if(!CAp::Assert(pcntpadded<=m_kmeansblocksize,__FUNCTION__": integrity error"))
return;
if(!CAp::Assert(ccntpadded<=m_kmeansblocksize,__FUNCTION__": integrity error"))
return;
for(int i=0; i<pcntpadded; i++)
for(int j=0; j<ccntpadded; j++)
buf.m_ra0.Set(i*stride+j,0.0);
for(int vblk=0; vblk<vblkcnt; vblk++)
{
//--- Fetch VBlk-th block of variables to arrays RA1 (points) and RA2 (centers).
//--- Pad points and centers with zeros.
v0=vblk*m_kmeansblocksize;
v1=MathMin(v0+m_kmeansblocksize,nvars);
vcnt=v1-v0;
for(int i=0; i<pcnt; i++)
for(int j=0; j<vcnt; j++)
buf.m_ra1.Set(i*stride+j,xy.Get(p0+i,v0+j));
for(int i=pcnt; i<pcntpadded; i++)
for(int j=0; j<vcnt; j++)
buf.m_ra1.Set(i*stride+j,0.0);
for(int i=0; i<ccnt; i++)
for(int j=0; j<vcnt; j++)
buf.m_ra2.Set(i*stride+j,ct.Get(c0+i,v0+j));
for(int i=ccnt; i<ccntpadded; i++)
for(int j=0; j<vcnt; j++)
buf.m_ra2.Set(i*stride+j,0.0);
//--- Update distance matrix with sums-of-squared-differences of RA1 and RA2
i0=0;
while(i0<pcntpadded)
{
i1=0;
while(i1<ccntpadded)
{
offs0=i0*stride+i1;
v00=buf.m_ra0[offs0];
v01=buf.m_ra0[offs0+1];
v10=buf.m_ra0[offs0+stride];
v11=buf.m_ra0[offs0+stride+1];
offs00=i0*stride;
offs01=offs00+stride;
offs10=i1*stride;
offs11=offs10+stride;
for(int j=0; j<vcnt; j++)
{
vp0=buf.m_ra1[offs00+j];
vp1=buf.m_ra1[offs01+j];
vc0=buf.m_ra2[offs10+j];
vc1=buf.m_ra2[offs11+j];
vv=vp0-vc0;
v00+=vv*vv;
vv=vp0-vc1;
v01+=vv*vv;
vv=vp1-vc0;
v10+=vv*vv;
vv=vp1-vc1;
v11+=vv*vv;
}
offs0=i0*stride+i1;
buf.m_ra0.Set(offs0,v00);
buf.m_ra0.Set(offs0+1,v01);
buf.m_ra0.Set(offs0+stride,v10);
buf.m_ra0.Set(offs0+stride+1,v11);
i1+=2;
}
i0+=2;
}
}
for(int i=0; i<pcnt; i++)
{
cclosest=buf.m_ia3[i];
dclosest=buf.m_ra3[i];
for(int j=0; j<ccnt; j++)
if(buf.m_ra0[i*stride+j]<dclosest)
{
dclosest=buf.m_ra0[i*stride+j];
cclosest=c0+j;
}
buf.m_ia3.Set(i,cclosest);
buf.m_ra3.Set(i,dclosest);
}
}
//--- Store best centers to XYC[]
for(int i=p0; i<p1; i++)
{
xyc.Set(i,buf.m_ia3[i-p0]);
xydist2.Set(i,buf.m_ra3[i-p0]);
}
}
}
//+------------------------------------------------------------------+
//| This function selects initial centers according to specified |
//| initialization algorithm. |
//| IMPORTANT: this function provides no guarantees regarding |
//| selection of DIFFERENT centers. Centers returned by |
//| this function may include duplicates(say, when random |
//| sampling is used). It is also possible that some |
//| centers are empty. Algorithm which uses this function |
//| must be able to deal with it. Say, you may want to use|
//| FixCenters() in order to fix empty centers. |
//| INPUT PARAMETERS: |
//| XY - dataset, array [0..NPoints-1, 0..NVars-1]. |
//| NPoints - points count |
//| NVars - number of variables, NVars >= 1 |
//| InitAlgo - initialization algorithm: |
//| * 0 - automatic selection of best algorithm |
//| * 1 - random selection |
//| * 2 - k-means++ |
//| * 3 - fast-greedy init |
//| * -1 - first K rows of dataset are used (debug |
//| algorithm) |
//| RS - RNG used to select centers |
//| K - number of centers, K >= 1 |
//| CT - possibly preallocated output buffer, resized if |
//| needed |
//| InitBuf - internal buffer, possibly unitialized instance of |
//| APBuffers. It is recommended to use this instance |
//| only with SelectInitialCenters() and FixCenters() |
//| functions, because these functions may allocate |
//| really large storage. |
//| OUTPUT PARAMETERS: |
//| CT - set of K clusters, one per row |
//| RESULT: True on success, False on failure(impossible to create K |
//| independent clusters) |
//+------------------------------------------------------------------+
void CClustering::SelectInitialCenters(CMatrixDouble &xy,int npoints,
int nvars,int initalgo,CHighQualityRandState &rs,
int k,CMatrixDouble &ct,CApBuff &initbuf)
{
//--- create variables
int cidx=0;
int i=0;
int j=0;
double v=0;
double vv=0;
double s=0;
int lastnz=0;
int ptidx=0;
int samplesize=0;
int samplescntnew=0;
int samplescntall=0;
double samplescale=0;
int i_=0;
//--- Check parameters
if(!CAp::Assert(npoints>0,__FUNCTION__": internal error"))
return;
if(!CAp::Assert(nvars>0,__FUNCTION__": internal error"))
return;
if(!CAp::Assert(k>0,__FUNCTION__": internal error"))
return;
if(initalgo==0)
initalgo=3;
CApServ::RMatrixSetLengthAtLeast(ct,k,nvars);
//--- Random initialization
if(initalgo==-1)
{
for(i=0; i<k; i++)
ct.Row(i,xy[i%npoints]+0);
return;
}
//--- Random initialization
if(initalgo==1)
{
for(i=0; i<k; i++)
{
j=CHighQualityRand::HQRndUniformI(rs,npoints);
ct.Row(i,xy[j]+0);
}
return;
}
//--- k-means++ initialization
if(initalgo==2)
{
//--- Prepare distances array.
//--- Select initial center at random.
initbuf.m_ra0=vector<double>::Full(npoints,CMath::m_maxrealnumber);
ptidx=CHighQualityRand::HQRndUniformI(rs,npoints);
ct.Row(0,xy[ptidx]+0);
//--- For each newly added center repeat:
//--- * reevaluate distances from points to best centers
//--- * sample points with probability dependent on distance
//--- * add new center
for(cidx=0; cidx<k-1; cidx++)
{
//--- Reevaluate distances
s=0.0;
for(i=0; i<npoints; i++)
{
v=0.0;
for(j=0; j<=nvars-1; j++)
{
vv=xy.Get(i,j)-ct.Get(cidx,j);
v+=vv*vv;
}
if(v<initbuf.m_ra0[i])
initbuf.m_ra0.Set(i,v);
s+=initbuf.m_ra0[i];
}
//
//--- If all distances are zero, it means that we can not find enough
//--- distinct points. In this case we just select non-distinct center
//--- at random and continue iterations. This issue will be handled
//--- later in the FixCenters() function.
//
if(s==0.0)
{
ptidx=CHighQualityRand::HQRndUniformI(rs,npoints);
ct.Row(cidx+1,xy[ptidx]+0);
continue;
}
//--- Select point as center using its distance.
//--- We also handle situation when because of rounding errors
//--- no point was selected - in this case, last non-zero one
//--- will be used.
v=CHighQualityRand::HQRndUniformR(rs);
vv=0.0;
lastnz=-1;
ptidx=-1;
for(i=0; i<npoints; i++)
{
if(initbuf.m_ra0[i]==0.0)
continue;
lastnz=i;
vv+=initbuf.m_ra0[i];
if(v<=vv/s)
{
ptidx=i;
break;
}
}
if(!CAp::Assert(lastnz>=0,__FUNCTION__": integrity error"))
return;
if(ptidx<0)
ptidx=lastnz;
ct.Row(cidx+1,xy[ptidx]+0);
}
return;
}
//--- "Fast-greedy" algorithm based on "Scalable k-means++".
//--- We perform several rounds, within each round we sample about 0.5*K points
//--- (not exactly 0.5*K) until we have 2*K points sampled. Before each round
//--- we calculate distances from dataset points to closest points sampled so far.
//--- We sample dataset points independently using distance xtimes 0.5*K divided by total
//--- as probability (similar to k-means++, but each point is sampled independently;
//--- after each round we have roughtly 0.5*K points added to sample).
//--- After sampling is done, we run "greedy" version of k-means++ on this subsample
//--- which selects most distant point on every round.
if(initalgo==3)
{
//--- Prepare arrays.
//--- Select initial center at random, add it to "new" part of sample,
//--- which is stored at the beginning of the array
samplesize=2*k;
samplescale=0.5*k;
CApServ::RMatrixSetLengthAtLeast(initbuf.m_rm0,samplesize,nvars);
ptidx=CHighQualityRand::HQRndUniformI(rs,npoints);
initbuf.m_rm0.Row(0,xy[ptidx]+0);
samplescntnew=1;
samplescntall=1;
initbuf.m_ra1=vector<double>::Zeros(npoints);
CApServ::IVectorSetLengthAtLeast(initbuf.m_ia1,npoints);
initbuf.m_ra0=vector<double>::Full(npoints,CMath::m_maxrealnumber);
//--- Repeat until samples count is 2*K
while(samplescntall<samplesize)
{
//--- Evaluate distances from points to NEW centers, store to RA1.
//--- Reset counter of "new" centers.
KMeansUpdateDistances(xy,0,npoints,nvars,initbuf.m_rm0,samplescntall-samplescntnew,samplescntall,initbuf.m_ia1,initbuf.m_ra1);
samplescntnew=0;
//--- Merge new distances with old ones.
//--- Calculate sum of distances, if sum is exactly zero - fill sample
//--- by randomly selected points and terminate.
s=0.0;
for(i=0; i<npoints; i++)
{
initbuf.m_ra0.Set(i,MathMin(initbuf.m_ra0[i],initbuf.m_ra1[i]));
s+=initbuf.m_ra0[i];
}
if(s==0.0)
{
while(samplescntall<samplesize)
{
ptidx=CHighQualityRand::HQRndUniformI(rs,npoints);
initbuf.m_rm0.Row(samplescntall,xy[ptidx]+0);
samplescntall++;
samplescntnew++;
}
break;
}
//--- Sample points independently.
for(i=0; i<npoints; i++)
{
if(samplescntall==samplesize)
break;
if(initbuf.m_ra0[i]==0.0)
continue;
if(CHighQualityRand::HQRndUniformR(rs)<=(samplescale*initbuf.m_ra0[i]/s))
{
initbuf.m_rm0.Row(samplescntall,xy[i]+0);
samplescntall++;
samplescntnew++;
}
}
}
//--- Run greedy version of k-means on sampled points
initbuf.m_ra0=vector<double>::Full(samplescntall,CMath::m_maxrealnumber);
ptidx=CHighQualityRand::HQRndUniformI(rs,samplescntall);
ct.Row(0,initbuf.m_rm0[ptidx]+0);
for(cidx=0; cidx<k-1; cidx++)
{
//--- Reevaluate distances
for(i=0; i<samplescntall; i++)
{
v=0.0;
for(j=0; j<nvars; j++)
{
vv=initbuf.m_rm0.Get(i,j)-ct.Get(cidx,j);
v+=vv*vv;
}
if(v<initbuf.m_ra0[i])
initbuf.m_ra0.Set(i,v);
}
//--- Select point as center in greedy manner - most distant
//--- point is selected.
ptidx=0;
for(i=0; i<samplescntall; i++)
{
if(initbuf.m_ra0[i]>initbuf.m_ra0[ptidx])
ptidx=i;
}
ct.Row(cidx+1,initbuf.m_rm0[ptidx]+0);
}
return;
}
//--- Internal error
CAp::Assert(false,__FUNCTION__": internal error");
}
//+------------------------------------------------------------------+
//| This function "fixes" centers, i.e. replaces ones which have no |
//| neighbor points by new centers which have at least one neighbor. |
//| If it is impossible to fix centers(not enough distinct points in |
//| the dataset), this function returns False. |
//| INPUT PARAMETERS: |
//| XY - dataset, array [0..NPoints-1, 0..NVars-1]. |
//| NPoints - points count, >= 1 |
//| NVars - number of variables, NVars >= 1 |
//| CT - centers |
//| K - number of centers, K >= 1 |
//| InitBuf - internal buffer, possibly unitialized instance of |
//| APBuffers. It is recommended to use this instance |
//| only with SelectInitialCenters() and FixCenters() |
//| functions, because these functions may allocate |
//| really large storage. |
//| UpdatePool - shared pool seeded with instance of APBuffers |
//| structure (seed instance can be unitialized). Used |
//| internally with KMeansUpdateDistances() function. |
//| It is recommended to use this pool ONLY with |
//| KMeansUpdateDistances() function. |
//| OUTPUT PARAMETERS: |
//| CT - set of K centers, one per row |
//| RESULT: True on success, False on failure(impossible to create K |
//| independent clusters) |
//+------------------------------------------------------------------+
bool CClustering::FixCenters(CMatrixDouble &xy,
int npoints,
int nvars,
CMatrixDouble &ct,
int k,
CApBuff &initbuf)
{
//--- create variables
int fixiteration=0;
int centertofix=0;
int pdistant=0;
double ddistant=0;
double v=0;
//--- check
if(!CAp::Assert(npoints>=1,__FUNCTION__": internal error"))
return(false);
if(!CAp::Assert(nvars>=1,__FUNCTION__": internal error"))
return(false);
if(!CAp::Assert(k>=1,__FUNCTION__": internal error"))
return(false);
//--- Calculate distances from points to best centers (RA0)
//--- and best center indexes (IA0)
CApServ::IVectorSetLengthAtLeast(initbuf.m_ia0,npoints);
CApServ::RVectorSetLengthAtLeast(initbuf.m_ra0,npoints);
KMeansUpdateDistances(xy,0,npoints,nvars,ct,0,k,initbuf.m_ia0,initbuf.m_ra0);
//--- Repeat loop:
//--- * find first center which has no corresponding point
//--- * set it to the most distant (from the rest of the centerset) point
//--- * recalculate distances, update IA0/RA0
//--- * repeat
//--- Loop is repeated for at most 2*K iterations. It is stopped once we have
//--- no "empty" clusters.
CApServ::BVectorSetLengthAtLeast(initbuf.m_ba,k);
for(fixiteration=0; fixiteration<=2*k; fixiteration++)
{
//--- Select center to fix (one which is not mentioned in IA0),
//--- terminate if there is no such center.
//--- BA0[] stores True for centers which have at least one point.
ArrayFill(initbuf.m_ba,0,k,false);
for(int i=0; i<npoints; i++)
initbuf.m_ba[initbuf.m_ia0[i]]=true;
centertofix=-1;
for(int i=0; i<k; i++)
if(!initbuf.m_ba[i])
{
centertofix=i;
break;
}
if(centertofix<0)
return(true);
//--- Replace center to fix by the most distant point.
//--- Update IA0/RA0
pdistant=0;
ddistant=initbuf.m_ra0[pdistant];
for(int i=0; i<npoints; i++)
if(initbuf.m_ra0[i]>ddistant)
{
ddistant=initbuf.m_ra0[i];
pdistant=i;
}
if(ddistant==0.0)
break;
ct.Row(centertofix,xy[pdistant]+0);
for(int i=0; i<npoints; i++)
{
v=0.0;
for(int j=0; j<nvars; j++)
v=v+CMath::Sqr(xy.Get(i,j)-ct.Get(centertofix,j));
if(v<initbuf.m_ra0[i])
{
initbuf.m_ra0.Set(i,v);
initbuf.m_ia0.Set(i,centertofix);
}
}
}
//--- return result
return(false);
}
//+------------------------------------------------------------------+
//| This function performs agglomerative hierarchical clustering |
//| using precomputed distance matrix. Internal function, should not |
//| be called directly. |
//| INPUT PARAMETERS: |
//| S - clusterizer state, initialized by |
//| ClusterizerCreate() |
//| D - distance matrix, array[S.m_nfeatures,S.m_nfeatures]|
//| Contents of the matrix is destroyed during |
//| algorithm operation. |
//| OUTPUT PARAMETERS: |
//| Rep - clustering results; see description of AHCReport |
//| structure for more information. |
//+------------------------------------------------------------------+
void CClustering::ClusterizerRunAHCInternal(CClusterizerState &s,
CMatrixDouble &d,
CAHCReport &rep)
{
//--- create variables
int i=0;
int j=0;
int k=0;
double v=0;
int mergeidx=0;
int c0=0;
int c1=0;
int s0=0;
int s1=0;
int ar=0;
int br=0;
CRowInt cidx;
CRowInt csizes;
CRowInt nnidx;
CMatrixInt cinfo;
int n0=0;
int n1=0;
int ni=0;
double d01=0;
int npoints=s.m_NPoints;
//--- Fill Rep.NPoints, quick exit when NPoints<=1
rep.m_NPoints=npoints;
//--- Quick exit
if(npoints==0)
{
rep.m_p.Resize(0);
rep.m_z.Resize(0,0);
rep.m_pz.Resize(0,0);
rep.m_pm.Resize(0,0);
rep.m_mergedist.Resize(0);
rep.m_terminationtype=1;
return;
}
if(npoints==1)
{
rep.m_p.Resize(1);
rep.m_z.Resize(0,0);
rep.m_pz.Resize(0,0);
rep.m_pm.Resize(0,0);
rep.m_mergedist.Resize(0);
rep.m_p.Set(0,0);
rep.m_terminationtype=1;
return;
}
//--- Allocate
rep.m_z.Resize(npoints-1,2);
rep.m_mergedist.Resize(npoints-1);
rep.m_terminationtype=1;
//--- Build list of nearest neighbors
nnidx.Resize(npoints);
for(i=0; i<npoints; i++)
{
//--- Calculate index of the nearest neighbor
k=-1;
v=CMath::m_maxrealnumber;
for(j=0; j<npoints; j++)
if(j!=i && d.Get(i,j)<v)
{
k=j;
v=d.Get(i,j);
}
if(!CAp::Assert(v<CMath::m_maxrealnumber,__FUNCTION__": internal error"))
return;
nnidx.Set(i,k);
}
//--- For AHCAlgo=4 (Ward's method) replace distances by their squares times 0.5
if(s.m_ahcalgo==4)
{
d.Resize(npoints,npoints);
d=MathPow(d.ToMatrix()+0,2.0)*0.5;
}
//--- Distance matrix is built, perform merges.
//--- NOTE 1: CIdx is array[NPoints] which maps rows/columns of the
//--- distance matrix D to indexes of clusters. Values of CIdx
//--- from [0,NPoints) denote single-point clusters, and values
//--- from [NPoints,2*NPoints-1) denote ones obtained by merging
//--- smaller clusters. Negative calues correspond to absent clusters.
//--- Initially it contains [0...NPoints-1], after each merge
//--- one element of CIdx (one with index C0) is replaced by
//--- NPoints+MergeIdx, and another one with index C1 is
//--- rewritten by -1.
//--- NOTE 2: CSizes is array[NPoints] which stores sizes of clusters.
cidx.Resize(npoints);
csizes.Resize(npoints);
for(i=0; i<npoints; i++)
{
cidx.Set(i,i);
csizes.Set(i,1);
}
for(mergeidx=0; mergeidx<npoints-1; mergeidx++)
{
//--- Select pair of clusters (C0,C1) with CIdx[C0]<CIdx[C1] to merge.
c0=-1;
c1=-1;
d01=CMath::m_maxrealnumber;
for(i=0; i<npoints; i++)
if(cidx[i]>=0)
{
if(d.Get(i,nnidx[i])<d01)
{
c0=i;
c1=nnidx[i];
d01=d.Get(i,nnidx[i]);
}
}
if(!CAp::Assert(d01<CMath::m_maxrealnumber,__FUNCTION__": internal error"))
return;
if(cidx[c0]>cidx[c1])
{
i=c1;
c1=c0;
c0=i;
}
//--- Fill one row of Rep.Z and one element of Rep.MergeDist
rep.m_z.Set(mergeidx,0,cidx[c0]);
rep.m_z.Set(mergeidx,1,cidx[c1]);
rep.m_mergedist.Set(mergeidx,d01);
//--- Update distance matrix:
//--- * row/column C0 are updated by distances to the new cluster
//--- * row/column C1 are considered empty (we can fill them by zeros,
//--- but do not want to spend time - we just ignore them)
//--- NOTE: it is important to update distance matrix BEFORE CIdx/CSizes
//--- are updated.
if(!CAp::Assert(s.m_ahcalgo==0 || s.m_ahcalgo==1 || s.m_ahcalgo==2 || s.m_ahcalgo==3 || s.m_ahcalgo==4,
__FUNCTION__": internal error"))
return;
for(i=0; i<npoints; i++)
{
if(i!=c0 && i!=c1)
{
n0=csizes[c0];
n1=csizes[c1];
ni=csizes[i];
switch(s.m_ahcalgo)
{
case 0:
d.Set(i,c0,MathMax(d.Get(i,c0),d.Get(i,c1)));
break;
case 1:
d.Set(i,c0,MathMin(d.Get(i,c0),d.Get(i,c1)));
break;
case 2:
d.Set(i,c0,(csizes[c0]*d.Get(i,c0)+csizes[c1]*d.Get(i,c1))/(csizes[c0]+csizes[c1]));
break;
case 3:
d.Set(i,c0,(d.Get(i,c0)+d.Get(i,c1))/2);
break;
case 4:
d.Set(i,c0,((n0+ni)*d.Get(i,c0)+(n1+ni)*d.Get(i,c1)-ni*d01)/(n0+n1+ni));
break;
}
d.Set(c0,i,d.Get(i,c0));
}
}
//--- Update CIdx and CSizes
cidx.Set(c0,npoints+mergeidx);
cidx.Set(c1,-1);
csizes.Set(c0,csizes[c0]+csizes[c1]);
csizes.Set(c1,0);
//--- Update nearest neighbors array:
//--- * update nearest neighbors of everything except for C0/C1
//--- * update neighbors of C0/C1
for(i=0; i<npoints; i++)
{
if(cidx[i]>=0 && i!=c0 && (nnidx[i]==c0 || nnidx[i]==c1))
{
//--- I-th cluster which is distinct from C0/C1 has former C0/C1 cluster as its nearest
//--- neighbor. We handle this issue depending on specific AHC algorithm being used.
if(s.m_ahcalgo==1)
{
//--- Single linkage. Merging of two clusters together
//--- does NOT change distances between new cluster and
//--- other clusters.
//
//--- The only thing we have to do is to update nearest neighbor index
nnidx.Set(i,c0);
}
else
{
//--- Something other than single linkage. We have to re-examine
//--- all the row to find nearest neighbor.
k=-1;
v=CMath::m_maxrealnumber;
for(j=0; j<npoints; j++)
{
if(cidx[j]>=0 && j!=i && d.Get(i,j)<v)
{
k=j;
v=d.Get(i,j);
}
}
if(!CAp::Assert(v<CMath::m_maxrealnumber || mergeidx==npoints-2,__FUNCTION__": internal error"))
return;
nnidx.Set(i,k);
}
}
}
k=-1;
v=CMath::m_maxrealnumber;
for(j=0; j<npoints; j++)
if(cidx[j]>=0 && j!=c0 && d.Get(c0,j)<v)
{
k=j;
v=d.Get(c0,j);
}
if(!CAp::Assert(v<CMath::m_maxrealnumber || mergeidx==npoints-2,__FUNCTION__": internal error"))
return;
nnidx.Set(c0,k);
}
//--- Calculate Rep.P and Rep.PM.
//--- In order to do that, we fill CInfo matrix - (2*NPoints-1)*3 matrix,
//--- with I-th row containing:
//--- * CInfo[I,0] - size of I-th cluster
//--- * CInfo[I,1] - beginning of I-th cluster
//--- * CInfo[I,2] - end of I-th cluster
//--- * CInfo[I,3] - height of I-th cluster
//--- We perform it as follows:
//--- * first NPoints clusters have unit size (CInfo[I,0]=1) and zero
//--- height (CInfo[I,3]=0)
//--- * we replay NPoints-1 merges from first to last and fill sizes of
//--- corresponding clusters (new size is a sum of sizes of clusters
//--- being merged) and height (new height is max(heights)+1).
//--- * now we ready to determine locations of clusters. Last cluster
//--- spans entire dataset, we know it. We replay merges from last to
//--- first, during each merge we already know location of the merge
//--- result, and we can position first cluster to the left part of
//--- the result, and second cluster to the right part.
rep.m_p.Resize(npoints);
rep.m_pm.Resize(npoints-1,6);
cinfo.Resize(2*npoints-1,4);
for(i=0; i<npoints; i++)
{
cinfo.Set(i,0,1);
cinfo.Set(i,3,0);
}
for(i=0; i<npoints-1; i++)
{
cinfo.Set(npoints+i,0,cinfo.Get(rep.m_z.Get(i,0),0)+cinfo.Get(rep.m_z.Get(i,1),0));
cinfo.Set(npoints+i,3,MathMax(cinfo.Get(rep.m_z.Get(i,0),3),cinfo.Get(rep.m_z.Get(i,1),3))+1);
}
cinfo.Set(2*npoints-2,1,0);
cinfo.Set(2*npoints-2,2,npoints-1);
for(i=npoints-2; i>=0; i--)
{
//--- We merge C0 which spans [A0,B0] and C1 (spans [A1,B1]),
//--- with unknown A0, B0, A1, B1. However, we know that result
//--- is CR, which spans [AR,BR] with known AR/BR, and we know
//--- sizes of C0, C1, CR (denotes as S0, S1, SR).
c0=rep.m_z.Get(i,0);
c1=rep.m_z.Get(i,1);
s0=cinfo.Get(c0,0);
s1=cinfo.Get(c1,0);
ar=cinfo.Get(npoints+i,1);
br=cinfo.Get(npoints+i,2);
cinfo.Set(c0,1,ar);
cinfo.Set(c0,2,ar+s0-1);
cinfo.Set(c1,1,br-(s1-1));
cinfo.Set(c1,2,br);
rep.m_pm.Set(i,0,cinfo.Get(c0,1));
rep.m_pm.Set(i,1,cinfo.Get(c0,2));
rep.m_pm.Set(i,2,cinfo.Get(c1,1));
rep.m_pm.Set(i,3,cinfo.Get(c1,2));
rep.m_pm.Set(i,4,cinfo.Get(c0,3));
rep.m_pm.Set(i,5,cinfo.Get(c1,3));
}
for(i=0; i<npoints; i++)
{
if(!CAp::Assert(cinfo.Get(i,1)==cinfo.Get(i,2)))
return;
rep.m_p.Set(i,cinfo.Get(i,1));
}
//--- Calculate Rep.PZ
rep.m_pz.Resize(npoints-1,2);
for(i=0; i<npoints-1; i++)
{
rep.m_pz.Set(i,0,rep.m_z.Get(i,0));
rep.m_pz.Set(i,1,rep.m_z.Get(i,1));
if(rep.m_pz.Get(i,0)<npoints)
rep.m_pz.Set(i,0,rep.m_p[rep.m_pz.Get(i,0)]);
if(rep.m_pz.Get(i,1)<npoints)
rep.m_pz.Set(i,1,rep.m_p[rep.m_pz.Get(i,1)]);
}
}
//+------------------------------------------------------------------+
//| This function recursively evaluates distance matrix for |
//| SOME(not all!) distance types. |
//| INPUT PARAMETERS: |
//| XY - array[?, NFeatures], dataset |
//| NFeatures - number of features, >= 1 |
//| DistType - distance function: |
//| * 0 Chebyshev distance(L - inf norm) |
//| * 1 city block distance(L1 norm) |
//| D - preallocated output matrix |
//| I0, I1 - half interval of rows to calculate: [I0, I1) is |
//| processed |
//| J0, J1 - half interval of cols to calculate: [J0, J1) is |
//| processed |
//| OUTPUT PARAMETERS: |
//| D - array[NPoints, NPoints], distance matrix upper |
//| triangle and main diagonal are initialized with |
//| data. |
//| NOTE: intersection of [I0, I1) and [J0, J1) may completely lie |
//| in upper triangle, only partially intersect with it, or |
//| have zero intersection. In any case, only intersection of |
//| submatrix given by [I0, I1)*[J0, J1) with upper triangle |
//| of the matrix is evaluated. |
//| Say, for 4x4 distance matrix A: |
//| * [0, 2)*[0, 2) will result in evaluation of A00, A01, A11 |
//| * [2, 4)*[2, 4) will result in evaluation of A22, A23, A32, A33|
//| * [2, 4)*[0, 2) will result in evaluation of empty set of |
//| elements |
//+------------------------------------------------------------------+
void CClustering::EvaluateDistanceMatrixRec(CMatrixDouble &xy,
int nfeatures,
int disttype,
CMatrixDouble &d,
int i0,
int i1,
int j0,
int j1)
{
//--- create variables
int len0=0;
int len1=0;
double v=0;
double vv=0;
//--- check
if(!CAp::Assert(disttype==0 || disttype==1,__FUNCTION__": incorrect DistType"))
return;
//--- Normalize J0/J1:
//--- * J0:=max(J0,I0) - we ignore lower triangle
//--- * J1:=max(J1,J0) - normalize J1
j0=MathMax(j0,i0);
j1=MathMax(j1,j0);
if(j1<=j0 || i1<=i0)
return;
//--- Sequential processing
for(int i=i0; i<i1; i++)
{
for(int j=j0; j<j1; j++)
{
if(j>=i)
{
v=0.0;
if(disttype==0)
for(int k=0; k<nfeatures; k++)
{
vv=MathAbs(xy.Get(i,k)-xy.Get(j,k));
if(vv>v)
v=vv;
}
else
if(disttype==1)
for(int k=0; k<nfeatures; k++)
{
vv=MathAbs(xy.Get(i,k)-xy.Get(j,k));
v+=vv;
}
d.Set(i,j,v);
}
}
}
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
class CFilters
{
public:
static void FilterSMA(CRowDouble &x,int n,int k);
static void FilterEMA(CRowDouble &x,int n,double alpha);
static void FilterLRMA(CRowDouble &x,int n,int k);
};
//+------------------------------------------------------------------+
//| Filters: simple moving averages (unsymmetric). |
//| This filter replaces array by results of SMA(K) filter. SMA(K) |
//| is defined as filter which averages at most K previous points |
//| (previous - not points AROUND central point) - or less, in case |
//| of the first K-1 points. |
//| INPUT PARAMETERS: |
//| X - array[N], array to process. It can be larger |
//| than N, in this case only first N points are |
//| processed. |
//| N - points count, N>=0 |
//| K - K>=1 (K can be larger than N, such cases will |
//| be correctly handled). Window width. K=1 |
//| corresponds to identity transformation (nothing |
//| changes). |
//| OUTPUT PARAMETERS: |
//| X - array, whose first N elements were processed |
//| with SMA(K) |
//| NOTE 1: this function uses efficient in-place algorithm which |
//| does not allocate temporary arrays. |
//| NOTE 2: this algorithm makes only one pass through array and |
//| uses running sum to speed-up calculation of the averages.|
//| Additional measures are taken to ensure that running sum |
//| on a long sequence of zero elements will be correctly |
//| reset to zero even in the presence of round-off error. |
//| NOTE 3: this is unsymmetric version of the algorithm, which does |
//| NOT averages points after the current one. Only |
//| X[i], X[i-1], ... are used when calculating new value |
//| of X[i]. We should also note that this algorithm uses |
//| BOTH previous points and current one, i.e. new value |
//| of X[i] depends on BOTH previous point and X[i] itself. |
//+------------------------------------------------------------------+
void CFilters::FilterSMA(CRowDouble &x,int n,int k)
{
//--- Quick exit, if necessary
if(n<=1 || k==1)
return;
//--- create variables
double runningsum=0;
double termsinsum=0;
int zeroprefix=0;
double v=0;
//--- check
if(!CAp::Assert(n>=0,__FUNCTION__+": N<0"))
return;
if(!CAp::Assert(x.Size()>=n,__FUNCTION__+": Length(X)<N"))
return;
if(!CAp::Assert(CApServ::IsFiniteVector(x,n),__FUNCTION__+": X contains INF or NAN"))
return;
if(!CAp::Assert(k>=1,__FUNCTION__+": K<1"))
return;
//--- Prepare variables (see below for explanation)
runningsum=0.0;
termsinsum=0;
for(int i=MathMax(n-k,0); i<n; i++)
{
runningsum=runningsum+x[i];
termsinsum++;
}
int i=MathMax(n-k,0);
zeroprefix=0;
while(i<n && x[i]==0.0)
{
zeroprefix++;
i++;
}
//--- General case: we assume that N>1 and K>1
//--- Make one pass through all elements. At the beginning of
//--- the iteration we have:
//--- * I element being processed
//--- * RunningSum current value of the running sum
//--- (including I-th element)
//--- * TermsInSum number of terms in sum, 0<=TermsInSum<=K
//--- * ZeroPrefix length of the sequence of zero elements
//--- which starts at X[I-K+1] and continues towards X[I].
//--- Equal to zero in case X[I-K+1] is non-zero.
//--- This value is used to make RunningSum exactly zero
//--- when it follows from the problem properties.
for(i=n-1; i>=0; i--)
{
//--- Store new value of X[i], save old value in V
v=x[i];
x.Set(i,runningsum/termsinsum);
//--- Update RunningSum and TermsInSum
if(i-k>=0)
runningsum+=x[i-k]-v;
else
{
runningsum-=v;
termsinsum --;
}
//--- Update ZeroPrefix.
//--- In case we have ZeroPrefix=TermsInSum,
//--- RunningSum is reset to zero.
if(i-k>=0)
{
if(x[i-k]!=0.0)
zeroprefix=0;
else
zeroprefix=MathMin(zeroprefix+1,k);
}
else
{
zeroprefix=MathMin(zeroprefix,i+1);
}
if(zeroprefix==termsinsum)
runningsum=0;
}
}
//+------------------------------------------------------------------+
//| Filters: exponential moving averages. |
//| This filter replaces array by results of EMA(alpha) filter. |
//| EMA(alpha) is defined as filter which replaces X[] by S[]: |
//| S[0] = X[0] |
//| S[t] = alpha * X[t] + (1 - alpha) * S[t - 1] |
//| INPUT PARAMETERS: |
//| X - array[N], array to process. It can be larger |
//| than N, in this case only first N points are |
//| processed. |
//| N - points count, N >= 0 |
//| alpha - 0 < alpha <= 1, smoothing parameter. |
//| OUTPUT PARAMETERS: |
//| X - array, whose first N elements were processed |
//| with EMA(alpha) |
//| NOTE 1: this function uses efficient in-place algorithm which |
//| does not allocate temporary arrays. |
//| NOTE 2: this algorithm uses BOTH previous points and current one,|
//| i.e. new value of X[i] depends on BOTH previous point and|
//| X[i] itself. |
//| NOTE 3: technical analytis users quite often work with EMA |
//| coefficient expressed in DAYS instead of fractions. If |
//| you want to calculate EMA(N), where N is a number of |
//| days, you can use alpha = 2 / (N + 1). |
//+------------------------------------------------------------------+
void CFilters::FilterEMA(CRowDouble &x,int n,double alpha)
{
//--- Quick exit, if necessary
if(n<=1 || alpha==1.0)
return;
//--- check
if(!CAp::Assert(n>=0,__FUNCTION__+": N<0"))
return;
if(!CAp::Assert(x.Size()>=n,__FUNCTION__+": Length(X)<N"))
return;
if(!CAp::Assert(CApServ::IsFiniteVector(x,n),__FUNCTION__+": X contains INF or NAN"))
return;
if(!CAp::Assert(alpha>0.0,__FUNCTION__+": Alpha<=0"))
return;
if(!CAp::Assert(alpha<=1.0,__FUNCTION__+": Alpha>1"))
return;
//--- Process
for(int i=1; i<n; i++)
x.Set(i,alpha*x[i]+(1-alpha)*x[i-1]);
}
//+------------------------------------------------------------------+
//| Filters: linear regression moving averages. |
//| This filter replaces array by results of LRMA(K) filter. |
//| LRMA(K) is defined as filter which, for each data point, builds |
//| linear regression model using K prevous points (point itself is |
//| included in these K points) and calculates value of this linear |
//| model at the point in question. |
//| INPUT PARAMETERS: |
//| X - array[N], array to process. It can be larger |
//| than N, in this case only first N points are |
//| processed. |
//| N - points count, N >= 0 |
//| K - K >= 1(K can be larger than N, such cases will |
//| be correctly handled). Window width. K = 1 |
//| corresponds to identity transformation(nothing |
//| changes). |
//| OUTPUT PARAMETERS: |
//| X - array, whose first N elements were processed |
//| with SMA(K) |
//| NOTE 1: this function uses efficient in-place algorithm which |
//| does not allocate temporary arrays. |
//| NOTE 2: this algorithm makes only one pass through array and |
//| uses running sum to speed-up calculation of the averages.|
//| Additional measures are taken to ensure that running sum |
//| on a long sequence of zero elements will be correctly |
//| reset to zero even in the presence of round - off error. |
//| NOTE 3: this is unsymmetric version of the algorithm, which does |
//| NOT averages points after the current one. Only |
//| X[i], X[i - 1], ... are used when calculating new value |
//| of X[i]. We should also note that this algorithm uses |
//| BOTH previous points and current one, i.e. new value of |
//| X[i] depends on BOTH previous point and X[i] itself. |
//+------------------------------------------------------------------+
void CFilters::FilterLRMA(CRowDouble &x,int n,int k)
{
//--- check
if(!CAp::Assert(n>=0,__FUNCTION__+": N<0"))
return;
if(!CAp::Assert(x.Size()>=n,__FUNCTION__+": Length(X)<N"))
return;
if(!CAp::Assert(CApServ::IsFiniteVector(x,n),__FUNCTION__+": X contains INF or NAN"))
return;
if(!CAp::Assert(k>=1,__FUNCTION__+": K<1"))
return;
//--- Quick exit, if necessary:
//--- * either N is equal to 1 (nothing to average)
//--- * or K is 1 (only point itself is used) or 2 (model is too simple,
//--- we will always get identity transformation)
if(n<=1 || k<=2)
return;
//--- create variables
CMatrixDouble xy;
CRowDouble s;
int info=0;
double a=0;
double b=0;
double vara=0;
double varb=0;
double covab=0;
double corrab=0;
double p=0;
int m=0;
int i1_=0;
//--- General case: K>2, N>1.
//--- We do not process points with I<2 because first two points (I=0 and I=1) will be
//--- left unmodified by LRMA filter in any case.
xy.Resize(k,2);
s=vector<double>::Ones(k);
for(int i=0; i<k; i++)
xy.Set(i,0,i);
for(int i=n-1; i>=2; i--)
{
m=MathMin(i+1,k);
i1_=i-m+1;
for(int i_=0; i_<m; i_++)
xy.Set(i_,1,x[i_+i1_]);
CLinReg::LRLines(xy,s,m,info,a,b,vara,varb,covab,corrab,p);
if(!CAp::Assert(info==1,__FUNCTION__+": internal error"))
return;
x.Set(i,a+b*(m-1));
}
}
//+------------------------------------------------------------------+
//| This object stores state of the SSA model. |
//| You should use ALGLIB functions to work with this object. |
//+------------------------------------------------------------------+
struct CSSAModel
{
int m_NSequences;
CRowInt m_SequenceIdx;
CRowDouble m_SequenceData;
int m_AlgoType;
int m_WindowWidth;
int m_RtPowerUp;
int m_TopK;
int m_PrecomputedWidth;
int m_PrecomputedNBasis;
CMatrixDouble m_PrecomputedBasis;
int m_DefaultSubspaceits;
int m_MemoryLimit;
bool m_AreBasisAndSolverValid;
CMatrixDouble m_Basis;
CMatrixDouble m_BasisT;
CRowDouble m_SV;
CRowDouble m_ForecastA;
int m_NBasis;
CEigSubSpaceState m_Solver;
CMatrixDouble m_XXT;
CHighQualityRandState m_RS;
int m_RngSeed;
CRowInt m_Rtqueue;
int m_RtqueueCnt;
int m_RtqueueChunk;
int m_DbgCntEVD;
CRowDouble m_Tmp0;
CRowDouble m_Tmp1;
CEigSubSpaceReport m_SolverRep;
CRowDouble m_AlongTrend;
CRowDouble m_AlongNoise;
CMatrixDouble m_AseqTrajectory;
CMatrixDouble m_AseqTbProduct;
CRowInt m_AseqCounts;
CRowDouble m_FcTrend;
CRowDouble m_FcNoise;
CMatrixDouble m_FcTrendM;
CMatrixDouble m_UxBatch;
int m_UxBatchWidth;
int m_UxBatchSize;
int m_UxBatchLimit;
//---
CSSAModel(void);
~CSSAModel(void) {}
void Copy(const CSSAModel &obj);
//--- overloading
void operator=(const CSSAModel &obj) { Copy(obj); }
};
//+------------------------------------------------------------------+
//| Constructor |
//+------------------------------------------------------------------+
CSSAModel::CSSAModel(void)
{
m_NSequences=0;
m_AlgoType=0;
m_WindowWidth=0;
m_RtPowerUp=0;
m_TopK=0;
m_PrecomputedWidth=0;
m_PrecomputedNBasis=0;
m_DefaultSubspaceits=0;
m_MemoryLimit=0;
m_AreBasisAndSolverValid=false;
m_NBasis=0;
m_RngSeed=0;
m_RtqueueCnt=0;
m_RtqueueChunk=0;
m_DbgCntEVD=0;
m_UxBatchWidth=0;
m_UxBatchSize=0;
m_UxBatchLimit=0;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CSSAModel::Copy(const CSSAModel &obj)
{
m_NSequences=obj.m_NSequences;
m_SequenceIdx=obj.m_SequenceIdx;
m_SequenceData=obj.m_SequenceData;
m_AlgoType=obj.m_AlgoType;
m_WindowWidth=obj.m_WindowWidth;
m_RtPowerUp=obj.m_RtPowerUp;
m_TopK=obj.m_TopK;
m_PrecomputedWidth=obj.m_PrecomputedWidth;
m_PrecomputedNBasis=obj.m_PrecomputedNBasis;
m_PrecomputedBasis=obj.m_PrecomputedBasis;
m_DefaultSubspaceits=obj.m_DefaultSubspaceits;
m_MemoryLimit=obj.m_MemoryLimit;
m_AreBasisAndSolverValid=obj.m_AreBasisAndSolverValid;
m_Basis=obj.m_Basis;
m_BasisT=obj.m_BasisT;
m_SV=obj.m_SV;
m_ForecastA=obj.m_ForecastA;
m_NBasis=obj.m_NBasis;
m_Solver=obj.m_Solver;
m_XXT=obj.m_XXT;
m_RS=obj.m_RS;
m_RngSeed=obj.m_RngSeed;
m_Rtqueue=obj.m_Rtqueue;
m_RtqueueCnt=obj.m_RtqueueCnt;
m_RtqueueChunk=obj.m_RtqueueChunk;
m_DbgCntEVD=obj.m_DbgCntEVD;
m_Tmp0=obj.m_Tmp0;
m_Tmp1=obj.m_Tmp1;
m_SolverRep=obj.m_SolverRep;
m_AlongTrend=obj.m_AlongTrend;
m_AlongNoise=obj.m_AlongNoise;
m_AseqTrajectory=obj.m_AseqTrajectory;
m_AseqTbProduct=obj.m_AseqTbProduct;
m_AseqCounts=obj.m_AseqCounts;
m_FcTrend=obj.m_FcTrend;
m_FcNoise=obj.m_FcNoise;
m_FcTrendM=obj.m_FcTrendM;
m_UxBatch=obj.m_UxBatch;
m_UxBatchWidth=obj.m_UxBatchWidth;
m_UxBatchSize=obj.m_UxBatchSize;
m_UxBatchLimit=obj.m_UxBatchLimit;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
class CSSA
{
public:
static void SSACreate(CSSAModel &s);
static void SSASetWindow(CSSAModel &s,int windowwidth);
static void SSASetSeed(CSSAModel &s,int seed);
static void SSASetPowerUpLength(CSSAModel &s,int pwlen);
static void SSASetMemoryLimit(CSSAModel &s,int memlimit);
static void SSAAddSequence(CSSAModel &s,CRowDouble &x,int n);
static void SSAAppendPointAndUpdate(CSSAModel &s,double x,double updateits);
static void SSAAppendSequenceAndUpdate(CSSAModel &s,CRowDouble &x,int nticks,double updateits);
static void SSASetAlgoPrecomputed(CSSAModel &s,CMatrixDouble &a,int windowwidth,int nbasis);
static void SSASetAlgoTopKDirect(CSSAModel &s,int topk);
static void SSASetAlgoTopKRealtime(CSSAModel &s,int topk);
static void SSAClearData(CSSAModel &s);
static void SSAGetBasis(CSSAModel &s,CMatrixDouble &a,CRowDouble &sv,int &windowwidth,int &nbasis);
static void SSAGetLRR(CSSAModel &s,CRowDouble &a,int &windowwidth);
static void SSAAnalyzeLastWindow(CSSAModel &s,CRowDouble &trend,CRowDouble &noise,int &nticks);
static void SSAAnalyzeLast(CSSAModel &s,int nticks,CRowDouble &trend,CRowDouble &noise);
static void SSAAnalyzeSequence(CSSAModel &s,CRowDouble &data,int nticks,CRowDouble &trend,CRowDouble &noise);
static void SSAForecastLast(CSSAModel &s,int nticks,CRowDouble &trend);
static void SSAForecastSequence(CSSAModel &s,CRowDouble &data,int datalen,int forecastlen,bool applysmoothing,CRowDouble &trend);
static void SSAForecastAvgLast(CSSAModel &s,int m,int nticks,CRowDouble &trend);
static void SSAForecastAvgSequence(CSSAModel &s,CRowDouble &data,int datalen,int m,int forecastlen,bool applysmoothing,CRowDouble &trend);
private:
static bool HasSomethingToAnalyze(CSSAModel &s);
static bool IsSequenceBigEnough(CSSAModel &s,int i);
static void UpdateBasis(CSSAModel &s,int appendlen,double updateits);
static void AnalyzeSequence(CSSAModel &s,CRowDouble &data,int i0,int i1,CRowDouble &trend,CRowDouble &noise,int offs);
static void ForecastAvgSequence(CSSAModel &s,CRowDouble &data,int i0,int i1,int m,int forecastlen,bool smooth,CRowDouble &trend,int offs);
static void RealtimeDequeue(CSSAModel &s,double beta,int cnt);
static void UpdateXXTPrepare(CSSAModel &s,int updatesize,int windowwidth,int memorylimit);
static void UpdateXXTSend(CSSAModel &s,CRowDouble &u,int i0,CMatrixDouble &xxt);
static void UpdateXXTFinalize(CSSAModel &s,CMatrixDouble &xxt);
};
//+------------------------------------------------------------------+
//| This function creates SSA model object. Right after creation |
//| model is in "dummy" mode - you can add data, but analyzing / |
//| prediction will return just zeros (it assumes that basis is |
//| empty). |
//| HOW TO USE SSA MODEL: |
//| 1. create model with SSACreate() |
//| 2. add data with one/many SSAAddSequence() calls |
//| 3. choose SSA algorithm with one of SSASetAlgo...() functions: |
//| * SSASetAlgoTopKDirect() for direct one-run analysis |
//| * SSASetAlgoTopKRealtime() for algorithm optimized for many |
//| subsequent runs with warm-start capabilities |
//| * SSASetAlgoPrecomputed() for user-supplied basis |
//| 4. set window width with SSASetWindow() |
//| 5. perform one of the analysis-related activities: |
//| a) call SSAGetBasis() to get basis |
//| b) call SSAAnalyzeLast() SSAAnalyzeSequence() or |
//| SSAAnalyzeLastWindow() to perform analysis (trend/noise |
//| separation) |
//| c) call one of the forecasting functions (SSAForecastLast() |
//| or SSAForecastSequence()) to perform prediction; |
//| alternatively, you can extract linear recurrence |
//| coefficients with SSAGetLRR(). |
//| SSA analysis will be performed during first call to analysis - |
//| related function. SSA model is smart enough to track all changes |
//| in the dataset and model settings, to cache previously computed |
//| basis and to re-evaluate basis only when necessary. |
//| Additionally, if your setting involves constant stream of |
//| incoming data, you can perform quick update already calculated |
//| model with one of the incremental append-and-update functions: |
//| SSAAppendPointAndUpdate() or SSAAppendSequenceAndUpdate(). |
//| NOTE: steps (2), (3), (4) can be performed in arbitrary order. |
//| INPUT PARAMETERS: |
//| none |
//| OUTPUT PARAMETERS: |
//| S - structure which stores model state |
//+------------------------------------------------------------------+
void CSSA::SSACreate(CSSAModel &s)
{
//--- Model data, algorithms and settings
s.m_NSequences=0;
s.m_SequenceIdx.Resize(1);
s.m_SequenceIdx.Set(0,0);
s.m_AlgoType=0;
s.m_WindowWidth=1;
s.m_RtPowerUp=1;
s.m_AreBasisAndSolverValid=false;
s.m_RngSeed=1;
s.m_DefaultSubspaceits=10;
s.m_MemoryLimit=50000000;
//--- Debug counters
s.m_DbgCntEVD=0;
}
//+------------------------------------------------------------------+
//| This function sets window width for SSA model. You should call it|
//| before analysis phase. Default window width is 1 (not for real |
//| use). |
//| Special notes: |
//| * this function call can be performed at any moment before |
//| first call to analysis-related functions |
//| * changing window width invalidates internally stored basis; |
//| if you change window width AFTER you call analysis-related |
//| function, next analysis phase will require re-calculation of |
//| the basis according to current algorithm. |
//| * calling this function with exactly same window width as |
//| current one has no effect |
//| * if you specify window width larger than any data sequence |
//| stored in the model, analysis will return zero basis. |
//| INPUT PARAMETERS: |
//| S - SSA model created with SSACreate() |
//| WindowWidth - >=1, new window width |
//| OUTPUT PARAMETERS: |
//| S - SSA model, updated |
//+------------------------------------------------------------------+
void CSSA::SSASetWindow(CSSAModel &s,int windowwidth)
{
//--- check
if(!CAp::Assert(windowwidth>=1,__FUNCTION__+": WindowWidth<1"))
return;
if(windowwidth==s.m_WindowWidth)
return;
s.m_WindowWidth=windowwidth;
s.m_AreBasisAndSolverValid=false;
}
//+------------------------------------------------------------------+
//| This function sets seed which is used to initialize internal RNG |
//| when we make pseudorandom decisions on model updates. |
//| By default, deterministic seed is used - which results in same |
//| sequence of pseudorandom decisions every time you run SSA model. |
//| If you specify non-deterministic seed value, then SSA model may |
//| return slightly different results after each run. |
//| This function can be useful when you have several SSA models |
//| updated with SSAAppendPointAndUpdate() called with 0<UpdateIts<1 |
//| (fractional value) and due to performance limitations want them |
//| to perform updates at different moments. |
//| INPUT PARAMETERS: |
//| S - SSA model |
//| Seed - seed: |
//| * positive values = use deterministic seed for |
//| each run of algorithms which depend on random |
//| initialization |
//| * zero or negative values=use non-deterministic |
//| seed |
//+------------------------------------------------------------------+
void CSSA::SSASetSeed(CSSAModel &s,int seed)
{
s.m_RngSeed=seed;
}
//+------------------------------------------------------------------+
//| This function sets length of power-up cycle for real-time |
//| algorithm. |
//| By default, this algorithm performs costly O(N*WindowWidth^2) |
//| init phase followed by full run of truncated EVD. However, if you|
//| are ready to live with a bit lower-quality basis during first few|
//| iterations, you can split this O(N*WindowWidth^2) initialization |
//| between several subsequent append-and-update rounds. It results |
//| in better latency of the algorithm. |
//| This function invalidates basis/solver, next analysis call will |
//| result in full recalculation of everything. |
//| INPUT PARAMETERS: |
//| S - SSA model |
//| PWLen - length of the power-up stage: |
//| * 0 means that no power-up is requested |
//| * 1 is the same as 0 |
//| * >1 means that delayed power-up is performed |
//+------------------------------------------------------------------+
void CSSA::SSASetPowerUpLength(CSSAModel &s,int pwlen)
{
if(!CAp::Assert(pwlen>=0,__FUNCTION__+": PWLen<0"))
return;
s.m_RtPowerUp=MathMax(pwlen,1);
s.m_AreBasisAndSolverValid=false;
}
//+------------------------------------------------------------------+
//| This function sets memory limit of SSA analysis. |
//| Straightforward SSA with sequence length T and window width W |
//| needs O(T*W) memory. It is possible to reduce memory consumption |
//| by splitting task into smaller chunks. |
//| Thus function allows you to specify approximate memory limit |
//| (measured in double precision numbers used for buffers). Actual |
//| memory consumption will be comparable to the number specified by |
//| you. |
//| Default memory limit is 50.000.000 (400Mbytes) in current |
//| version. |
//| INPUT PARAMETERS: |
//| S - SSA model |
//| MemLimit - memory limit, >=0. Zero value means no limit. |
//+------------------------------------------------------------------+
void CSSA::SSASetMemoryLimit(CSSAModel &s,int memlimit)
{
if(memlimit<0)
memlimit=0;
s.m_MemoryLimit=memlimit;
}
//+------------------------------------------------------------------+
//| This function adds data sequence to SSA model. Only single- |
//| dimensional sequences are supported. |
//| What is a sequences? Following definitions/requirements apply: |
//| * a sequence is an array of values measured in subsequent, |
//| equally separated time moments (ticks). |
//| * you may have many sequences in your dataset; say, one |
//| sequence may correspond to one trading session. |
//| * sequence length should be larger than current window length |
//| (shorter sequences will be ignored during analysis). |
//| * analysis is performed within a sequence; different sequences |
//| are NOT stacked together to produce one large contiguous |
//| stream of data. |
//| * analysis is performed for all sequences at once, i.e. same |
//| set of basis vectors is computed for all sequences |
//| INCREMENTAL ANALYSIS |
//| This function is non intended for incremental updates of |
//| previously found SSA basis. Calling it invalidates all previous |
//| analysis results (basis is reset and will be recalculated from |
//| zero during next analysis). |
//| If you want to perform incremental/real-time SSA, consider using |
//| following functions: |
//| * SSAAppendPointAndUpdate() for appending one point |
//| * SSAAppendSequenceAndUpdate() for appending new sequence |
//| INPUT PARAMETERS: |
//| S - SSA model created with SSACreate() |
//| X - array[N], data, can be larger (additional values|
//| are ignored) |
//| N - data length, can be automatically determined |
//| from the array length. N>=0. |
//| OUTPUT PARAMETERS: |
//| S - SSA model, updated |
//| NOTE: you can clear dataset with SSAClearData() |
//+------------------------------------------------------------------+
void CSSA::SSAAddSequence(CSSAModel &s,CRowDouble &x,int n)
{
//--- check
if(!CAp::Assert(n>=0,__FUNCTION__+": N<0"))
return;
if(!CAp::Assert(x.Size()>=n,__FUNCTION__+": X is too short"))
return;
if(!CAp::Assert(CApServ::IsFiniteVector(x,n),__FUNCTION__+": X contains infinities NANs"))
return;
//--- Invalidate model
s.m_AreBasisAndSolverValid=false;
//--- Add sequence
s.m_SequenceIdx.Resize(s.m_NSequences+2);
s.m_SequenceIdx.Set(s.m_NSequences+1,s.m_SequenceIdx[s.m_NSequences]+n);
s.m_SequenceData.Resize(s.m_SequenceIdx[s.m_NSequences+1]);
int offs=s.m_SequenceIdx[s.m_NSequences];
for(int i=0; i<n; i++)
s.m_SequenceData.Set(offs+i,x[i]);
s.m_NSequences++;
}
//+------------------------------------------------------------------+
//| This function appends single point to last data sequence stored |
//| in the SSA model and tries to update model in the incremental |
//| manner (if possible with current algorithm). |
//| If you want to add more than one point at once: |
//| * if you want to add M points to the same sequence, perform |
//| M-1 calls with UpdateIts parameter set to 0.0, and last call |
//| with non-zero UpdateIts. |
//| * if you want to add new sequence, use |
//| SSAAppendSequenceAndUpdate() |
//| Running time of this function does NOT depend on dataset size, |
//| only on window width and number of singular vectors. Depending |
//| on algorithm being used, incremental update has complexity: |
//| * for top-K real time - O(UpdateIts*K*Width^2), with |
//| fractional UpdateIts |
//| * for top-K direct - O(Width^3) for any non-zero |
//| UpdateIts |
//| * for precomputed basis - O(1), no update is performed |
//| INPUT PARAMETERS: |
//| S - SSA model created with SSACreate() |
//| X - new point |
//| UpdateIts - >=0, floating point(!) value, desired update |
//| frequency: |
//| * zero value means that point is stored, but no update|
//| is performed |
//| * integer part of the value means that specified |
//| number of iterations is always performed |
//| * fractional part of the value means that one |
//| iteration is performed with this probability. |
//| Recommended value: 0<UpdateIts<=1. Values larger than 1 are VERY |
//| seldom needed. If your dataset changes slowly, you can set it to |
//| 0.1 and skip 90% of updates. |
//| In any case, no information is lost even with zero value of |
//| UpdateIts! It will be incorporated into model, sooner or later. |
//| OUTPUT PARAMETERS: |
//| S - SSA model, updated |
//| NOTE: this function uses internal RNG to handle fractional values|
//| of UpdateIts. By default it is initialized with fixed seed |
//| during initial calculation of basis. Thus subsequent calls |
//| to this function will result in the same sequence of |
//| pseudorandom decisions. |
//| However, if you have several SSA models which are calculated |
//| simultaneously, and if you want to reduce computational |
//| bottlenecks by performing random updates at random moments, then |
//| fixed seed is not an option - all updates will fire at same |
//| moments. |
//| You may change it with SSASetSeed() function. |
//| NOTE: this function throws an exception if called for empty |
//| dataset (there is no "last" sequence to modify). |
//+------------------------------------------------------------------+
void CSSA::SSAAppendPointAndUpdate(CSSAModel &s,double x,double updateits)
{
//--- check
if(!CAp::Assert(MathIsValidNumber(x),__FUNCTION__+": X is not finite"))
return;
//--- check
if(!CAp::Assert(MathIsValidNumber(updateits),__FUNCTION__+": UpdateIts is not finite"))
return;
//--- check
if(!CAp::Assert(updateits>=0.0,__FUNCTION__+": UpdateIts<0"))
return;
//--- check
if(!CAp::Assert(s.m_NSequences>0,__FUNCTION__+": dataset is empty,no sequence to modify"))
return;
//--- Append point to dataset
s.m_SequenceData.Resize(s.m_SequenceIdx[s.m_NSequences]+1);
s.m_SequenceData.Set(s.m_SequenceIdx[s.m_NSequences],x);
s.m_SequenceIdx.Set(s.m_NSequences,s.m_SequenceIdx[s.m_NSequences]+1);
//--- Do we have something to analyze? If no, invalidate basis
//--- (just to be sure) and exit.
if(!HasSomethingToAnalyze(s))
{
s.m_AreBasisAndSolverValid=false;
return;
}
//--- Well, we have data to analyze and algorithm set, but basis is
//--- invalid. Let's calculate it from scratch and exit.
if(!s.m_AreBasisAndSolverValid)
{
UpdateBasis(s,0,0.0);
return;
}
//--- Update already computed basis
UpdateBasis(s,1,updateits);
}
//+------------------------------------------------------------------+
//| This function appends new sequence to dataset stored in the SSA |
//| model and tries to update model in the incremental manner (if |
//| possible with current algorithm). |
//| Notes: |
//| * if you want to add M sequences at once, perform M-1 calls |
//| with UpdateIts parameter set to 0.0, and last call with |
//| non-zero UpdateIts. |
//| * if you want to add just one point, use |
//| SSAAppendPointAndUpdate() |
//| Running time of this function does NOT depend on dataset size, |
//| only on sequence length, window width and number of singular |
//| vectors. Depending on algorithm being used, incremental update |
//| has complexity: |
//| * for top-K real time - O(UpdateIts*K*Width^2+ |
//| (NTicks-Width)*Width^2) |
//| * for top-K direct - O(Width^3+(NTicks-Width)*Width^2) |
//| * for precomputed basis - O(1), no update is performed |
//| INPUT PARAMETERS: |
//| S - SSA model created with SSACreate() |
//| X - new sequence, array[NTicks] or larget |
//| NTicks - >=1, number of ticks in the sequence |
//| UpdateIts - >=0, floating point(!) value, desired update |
//| frequency: |
//| * zero value means that point is stored, but no |
//| update is performed |
//| * integer part of the value means that specified |
//| number of iterations is always performed |
//| * fractional part of the value means that one |
//| iteration is performed with this probability. |
//| Recommended value: 0<UpdateIts<=1. Values larger than 1 are VERY |
//| seldom needed. If your dataset changes slowly, you can set it to |
//| 0.1 and skip 90% of updates. |
//| In any case, no information is lost even with zero value of |
//| UpdateIts! It will be incorporated into model, sooner or later. |
//| OUTPUT PARAMETERS: |
//| S - SSA model, updated |
//| NOTE: this function uses internal RNG to handle fractional values|
//| of UpdateIts. By default it is initialized with fixed seed |
//| during initial calculation of basis. Thus subsequent calls |
//| to this function will result in the same sequence of |
//| pseudorandom decisions. |
//| However, if you have several SSA models which are calculated |
//| simultaneously, and if you want to reduce computational |
//| bottlenecks by performing random updates at random moments, then |
//| fixed seed is not an option - all updates will fire at same |
//| moments. |
//| You may change it with SSASetSeed() function. |
//+------------------------------------------------------------------+
void CSSA::SSAAppendSequenceAndUpdate(CSSAModel &s,CRowDouble &x,
int nticks,double updateits)
{
//--- check
if(!CAp::Assert(nticks>=0,__FUNCTION__+": NTicks<0"))
return;
if(!CAp::Assert(x.Size()>=nticks,__FUNCTION__+": X is too short"))
return;
if(!CAp::Assert(CApServ::IsFiniteVector(x,nticks),__FUNCTION__+": X contains infinities NANs"))
return;
//--- Add sequence
s.m_SequenceIdx.Resize(s.m_NSequences+2);
s.m_SequenceIdx.Set(s.m_NSequences+1,s.m_SequenceIdx[s.m_NSequences]+nticks);
s.m_SequenceData.Resize(s.m_SequenceIdx[s.m_NSequences+1]);
int offs=s.m_SequenceIdx[s.m_NSequences];
for(int i=0; i<nticks; i++)
s.m_SequenceData.Set(offs+i,x[i]);
s.m_NSequences++;
//--- Do we have something to analyze? If no, invalidate basis
//--- (just to be sure) and exit.
if(!HasSomethingToAnalyze(s))
{
s.m_AreBasisAndSolverValid=false;
return;
}
//--- Well, we have data to analyze and algorithm set, but basis is
//--- invalid. Let's calculate it from scratch and exit.
if(!s.m_AreBasisAndSolverValid)
{
UpdateBasis(s,0,0.0);
return;
}
//--- Update already computed basis
if(nticks>=s.m_WindowWidth)
UpdateBasis(s,nticks-s.m_WindowWidth+1,updateits);
}
//+------------------------------------------------------------------+
//| This function sets SSA algorithm to "precomputed vectors" |
//| algorithm. |
//| This algorithm uses precomputed set of orthonormal(orthogonal |
//| AND normalized) basis vectors supplied by user. Thus, basis |
//| calculation phase is not performed - we already have our basis - |
//| and only analysis/forecasting phase requires actual calculations.|
//| This algorithm may handle "append" requests which add just one/ |
//| few ticks to the end of the last sequence in O(1) time. |
//| NOTE: this algorithm accepts both basis and window width, because|
//| these two parameters are naturally aligned. Calling this |
//| function sets window width; if you call SSASetWindow() |
//| with other window width, then during analysis stage |
//| algorithm will detect conflict and reset to zero basis. |
//| INPUT PARAMETERS: |
//| S - SSA model |
//| A - array[WindowWidth, NBasis], orthonormalized |
//| basis; this function does NOT control |
//| orthogonality and does NOT perform any kind of |
//| renormalization. It is your responsibility to |
//| provide it with correct basis. |
//| WindowWidth - window width, >= 1 |
//| NBasis - number of basis vectors, |
//| 1 <= NBasis <= WindowWidth |
//| OUTPUT PARAMETERS: |
//| S - updated model |
//| NOTE: calling this function invalidates basis in all cases. |
//+------------------------------------------------------------------+
void CSSA::SSASetAlgoPrecomputed(CSSAModel &s,CMatrixDouble &a,
int windowwidth,int nbasis)
{
//--- check
if(!CAp::Assert(windowwidth>=1,__FUNCTION__+": WindowWidth<1"))
return;
if(!CAp::Assert(nbasis>=1,__FUNCTION__+": NBasis<1"))
return;
if(!CAp::Assert(nbasis<=windowwidth,__FUNCTION__+": NBasis>WindowWidth"))
return;
if(!CAp::Assert(a.Rows()>=windowwidth,__FUNCTION__+": Rows(A)<WindowWidth"))
return;
if(!CAp::Assert(a.Cols()>=nbasis,__FUNCTION__+": Rows(A)<NBasis"))
return;
if(!CAp::Assert(CApServ::IsFiniteMatrix(a,windowwidth,nbasis),__FUNCTION__+": Rows(A)<NBasis"))
return;
s.m_AlgoType=1;
s.m_PrecomputedWidth=windowwidth;
s.m_PrecomputedNBasis=nbasis;
s.m_WindowWidth=windowwidth;
s.m_PrecomputedBasis=a;
s.m_PrecomputedBasis.Resize(windowwidth,nbasis);
s.m_AreBasisAndSolverValid=false;
}
//+------------------------------------------------------------------+
//| This function sets SSA algorithm to "direct top-K" algorithm. |
//| "Direct top-K" algorithm performs full SVD of the N*WINDOW |
//| trajectory matrix (hence its name - direct solver is used), then |
//| extracts top K components. Overall running time is |
//| O(N * WINDOW ^ 2), where N is a number of ticks in the dataset, |
//| WINDOW is window width. |
//| This algorithm may handle "append" requests which add just one / |
//| few ticks to the end of the last sequence in O(WINDOW ^ 3) time, |
//| which is ~N/WINDOW times faster than re-computing everything from|
//| scratch. |
//| INPUT PARAMETERS: |
//| S - SSA model |
//| TopK - number of components to analyze; TopK >= 1. |
//| OUTPUT PARAMETERS: |
//| S - updated model |
//| NOTE: TopK>WindowWidth is silently decreased to WindowWidth |
//| during analysis phase |
//| NOTE: calling this function invalidates basis, except for the |
//| situation when this algorithm was already set with same |
//| parameters. |
//+------------------------------------------------------------------+
void CSSA::SSASetAlgoTopKDirect(CSSAModel &s,int topk)
{
//--- check
if(!CAp::Assert(topk>=1,__FUNCTION__+": TopK<1"))
return;
//--- Ignore calls which change nothing
if(s.m_AlgoType==2 && s.m_TopK==topk)
return;
//--- Update settings, invalidate model
s.m_AlgoType=2;
s.m_TopK=topk;
s.m_AreBasisAndSolverValid=false;
}
//+------------------------------------------------------------------+
//| This function sets SSA algorithm to "top-K real time algorithm". |
//| This algo extracts K components with largest singular values. |
//| It is real-time version of top-K algorithm which is optimized for|
//| incremental processing and fast start-up. Internally it uses |
//| subspace eigensolver for truncated SVD. It results in ability to |
//| perform quick updates of the basis when only a few points / |
//| sequences is added to dataset. |
//| Performance profile of the algorithm is given below: |
//| * O(K * WindowWidth ^ 2) running time for incremental update |
//| of the dataset with one of the "append-and-update" functions |
//| (SSAAppendPointAndUpdate() or SSAAppendSequenceAndUpdate()). |
//| * O(N * WindowWidth ^ 2) running time for initial basis |
//| evaluation(N = size of dataset) |
//| * ability to split costly initialization across several |
//| incremental updates of the basis(so called "Power-Up" |
//| functionality, activated by SSASetPowerUpLength() function) |
//| INPUT PARAMETERS: |
//| S - SSA model |
//| TopK - number of components to analyze; TopK >= 1. |
//| OUTPUT PARAMETERS: |
//| S - updated model |
//| NOTE: this algorithm is optimized for large-scale tasks with |
//| large datasets. On toy problems with just 5-10 points it |
//| can return basis which is slightly different from that |
//| returned by direct algorithm (SSASetAlgoTopKDirect() |
//| function). However, the difference becomes negligible as |
//| dataset grows. |
//| NOTE: TopK > WindowWidth is silently decreased to WindowWidth |
//| during analysis phase |
//| NOTE: calling this function invalidates basis, except for the |
//| situation when this algorithm was already set with same |
//| parameters. |
//+------------------------------------------------------------------+
void CSSA::SSASetAlgoTopKRealtime(CSSAModel &s,int topk)
{
//--- check
if(!CAp::Assert(topk>=1,__FUNCTION__+": TopK<1"))
return;
//--- Ignore calls which change nothing
if(s.m_AlgoType==3 && s.m_TopK==topk)
return;
//--- Update settings, invalidate model
s.m_AlgoType=3;
s.m_TopK=topk;
s.m_AreBasisAndSolverValid=false;
}
//+------------------------------------------------------------------+
//| This function clears all data stored in the model and invalidates|
//| all basis components found so far. |
//| INPUT PARAMETERS: |
//| S - SSA model created with SSACreate() |
//| OUTPUT PARAMETERS: |
//| S - SSA model, updated |
//+------------------------------------------------------------------+
void CSSA::SSAClearData(CSSAModel &s)
{
s.m_NSequences=0;
s.m_AreBasisAndSolverValid=false;
}
//+------------------------------------------------------------------+
//| This function executes SSA on internally stored dataset and |
//| returns basis found by current method. |
//| INPUT PARAMETERS: |
//| S - SSA model |
//| OUTPUT PARAMETERS: |
//| A - array[WindowWidth, NBasis], basis; vectors are |
//| stored in matrix columns, by descreasing variance |
//| SV - array[NBasis]: |
//| * zeros - for model initialized with |
//| SSASetAlgoPrecomputed() |
//| * singular values - for other algorithms |
//| WindowWidth - current window |
//| NBasis - basis size |
//| CACHING / REUSE OF THE BASIS |
//| Caching / reuse of previous results is performed: |
//| * first call performs full run of SSA; basis is stored in the |
//| cache |
//| * subsequent calls reuse previously cached basis |
//| * if you call any function which changes model properties |
//| (window length, algorithm, dataset), internal basis will be |
//| invalidated. |
//| * the only calls which do NOT invalidate basis are listed |
//| below: |
//| a) SSASetWindow() with same window length |
//| b) SSAAppendPointAndUpdate() |
//| c) SSAAppendSequenceAndUpdate() |
//| d) SSASetAlgoTopK...() with exactly same K Calling these |
//| functions will result in reuse of previously found basis.|
//| HANDLING OF DEGENERATE CASES |
//| Calling this function in degenerate cases(no data or all data are|
//| shorter than window size; no algorithm is specified) returns |
//| basis with just one zero vector. |
//+------------------------------------------------------------------+
void CSSA::SSAGetBasis(CSSAModel &s,CMatrixDouble &a,
CRowDouble &sv,int &windowwidth,
int &nbasis)
{
//--- initialization
windowwidth=0;
nbasis=0;
//--- Is it degenerate case?
if(!HasSomethingToAnalyze(s))
{
windowwidth=s.m_WindowWidth;
nbasis=1;
a=matrix<double>::Zeros(windowwidth,1);
sv=vector<double>::Zeros(1);
return;
}
//--- Update basis.
//--- It will take care of basis validity flags. AppendLen=0 which means
//--- that we perform initial basis evaluation.
UpdateBasis(s,0,0.0);
//--- check
if(!CAp::Assert(s.m_NBasis>0,__FUNCTION__+": integrity check failed"))
return;
if(!CAp::Assert(s.m_WindowWidth>0,__FUNCTION__+": integrity check failed"))
return;
//--- Output
nbasis=s.m_NBasis;
windowwidth=s.m_WindowWidth;
a=s.m_Basis;
sv=s.m_SV;
a.Resize(windowwidth,nbasis);
sv.Resize(nbasis);
}
//+------------------------------------------------------------------+
//| This function returns linear recurrence relation(LRR) |
//| coefficients found by current SSA algorithm. |
//| INPUT PARAMETERS: |
//| S - SSA model |
//| OUTPUT PARAMETERS: |
//| A - array[WindowWidth - 1]. Coefficients of the linear |
//| recurrence of the form: |
//| X[W - 1] = X[W - 2] * A[W - 2] + |
//| X[W - 3] * A[W - 3] + ... + X[0] * A[0].|
//| Empty array for WindowWidth = 1. |
//| WindowWidth - current window width |
//| CACHING / REUSE OF THE BASIS |
//| Caching / reuse of previous results is performed: |
//| * first call performs full run of SSA; basis is stored in the |
//| cache |
//| * subsequent calls reuse previously cached basis |
//| * if you call any function which changes model properties |
//| (window length, algorithm, dataset), internal basis will be |
//| invalidated. |
//| * the only calls which do NOT invalidate basis are listed |
//| below: |
//| a) SSASetWindow() with same window length |
//| b) SSAAppendPointAndUpdate() |
//| c) SSAAppendSequenceAndUpdate() |
//| d) SSASetAlgoTopK...() with exactly same K |
//| Calling these functions will result in reuse of previously found |
//| basis. |
//| HANDLING OF DEGENERATE CASES |
//| Calling this function in degenerate cases (no data or all data |
//| are shorter than window size; no algorithm is specified) returns |
//| zeros. |
//+------------------------------------------------------------------+
void CSSA::SSAGetLRR(CSSAModel &s,CRowDouble &a,int &windowwidth)
{
a.Resize(0);
windowwidth=0;
//--- check
if(!CAp::Assert(s.m_WindowWidth>0,__FUNCTION__+": integrity check failed"))
return;
//--- Is it degenerate case?
if(!HasSomethingToAnalyze(s))
{
windowwidth=s.m_WindowWidth;
a=vector<double>::Zeros(windowwidth-1);
return;
}
//--- Update basis.
//--- It will take care of basis validity flags. AppendLen=0 which means
//--- that we perform initial basis evaluation.
UpdateBasis(s,0,0.0);
//--- Output
windowwidth=s.m_WindowWidth;
a=s.m_ForecastA;
a.Resize(windowwidth-1);
}
//+------------------------------------------------------------------+
//| This function executes SSA on internally stored dataset and |
//| returns analysis for the last window of the last sequence. Such |
//| analysis is an lightweight alternative for full scale |
//| reconstruction (see below). |
//| Typical use case for this function is real-time setting, when |
//| you are interested in quick-and-dirty (very quick and very dirty)|
//| processing of just a few last ticks of the trend. |
//| IMPORTANT: full scale SSA involves analysis of the ENTIRE |
//| dataset, with reconstruction being done for all |
//| positions of sliding window with subsequent |
//| hankelization (diagonal averaging) of the resulting |
//| matrix. |
//| Such analysis requires O((DataLen - Window)*Window*NBasis) FLOPs |
//| and can be quite costly. However, it has nice noise - canceling |
//| effects due to averaging. |
//| This function performs REDUCED analysis of the last window. It |
//| is much faster - just O(Window*NBasis), but its results are |
//| DIFFERENT from that of SSAAnalyzeLast(). In particular, first few|
//| points of the trend are much more prone to noise. |
//| INPUT PARAMETERS: |
//| S - SSA model |
//| OUTPUT PARAMETERS: |
//| Trend - array[WindowSize], reconstructed trend line |
//| Noise - array[WindowSize], the rest of the signal; it |
//| holds that ActualData = Trend + Noise. |
//| NTicks - current WindowSize |
//| CACHING / REUSE OF THE BASIS |
//| Caching / reuse of previous results is performed: |
//| * first call performs full run of SSA; basis is stored in the |
//| cache |
//| * subsequent calls reuse previously cached basis |
//| * if you call any function which changes model properties |
//| (window length, algorithm, dataset), internal basis will |
//| be invalidated. |
//| * the only calls which do NOT invalidate basis are listed |
//| below: |
//| a) SSASetWindow() with same window length |
//| b) SSAAppendPointAndUpdate() |
//| c) SSAAppendSequenceAndUpdate() |
//| d) SSASetAlgoTopK...() with exactly same K |
//| Calling these functions will result in reuse of previously found |
//| basis. |
//| In any case, only basis is reused. Reconstruction is performed |
//| from scratch every time you call this function. |
//| HANDLING OF DEGENERATE CASES |
//| Following degenerate cases may happen: |
//| * dataset is empty(no analysis can be done) |
//| * all sequences are shorter than the window length, no analysis|
//| can be done |
//| * no algorithm is specified(no analysis can be done) |
//| * last sequence is shorter than the window length (analysis |
//| can be done, but we can not perform reconstruction on the |
//| last sequence) |
//| Calling this function in degenerate cases returns following |
//| result: |
//| * in any case, WindowWidth ticks is returned |
//| * trend is assumed to be zero |
//| * noise is initialized by the last sequence; if last sequence |
//| is shorter than the window size, it is moved to the end of |
//| the array, and the beginning of the noise array is filled by |
//| zeros |
//| No analysis is performed in degenerate cases (we immediately |
//| return dummy values, no basis is constructed). |
//+------------------------------------------------------------------+
void CSSA::SSAAnalyzeLastWindow(CSSAModel &s,CRowDouble &trend,
CRowDouble &noise,int &nticks)
{
//--- create variables
int offs=0;
int cnt=0;
//--- Init
nticks=s.m_WindowWidth;
trend.Resize(s.m_WindowWidth);
noise.Resize(s.m_WindowWidth);
//--- Is it degenerate case?
if(!HasSomethingToAnalyze(s) || !IsSequenceBigEnough(s,-1))
{
trend.Fill(0);
noise.Fill(0);
if(s.m_NSequences>=1)
{
cnt=MathMin(s.m_SequenceIdx[s.m_NSequences]-s.m_SequenceIdx[s.m_NSequences-1],nticks);
offs=s.m_SequenceIdx[s.m_NSequences]-cnt;
for(int i=0; i<cnt; i++)
noise.Set(nticks-cnt+i,s.m_SequenceData[offs+i]);
}
return;
}
//--- Update basis.
//--- It will take care of basis validity flags. AppendLen=0 which means
//--- that we perform initial basis evaluation.
UpdateBasis(s,0,0.0);
//--- Perform analysis of the last window
//--- check
if(!CAp::Assert(s.m_SequenceIdx[s.m_NSequences]-s.m_WindowWidth>=0,__FUNCTION__+": integrity check failed"))
return;
CApServ::RVectorSetLengthAtLeast(s.m_Tmp0,s.m_NBasis);
CAblas::RMatrixGemVect(s.m_NBasis,s.m_WindowWidth,1.0,s.m_BasisT,0,0,0,s.m_SequenceData,s.m_SequenceIdx[s.m_NSequences]-s.m_WindowWidth,0.0,s.m_Tmp0,0);
CAblas::RMatrixGemVect(s.m_WindowWidth,s.m_NBasis,1.0,s.m_Basis,0,0,0,s.m_Tmp0,0,0.0,trend,0);
offs=s.m_SequenceIdx[s.m_NSequences]-s.m_WindowWidth;
cnt=s.m_WindowWidth;
for(int i=0; i<cnt; i++)
noise.Set(i,s.m_SequenceData[offs+i]-trend[i]);
}
//+------------------------------------------------------------------+
//| This function: |
//| * builds SSA basis using internally stored(entire) dataset |
//| * returns reconstruction for the last NTicks of the last |
//| sequence |
//| If you want to analyze some other sequence, use |
//| SSAAnalyzeSequence(). |
//| Reconstruction phase involves generation of NTicks-WindowWidth |
//| sliding windows, their decomposition using empirical orthogonal |
//| functions found by SSA, followed by averaging of each data point |
//| across several overlapping windows. Thus, every point in the |
//| output trend is reconstructed using up to WindowWidth overlapping|
//| windows(WindowWidth windows exactly in the inner points, just one|
//| window at the extremal points). |
//| IMPORTANT: due to averaging this function returns different |
//| results for different values of NTicks. It is expected|
//| and not a bug. |
//| For example: |
//| * Trend[NTicks - 1] is always same because it is not averaged |
//| in any case(same applies to Trend[0]). |
//| * Trend[NTicks - 2] has different values fo NTicks=WindowWidth |
//| and NTicks=WindowWidth+1 because former case means that no |
//| averaging is performed, and latter case means that averaging |
//| using two sliding windows is performed. Larger values of |
//| NTicks produce same results as NTicks = WindowWidth + 1. |
//| * ...and so on... |
//| PERFORMANCE: this function has |
//| O((NTicks - WindowWidth) * WindowWidth*NBasis) |
//| running time. If you work in time-constrained setting and have |
//| to analyze just a few last ticks, choosing NTicks equal to |
//| WindowWidth + SmoothingLen, with SmoothingLen = 1...WindowWidth |
//| will result in good compromise between noise cancellation and |
//| analysis speed. |
//| INPUT PARAMETERS: |
//| S - SSA model |
//| NTicks - number of ticks to analyze, Nticks >= 1. |
//| * special case of NTicks<=WindowWidth is handled |
//| by analyzing last window and returning NTicks |
//| last ticks. |
//| * special case NTicks>LastSequenceLen is handled by |
//| prepending result with NTicks-LastSequenceLen zeros.|
//| OUTPUT PARAMETERS: |
//| Trend - array[NTicks], reconstructed trend line |
//| Noise - array[NTicks], the rest of the signal; it holds |
//| that ActualData = Trend + Noise. |
//| CACHING / REUSE OF THE BASIS |
//| Caching / reuse of previous results is performed: |
//| * first call performs full run of SSA; basis is stored in |
//| the cache |
//| * subsequent calls reuse previously cached basis |
//| * if you call any function which changes model properties |
//| (window length, algorithm, dataset), internal basis will |
//| be invalidated. |
//| * the only calls which do NOT invalidate basis are listed |
//| below: |
//| a) SSASetWindow() with same window length |
//| b) SSAAppendPointAndUpdate() |
//| c) SSAAppendSequenceAndUpdate() |
//| d) SSASetAlgoTopK...() with exactly same K |
//| Calling these functions will result in reuse of previously found |
//| basis. |
//| In any case, only basis is reused. Reconstruction is performed |
//| from scratch every time you call this function. |
//| HANDLING OF DEGENERATE CASES |
//| Following degenerate cases may happen: |
//| * dataset is empty(no analysis can be done) |
//| * all sequences are shorter than the window length, no |
//| analysis can be done |
//| * no algorithm is specified(no analysis can be done) |
//| * last sequence is shorter than the window length(analysis |
//| can be done, but we can not perform reconstruction on the |
//| last sequence) |
//| Calling this function in degenerate cases returns following |
//| result: |
//| * in any case, NTicks ticks is returned |
//| * trend is assumed to be zero |
//| * noise is initialized by the last sequence; if last |
//| sequence is shorter than the window size, it is moved |
//| to the end of the array, and the beginning of the noise |
//| array is filled by zeros |
//| No analysis is performed in degenerate cases(we immediately |
//| return dummy values, no basis is constructed). |
//+------------------------------------------------------------------+
void CSSA::SSAAnalyzeLast(CSSAModel &s,int nticks,CRowDouble &trend,
CRowDouble &noise)
{
//--- create variables
int offs=0;
int cnt=0;
int cntzeros=0;
//--- check
if(!CAp::Assert(nticks>=1,__FUNCTION__+": NTicks<1"))
return;
//--- Init
trend.Resize(nticks);
noise.Resize(nticks);
//--- Is it degenerate case?
if(!HasSomethingToAnalyze(s) || !IsSequenceBigEnough(s,-1))
{
trend.Fill(0);
noise.Fill(0);
if(s.m_NSequences>=1)
{
cnt=MathMin(s.m_SequenceIdx[s.m_NSequences]-s.m_SequenceIdx[s.m_NSequences-1],nticks);
offs=s.m_SequenceIdx[s.m_NSequences]-cnt;
for(int i=0; i<cnt; i++)
noise.Set(nticks-cnt+i,s.m_SequenceData[offs+i]);
}
return;
}
//--- Fast exit: NTicks<=WindowWidth, just last window is analyzed
if(nticks<=s.m_WindowWidth)
{
SSAAnalyzeLastWindow(s,s.m_AlongTrend,s.m_AlongNoise,cnt);
offs=s.m_WindowWidth-nticks;
for(int i=0; i<nticks; i++)
{
trend.Set(i,s.m_AlongTrend[offs+i]);
noise.Set(i,s.m_AlongNoise[offs+i]);
}
return;
}
//--- Update basis.
//--- It will take care of basis validity flags. AppendLen=0 which means
//--- that we perform initial basis evaluation.
UpdateBasis(s,0,0.0);
//--- Perform analysis:
//--- * prepend max(NTicks-LastSequenceLength,0) zeros to the beginning
//--- of array
//--- * analyze the rest with AnalyzeSequence() which assumes that we
//--- already have basis
if(!CAp::Assert(s.m_SequenceIdx[s.m_NSequences]-s.m_SequenceIdx[s.m_NSequences-1]>=s.m_WindowWidth,__FUNCTION__+": integrity check failed / 23vd4"))
return;
cntzeros=MathMax(nticks-(s.m_SequenceIdx[s.m_NSequences]-s.m_SequenceIdx[s.m_NSequences-1]),0);
for(int i=0; i<cntzeros; i++)
{
trend.Set(i,0.0);
noise.Set(i,0.0);
}
cnt=MathMin(nticks,s.m_SequenceIdx[s.m_NSequences]-s.m_SequenceIdx[s.m_NSequences-1]);
AnalyzeSequence(s,s.m_SequenceData,s.m_SequenceIdx[s.m_NSequences]-cnt,s.m_SequenceIdx[s.m_NSequences],trend,noise,cntzeros);
}
//+------------------------------------------------------------------+
//| This function: |
//| * builds SSA basis using internally stored(entire) dataset |
//| * returns reconstruction for the sequence being passed to |
//| this function |
//| If you want to analyze last sequence stored in the model, use |
//| SSAAnalyzeLast(). |
//| Reconstruction phase involves generation of NTicks-WindowWidth |
//| sliding windows, their decomposition using empirical orthogonal |
//| functions found by SSA, followed by averaging of each data point |
//| across several overlapping windows. Thus, every point in the |
//| output trend is reconstructed using up to WindowWidth overlapping|
//| windows(WindowWidth windows exactly in the inner points, just one|
//| window at the extremal points). |
//| PERFORMANCE: this function has |
//| O((NTicks - WindowWidth)*WindowWidth*NBasis) |
//| running time. If you work in time-constrained setting and have |
//| to analyze just a few last ticks, choosing NTicks equal to |
//| WindowWidth + SmoothingLen, with SmoothingLen = 1...WindowWidth |
//| will result in good compromise between noise cancellation and |
//| analysis speed. |
//| INPUT PARAMETERS: |
//| S - SSA model |
//| Data - array[NTicks], can be larger(only NTicks leading|
//| elements will be used) |
//| NTicks - number of ticks to analyze, Nticks >= 1. |
//| * special case of NTicks<WindowWidth is handled by |
//| returning zeros as trend, and signal as noise |
//| OUTPUT PARAMETERS: |
//| Trend - array[NTicks], reconstructed trend line |
//| Noise - array[NTicks], the rest of the signal; it holds |
//| that ActualData = Trend + Noise. |
//| CACHING / REUSE OF THE BASIS |
//| Caching / reuse of previous results is performed: |
//| * first call performs full run of SSA; basis is stored in |
//| the cache |
//| * subsequent calls reuse previously cached basis |
//| * if you call any function which changes model properties |
//| (window length, algorithm, dataset), internal basis will |
//| be invalidated. |
//| * the only calls which do NOT invalidate basis are listed |
//| below: |
//| a) SSASetWindow() with same window length |
//| b) SSAAppendPointAndUpdate() |
//| c) SSAAppendSequenceAndUpdate() |
//| d) SSASetAlgoTopK...() with exactly same K |
//| Calling these functions will result in reuse of previously found |
//| basis. |
//| In any case, only basis is reused. Reconstruction is performed |
//| from scratch every time you call this function. |
//| HANDLING OF DEGENERATE CASES |
//| Following degenerate cases may happen: |
//| * dataset is empty(no analysis can be done) |
//| * all sequences are shorter than the window length, no |
//| analysis can be done |
//| * no algorithm is specified(no analysis can be done) |
//| * sequence being passed is shorter than the window length |
//| Calling this function in degenerate cases returns following |
//| result: |
//| * in any case, NTicks ticks is returned |
//| * trend is assumed to be zero |
//| * noise is initialized by the sequence. |
//| No analysis is performed in degenerate cases(we immediately |
//| return dummy values, no basis is constructed). |
//+------------------------------------------------------------------+
void CSSA::SSAAnalyzeSequence(CSSAModel &s,CRowDouble &data,int nticks,
CRowDouble &trend,CRowDouble &noise)
{
//--- check
if(!CAp::Assert(nticks>=1,__FUNCTION__+": NTicks<1"))
return;
if(!CAp::Assert(data.Size()>=nticks,__FUNCTION__+": Data is too short"))
return;
if(!CAp::Assert(CApServ::IsFiniteVector(data,nticks),__FUNCTION__+": Data contains infinities NANs"))
return;
//--- Init
trend=vector<double>::Zeros(nticks);
noise=vector<double>::Zeros(nticks);
//--- Is it degenerate case?
if(!HasSomethingToAnalyze(s) || nticks<s.m_WindowWidth)
{
noise=data;
return;
}
//--- Update basis.
//--- It will take care of basis validity flags. AppendLen=0 which means
//--- that we perform initial basis evaluation.
UpdateBasis(s,0,0.0);
//--- Perform analysis
AnalyzeSequence(s,data,0,nticks,trend,noise,0);
}
//+------------------------------------------------------------------+
//| This function builds SSA basis and performs forecasting for a |
//| specified number of ticks, returning value of trend. |
//| Forecast is performed as follows: |
//| * SSA trend extraction is applied to last WindowWidth |
//| elements of the internally stored dataset; this step is |
//| basically a noise reduction. |
//| * linear recurrence relation is applied to extracted trend |
//| This function has following running time: |
//| * O(NBasis*WindowWidth) for trend extraction phase (always |
//| performed) |
//| * O(WindowWidth*NTicks) for forecast phase |
//| NOTE: noise reduction is ALWAYS applied by this algorithm; if you|
//| want to apply recurrence relation to raw unprocessed data, |
//| use another function - SSAForecastSequence() which allows |
//| to turn on and off noise reduction phase. |
//| NOTE: this algorithm performs prediction using only one-last - |
//| sliding window. Predictions produced by such approach are |
//| smooth continuations of the reconstructed trend line, but |
//| they can be easily corrupted by noise. If you need noise - |
//| resistant prediction, use SSAForecastAvgLast() function, |
//| which averages predictions built using several sliding |
//| windows. |
//| INPUT PARAMETERS: |
//| S - SSA model |
//| NTicks - number of ticks to forecast, NTicks >= 1 |
//| OUTPUT PARAMETERS: |
//| Trend - array[NTicks], predicted trend line |
//| CACHING / REUSE OF THE BASIS |
//| Caching / reuse of previous results is performed: |
//| * first call performs full run of SSA; basis is stored in |
//| the cache |
//| * subsequent calls reuse previously cached basis |
//| * if you call any function which changes model properties |
//| (window length, algorithm, dataset), internal basis will |
//| be invalidated. |
//| * the only calls which do NOT invalidate basis are listed |
//| below: |
//| a) SSASetWindow() with same window length |
//| b) SSAAppendPointAndUpdate() |
//| c) SSAAppendSequenceAndUpdate() |
//| d) SSASetAlgoTopK...() with exactly same K |
//| Calling these functions will result in reuse of previously found |
//| basis. |
//| HANDLING OF DEGENERATE CASES |
//| Following degenerate cases may happen: |
//| * dataset is empty(no analysis can be done) |
//| * all sequences are shorter than the window length, no |
//| analysis can be done |
//| * no algorithm is specified(no analysis can be done) |
//| * last sequence is shorter than the WindowWidth(analysis can|
//| be done, but we can not perform forecasting on the last |
//| sequence) |
//| * window lentgh is 1(impossible to use for forecasting) |
//| * SSA analysis algorithm is configured to extract basis |
//| whose size is equal to window length(impossible to use for|
//| forecasting; only basis whose size is less than window |
//| length can be used). |
//| Calling this function in degenerate cases returns following |
//| result: |
//| * NTicks copies of the last value is returned for non-empty |
//| task with large enough dataset, but with overcomplete |
//| basis (window width = 1 or basis size is equal to window |
//| width) |
//| * zero trend with length = NTicks is returned for empty task|
//| No analysis is performed in degenerate cases (we immediately |
//| return dummy values, no basis is ever constructed). |
//+------------------------------------------------------------------+
void CSSA::SSAForecastLast(CSSAModel &s,int nticks,CRowDouble &trend)
{
//--- create variables
int winw=s.m_WindowWidth;
double v=0;
//--- check
if(!CAp::Assert(nticks>=1,__FUNCTION__+": NTicks<1"))
return;
//--- Init
trend.Resize(nticks);
//--- Is it degenerate case?
if(!HasSomethingToAnalyze(s))
{
trend.Fill(0);
return;
}
//--- check
if(!CAp::Assert(s.m_NSequences>0,__FUNCTION__+": integrity check failed"))
return;
if(s.m_SequenceIdx[s.m_NSequences]-s.m_SequenceIdx[s.m_NSequences-1]<winw)
{
trend.Fill(0);
return;
}
if(winw==1)
{
//--- check
if(!CAp::Assert(s.m_NSequences>0,__FUNCTION__+": integrity check failed / 2355"))
return;
if(!CAp::Assert(s.m_SequenceIdx[s.m_NSequences]-s.m_SequenceIdx[s.m_NSequences-1]>0,__FUNCTION__+": integrity check failed"))
return;
for(int i=0; i<nticks; i++)
trend.Set(i,s.m_SequenceData[s.m_SequenceIdx[s.m_NSequences]-1]);
return;
}
//--- Update basis and recurrent relation.
//--- It will take care of basis validity flags. AppendLen=0 which means
//--- that we perform initial basis evaluation.
UpdateBasis(s,0,0.0);
//--- check
if(!CAp::Assert(s.m_NBasis<=winw && s.m_NBasis>0,__FUNCTION__+": integrity check failed / 4f5et"))
return;
if(s.m_NBasis==winw)
{
//--- Handle degenerate situation with basis whose size
//--- is equal to window length.
//--- check
if(!CAp::Assert(s.m_NSequences>0,__FUNCTION__+": integrity check failed / 2355"))
return;
if(!CAp::Assert(s.m_SequenceIdx[s.m_NSequences]-s.m_SequenceIdx[s.m_NSequences-1]>0,__FUNCTION__+": integrity check failed"))
return;
for(int i=0; i<nticks; i++)
trend.Set(i,s.m_SequenceData[s.m_SequenceIdx[s.m_NSequences]-1]);
return;
}
//--- Apply recurrent formula for SSA forecasting:
//--- * first, perform smoothing of the last window
//--- * second, perform analysis phase
//--- check
if(!CAp::Assert(s.m_NSequences>0,__FUNCTION__+": integrity check failed"))
return;
if(!CAp::Assert(s.m_SequenceIdx[s.m_NSequences]-s.m_SequenceIdx[s.m_NSequences-1]>=s.m_WindowWidth,__FUNCTION__+": integrity check failed"))
return;
s.m_Tmp0.Resize(s.m_NBasis);
s.m_FcTrend.Resize(s.m_WindowWidth);
CAblas::RMatrixGemVect(s.m_NBasis,s.m_WindowWidth,1.0,s.m_BasisT,0,0,0,s.m_SequenceData,s.m_SequenceIdx[s.m_NSequences]-s.m_WindowWidth,0.0,s.m_Tmp0,0);
CAblas::RMatrixGemVect(s.m_WindowWidth,s.m_NBasis,1.0,s.m_Basis,0,0,0,s.m_Tmp0,0,0.0,s.m_FcTrend,0);
s.m_Tmp1.Resize(winw-1);
for(int i=1; i<winw; i++)
s.m_Tmp1.Set(i-1,s.m_FcTrend[i]);
for(int i=0; i<nticks; i++)
{
v=s.m_ForecastA[0]*s.m_Tmp1[0];
for(int j=1; j<winw-1; j++)
{
v=v+s.m_ForecastA[j]*s.m_Tmp1[j];
s.m_Tmp1.Set(j-1,s.m_Tmp1[j]);
}
trend.Set(i,v);
s.m_Tmp1.Set(winw-2,v);
}
}
//+------------------------------------------------------------------+
//| This function builds SSA basis and performs forecasting for a |
//| user - specified sequence, returning value of trend. |
//| Forecasting is done in two stages: |
//| * first, we extract trend from the WindowWidth last |
//| elements of the sequence. This stage is optional, you can |
//| turn it off if you pass data which are already processed |
//| with SSA. Of course, you can turn it off even for raw |
//| data, but it is not recommended - noise suppression is |
//| very important for correct prediction. |
//| * then, we apply LRR for last WindowWidth - 1 elements of |
//| the extracted trend. |
//| This function has following running time: |
//| * O(NBasis*WindowWidth) for trend extraction phase |
//| * O(WindowWidth*NTicks) for forecast phase |
//| NOTE: this algorithm performs prediction using only one-last - |
//| sliding window. Predictions produced by such approach are |
//| smooth continuations of the reconstructed trend line, but |
//| they can be easily corrupted by noise. If you need noise - |
//| resistant prediction, use SSAForecastAvgSequence() |
//| function, which averages predictions built using several |
//| sliding windows. |
//| INPUT PARAMETERS: |
//| S - SSA model |
//| Data - array[NTicks], data to forecast |
//| DataLen - number of ticks in the data, DataLen >= 1 |
//| ForecastLen - number of ticks to predict, ForecastLen >= 1 |
//| ApplySmoothing - whether to apply smoothing trend extraction or|
//| not; if you do not know what to specify, pass True.|
//| OUTPUT PARAMETERS: |
//| Trend - array[ForecastLen], forecasted trend |
//| CACHING / REUSE OF THE BASIS |
//| Caching / reuse of previous results is performed: |
//| * first call performs full run of SSA; basis is stored in |
//| the cache |
//| * subsequent calls reuse previously cached basis |
//| * if you call any function which changes model properties |
//| (window length, algorithm, dataset), internal basis will |
//| be invalidated. |
//| * the only calls which do NOT invalidate basis are listed |
//| below: |
//| a) SSASetWindow() with same window length |
//| b) SSAAppendPointAndUpdate() |
//| c) SSAAppendSequenceAndUpdate() |
//| d) SSASetAlgoTopK...() with exactly same K |
//| Calling these functions will result in reuse of previously found |
//| basis. |
//| HANDLING OF DEGENERATE CASES |
//| Following degenerate cases may happen: |
//| * dataset is empty(no analysis can be done) |
//| * all sequences are shorter than the window length, no |
//| analysis can be done |
//| * no algorithm is specified(no analysis can be done) |
//| * data sequence is shorter than the WindowWidth(analysis can|
//| be done, but we can not perform forecasting on the last |
//| sequence) |
//| * window lentgh is 1(impossible to use for forecasting) |
//| * SSA analysis algorithm is configured to extract basis |
//| whose size is equal to window length (impossible to use |
//| for forecasting; only basis whose size is less than window|
//| length can be used). |
//| Calling this function in degenerate cases returns following |
//| result: |
//| * ForecastLen copies of the last value is returned for |
//| non-empty task with large enough dataset, but with |
//| overcomplete basis (window width = 1 or basis size is |
//| equal to window width) |
//| * zero trend with length = ForecastLen is returned for empty|
//| task |
//| No analysis is performed in degenerate cases (we immediately |
//| return dummy values, no basis is ever constructed). |
//+------------------------------------------------------------------+
void CSSA::SSAForecastSequence(CSSAModel &s,CRowDouble &data,
int datalen,int forecastlen,
bool applysmoothing,
CRowDouble &trend)
{
//--- create variables
int winw=s.m_WindowWidth;
double v=0;
//--- check
if(!CAp::Assert(datalen>=1,__FUNCTION__+": DataLen<1"))
return;
if(!CAp::Assert(data.Size()>=datalen,__FUNCTION__+": Data is too short"))
return;
if(!CAp::Assert(CApServ::IsFiniteVector(data,datalen),__FUNCTION__+": Data contains infinities NANs"))
return;
if(!CAp::Assert(forecastlen>=1,__FUNCTION__+": ForecastLen<1"))
return;
//--- Init
trend.Resize(forecastlen);
//--- Is it degenerate case?
if(!HasSomethingToAnalyze(s) || datalen<winw)
{
trend.Fill(0);
return;
}
if(winw==1)
{
trend.Fill(data[datalen-1]);
return;
}
//--- Update basis.
//--- It will take care of basis validity flags. AppendLen=0 which means
//--- that we perform initial basis evaluation.
UpdateBasis(s,0,0.0);
//--- check
if(!CAp::Assert(s.m_NBasis<=winw && s.m_NBasis>0,__FUNCTION__+": integrity check failed / 4f5et"))
return;
if(s.m_NBasis==winw)
{
//--- Handle degenerate situation with basis whose size
//--- is equal to window length.
trend.Fill(data[datalen-1]);
return;
}
//--- Perform trend extraction
CApServ::RVectorSetLengthAtLeast(s.m_FcTrend,s.m_WindowWidth);
if(applysmoothing)
{
//--- check
if(!CAp::Assert(datalen>=winw,__FUNCTION__+": integrity check failed"))
return;
CApServ::RVectorSetLengthAtLeast(s.m_Tmp0,s.m_NBasis);
CAblas::RMatrixGemVect(s.m_NBasis,winw,1.0,s.m_BasisT,0,0,0,data,datalen-winw,0.0,s.m_Tmp0,0);
CAblas::RMatrixGemVect(winw,s.m_NBasis,1.0,s.m_Basis,0,0,0,s.m_Tmp0,0,0.0,s.m_FcTrend,0);
}
else
{
for(int i=0; i<winw; i++)
s.m_FcTrend.Set(i,data[datalen+i-winw]);
}
//--- Apply recurrent formula for SSA forecasting
CApServ::RVectorSetLengthAtLeast(s.m_Tmp1,winw-1);
for(int i=1; i<winw; i++)
s.m_Tmp1.Set(i-1,s.m_FcTrend[i]);
for(int i=0; i<forecastlen; i++)
{
v=s.m_ForecastA[0]*s.m_Tmp1[0];
for(int j=1; j<=winw-2; j++)
{
v=v+s.m_ForecastA[j]*s.m_Tmp1[j];
s.m_Tmp1.Set(j-1,s.m_Tmp1[j]);
}
trend.Set(i,v);
s.m_Tmp1.Set(winw-2,v);
}
}
//+------------------------------------------------------------------+
//| This function builds SSA basis and performs forecasting for a |
//| specified number of ticks, returning value of trend. |
//| Forecast is performed as follows: |
//| * SSA trend extraction is applied to last M sliding windows|
//| of the internally stored dataset |
//| * for each of M sliding windows, M predictions are built |
//| * average value of M predictions is returned |
//| This function has following running time: |
//| * O(NBasis*WindowWidth*M) for trend extraction phase |
//| (always performed) |
//| * O(WindowWidth*NTicks*M) for forecast phase |
//| NOTE: noise reduction is ALWAYS applied by this algorithm; if you|
//| want to apply recurrence relation to raw unprocessed data, |
//| use another function - SSAForecastSequence() which allows |
//| to turn on and off noise reduction phase. |
//| NOTE: combination of several predictions results in lesser |
//| sensitivity to noise, but it may produce undesirable |
//| discontinuities between last point of the trend and first |
//| point of the prediction. The reason is that last point of |
//| the trend is usually corrupted by noise, but average value |
//| of several predictions is less sensitive to noise, thus |
//| discontinuity appears. It is not a bug. |
//| INPUT PARAMETERS: |
//| S - SSA model |
//| M - number of sliding windows to combine, M >= 1. If |
//| your dataset has less than M sliding windows, this |
//| parameter will be silently reduced. |
//| NTicks - number of ticks to forecast, NTicks >= 1 |
//| OUTPUT PARAMETERS: |
//| Trend - array[NTicks], predicted trend line |
//| CACHING / REUSE OF THE BASIS |
//| Caching / reuse of previous results is performed: |
//| * first call performs full run of SSA; basis is stored in |
//| the cache |
//| * subsequent calls reuse previously cached basis |
//| * if you call any function which changes model properties |
//| (window length, algorithm, dataset), internal basis will |
//| be invalidated. |
//| * the only calls which do NOT invalidate basis are listed |
//| below: |
//| a) SSASetWindow() with same window length |
//| b) SSAAppendPointAndUpdate() |
//| c) SSAAppendSequenceAndUpdate() |
//| d) SSASetAlgoTopK...() with exactly same K |
//| Calling these functions will result in reuse of previously found |
//| basis. |
//| HANDLING OF DEGENERATE CASES |
//| Following degenerate cases may happen: |
//| * dataset is empty(no analysis can be done) |
//| * all sequences are shorter than the window length, no |
//| analysis can be done |
//| * no algorithm is specified(no analysis can be done) |
//| * last sequence is shorter than the WindowWidth(analysis can|
//| be done, but we can not perform forecasting on the last |
//| sequence) |
//| * window lentgh is 1(impossible to use for forecasting) |
//| * SSA analysis algorithm is configured to extract basis |
//| whose size is equal to window length(impossible to use for|
//| forecasting; only basis whose size is less than window |
//| length can be used). |
//| Calling this function in degenerate cases returns following |
//| result: |
//| * NTicks copies of the last value is returned for non-empty |
//| task with large enough dataset, but with overcomplete basis |
//| (window width = 1 or basis size is equal to window width) |
//| * zero trend with length = NTicks is returned for empty task |
//| No analysis is performed in degenerate cases (we immediately |
//| return dummy values, no basis is ever constructed). |
//+------------------------------------------------------------------+
void CSSA::SSAForecastAvgLast(CSSAModel &s,int m,int nticks,
CRowDouble &trend)
{
int winw=s.m_WindowWidth;
//--- check
if(!CAp::Assert(nticks>=1,__FUNCTION__+": NTicks<1"))
return;
if(!CAp::Assert(m>=1,__FUNCTION__+": M<1"))
return;
//--- Init
trend.Resize(nticks);
//--- Is it degenerate case?
if(!HasSomethingToAnalyze(s))
{
trend.Fill(0);
return;
}
//--- check
if(!CAp::Assert(s.m_NSequences>0,__FUNCTION__+": integrity check failed"))
return;
if(s.m_SequenceIdx[s.m_NSequences]-s.m_SequenceIdx[s.m_NSequences-1]<winw)
{
trend.Fill(0);
return;
}
if(winw==1)
{
//--- check
if(!CAp::Assert(s.m_NSequences>0,__FUNCTION__+": integrity check failed / 2355"))
return;
if(!CAp::Assert(s.m_SequenceIdx[s.m_NSequences]-s.m_SequenceIdx[s.m_NSequences-1]>0,__FUNCTION__+": integrity check failed"))
return;
trend.Fill(s.m_SequenceData[s.m_SequenceIdx[s.m_NSequences]-1]);
return;
}
//--- Update basis and recurrent relation.
//--- It will take care of basis validity flags. AppendLen=0 which means
//--- that we perform initial basis evaluation.
UpdateBasis(s,0,0.0);
//--- check
if(!CAp::Assert(s.m_NBasis<=winw && s.m_NBasis>0,__FUNCTION__+": integrity check failed / 4f5et"))
return;
if(s.m_NBasis==winw)
{
//--- Handle degenerate situation with basis whose size
//--- is equal to window length.
//--- check
if(!CAp::Assert(s.m_NSequences>0,__FUNCTION__+": integrity check failed / 2355"))
return;
if(!CAp::Assert(s.m_SequenceIdx[s.m_NSequences]-s.m_SequenceIdx[s.m_NSequences-1]>0,__FUNCTION__+": integrity check failed"))
return;
trend.Fill(s.m_SequenceData[s.m_SequenceIdx[s.m_NSequences]-1]);
return;
}
//--- Decrease M if we have less than M sliding windows.
//--- Forecast.
m=MathMin(m,s.m_SequenceIdx[s.m_NSequences]-s.m_SequenceIdx[s.m_NSequences-1]-winw+1);
if(!CAp::Assert(m>=1,__FUNCTION__+": integrity check failed"))
return;
ForecastAvgSequence(s,s.m_SequenceData,s.m_SequenceIdx[s.m_NSequences-1],s.m_SequenceIdx[s.m_NSequences],m,nticks,true,trend,0);
}
//+------------------------------------------------------------------+
//| This function builds SSA basis and performs forecasting for a |
//| user - specified sequence, returning value of trend. |
//| Forecasting is done in two stages: |
//| * first, we extract trend from M last sliding windows of the |
//| sequence. This stage is optional, you can turn it off i you |
//| pass data which are already processed with SSA. Of course, |
//| you can turn it off even for raw data, but it is not |
//| recommended - noise suppression is very important for correct|
//| prediction. |
//| * then, we apply LRR independently for M sliding windows |
//| * average of M predictions is returned |
//| This function has following running time: |
//| * O(NBasis*WindowWidth*M) for trend extraction phase |
//| * O(WindowWidth*NTicks*M) for forecast phase |
//| NOTE: combination of several predictions results in lesser |
//| sensitivity to noise, but it may produce undesirable |
//| discontinuities between last point of the trend and first |
//| point of the prediction. The reason is that last point of |
//| the trend is usually corrupted by noise, but average value |
//| of several predictions is less sensitive to noise, thus |
//| discontinuity appears. It is not a bug. |
//| INPUT PARAMETERS: |
//| S - SSA model |
//| Data - array[NTicks], data to forecast |
//| DataLen - number of ticks in the data, DataLen >= 1 |
//| M - number of sliding windows to combine, M >= 1. |
//| If your dataset has less than M sliding windows,|
//| this parameter will be silently reduced. |
//| ForecastLen - number of ticks to predict, ForecastLen >= 1 |
//| ApplySmoothing - whether to apply smoothing trend extraction |
//| or not. if you do not know what to specify, |
//| pass true. |
//| OUTPUT PARAMETERS: |
//| Trend - array[ForecastLen], forecasted trend |
//| CACHING / REUSE OF THE BASIS |
//| Caching / reuse of previous results is performed: |
//| * first call performs full run of SSA; basis is stored in |
//| the cache |
//| * subsequent calls reuse previously cached basis |
//| * if you call any function which changes model properties |
//| (window length, algorithm, dataset), internal basis will be |
//| invalidated. |
//| * the only calls which do NOT invalidate basis are listed |
//| below: |
//| a) SSASetWindow() with same window length |
//| b) SSAAppendPointAndUpdate() |
//| c) SSAAppendSequenceAndUpdate() |
//| d) SSASetAlgoTopK...() with exactly same K |
//| Calling these functions will result in reuse of previously found |
//| basis. |
//| HANDLING OF DEGENERATE CASES |
//| Following degenerate cases may happen: |
//| * dataset is empty(no analysis can be done) |
//| * all sequences are shorter than the window length, no analysis|
//| can be done |
//| * no algorithm is specified(no analysis can be done) |
//| * data sequence is shorter than the WindowWidth (analysis can |
//| be done, but we can not perform forecasting on the last |
//| sequence) |
//| * window lentgh is 1 (impossible to use for forecasting) |
//| * SSA analysis algorithm is configured to extract basis whose |
//| size is equal to window length (impossible to use for |
//| forecasting; only basis whose size is less than window length|
//| can be used). |
//| Calling this function in degenerate cases returns following |
//| result: |
//| * ForecastLen copies of the last value is returned for |
//| non-empty task with large enough dataset, but with |
//| overcomplete basis (window width = 1 or basis size |
//| is equal to window width) |
//| * zero trend with length = ForecastLen is returned for |
//| empty task |
//| No analysis is performed in degenerate case s(we immediately |
//| return dummy values, no basis is ever constructed). |
//+------------------------------------------------------------------+
void CSSA::SSAForecastAvgSequence(CSSAModel &s,CRowDouble &data,
int datalen,int m,
int forecastlen,bool applysmoothing,
CRowDouble &trend)
{
int winw=s.m_WindowWidth;
//--- check
if(!CAp::Assert(datalen>=1,__FUNCTION__+": DataLen<1"))
return;
if(!CAp::Assert(m>=1,__FUNCTION__+": M<1"))
return;
if(!CAp::Assert(data.Size()>=datalen,__FUNCTION__+": Data is too short"))
return;
if(!CAp::Assert(CApServ::IsFiniteVector(data,datalen),__FUNCTION__+": Data contains infinities NANs"))
return;
if(!CAp::Assert(forecastlen>=1,__FUNCTION__+": ForecastLen<1"))
return;
//--- Init
trend.Resize(forecastlen);
//--- Is it degenerate case?
if(!HasSomethingToAnalyze(s) || datalen<winw)
{
trend.Fill(0);
return;
}
if(winw==1)
{
trend.Fill(data[datalen-1]);
return;
}
//--- Update basis.
//--- It will take care of basis validity flags. AppendLen=0 which means
//--- that we perform initial basis evaluation.
UpdateBasis(s,0,0.0);
//--- check
if(!CAp::Assert(s.m_NBasis<=winw && s.m_NBasis>0,__FUNCTION__+": integrity check failed / 4f5et"))
return;
if(s.m_NBasis==winw)
{
//--- Handle degenerate situation with basis whose size
//--- is equal to window length.
trend.Fill(data[datalen-1]);
return;
}
//--- Decrease M if we have less than M sliding windows.
//--- Forecast.
m=MathMin(m,datalen-winw+1);
//--- check
if(!CAp::Assert(m>=1,__FUNCTION__+": integrity check failed"))
return;
ForecastAvgSequence(s,data,0,datalen,m,forecastlen,applysmoothing,trend,0);
}
//+------------------------------------------------------------------+
//| This function evaluates current model and tells whether we have |
//| some data which can be analyzed by current algorithm, or not. |
//| No analysis can be done in the following degenerate cases: |
//| * dataset is empty |
//| * all sequences are shorter than the window length |
//| * no algorithm is specified |
//+------------------------------------------------------------------+
bool CSSA::HasSomethingToAnalyze(CSSAModel &s)
{
//--- create variables
bool result;
bool allsmaller=true;
bool isdegenerate=false;
//--- initialization
isdegenerate=isdegenerate || s.m_AlgoType==0;
isdegenerate=isdegenerate || s.m_NSequences==0;
//---
for(int i=0; i<s.m_NSequences; i++)
allsmaller=allsmaller && s.m_SequenceIdx[i+1]-s.m_SequenceIdx[i]<s.m_WindowWidth;
isdegenerate=isdegenerate || allsmaller;
result=!isdegenerate;
//--- return result
return(result);
}
//+------------------------------------------------------------------+
//| This function checks whether I-th sequence is big enough for |
//| analysis or not. |
//| I = -1 is used to denote last sequence(for NSequences = 0) |
//+------------------------------------------------------------------+
bool CSSA::IsSequenceBigEnough(CSSAModel &s,int i)
{
//--- check
if(!CAp::Assert(i>=-1 && i<s.m_NSequences))
return(false);
if(s.m_NSequences==0)
return(false);
//---
if(i<0)
i=s.m_NSequences-1;
//--- return result
return (s.m_SequenceIdx[i+1]-s.m_SequenceIdx[i]>=s.m_WindowWidth);
}
//+------------------------------------------------------------------+
//| This function performs basis update. Either full update |
//| (recalculated from the very beginning) or partial update |
//| (handles append to the end of the dataset). |
//| With AppendLen = 0 this function behaves as follows: |
//| * if AreBasisAndSolverValid = False, then solver object is |
//| created from scratch, initial calculations are performed |
//| according to specific SSA algorithm being chosen. Basis / |
//| Solver validity flag is set to True, then we immediately |
//| return. |
//| * if AreBasisAndSolverValid = True, then nothing is done - we |
//| immediately return. |
//| With AppendLen > 0 this function behaves as follows: |
//| * if AreBasisAndSolverValid = False, then exception is |
//| generated; you can append points only to fully constructed |
//| basis. Call this function with zero AppendLen BEFORE append, |
//| then perform append, then call it one more time with non-zero|
//| AppendLen. |
//| * if AreBasisAndSolverValid = True, then basis is incrementally|
//| updated. It also updates recurrence relation used for |
//| prediction. It is expected that either AppendLen = 1, or |
//| AppendLen = length (last_sequence). Basis update is |
//| performed with probability UpdateIts (larger-than-one values |
//| mean that some amount of iterations is always performed). |
//| In any case, after calling this function we either: |
//| * have an exception |
//| * have completely valid basis |
//| IMPORTANT: this function expects that we do NOT call it for |
//| degenerate tasks (no data). So, call it after check |
//| with HasSomethingToAnalyze() returned True. |
//+------------------------------------------------------------------+
void CSSA::UpdateBasis(CSSAModel &s,int appendlen,double updateits)
{
//--- create variables
int i=0;
int j=0;
int k=0;
int srcoffs=0;
int dstoffs=0;
int winw=s.m_WindowWidth;
int windowstotal=0;
int requesttype=0;
int requestsize=0;
double v=0;
bool degeneraterecurrence=false;
double nu2=0;
int subspaceits=0;
bool needevd=false;
//--- Critical checks
if(!CAp::Assert(appendlen>=0,__FUNCTION__+": incorrect parameters passed to UpdateBasis(),integrity check failed"))
return;
if(!CAp::Assert(!(!s.m_AreBasisAndSolverValid && appendlen!=0),__FUNCTION__+": incorrect parameters passed to UpdateBasis(),integrity check failed"))
return;
if(!CAp::Assert(!(appendlen==0 && updateits>0.0),__FUNCTION__+": incorrect parameters passed to UpdateBasis(),integrity check failed"))
return;
//--- Everything is OK, nothing to do
if(s.m_AreBasisAndSolverValid && appendlen==0)
return;
//--- Seed RNG with fixed or random seed.
//--- RNG used when pseudorandomly deciding whether
//--- to re-evaluate basis or not. Sandom seed is
//--- important when we have several simultaneously
//--- calculated SSA models - we do not want them
//--- to be re-evaluated in same moments).
if(!s.m_AreBasisAndSolverValid)
{
if(s.m_RngSeed>0)
CHighQualityRand::HQRndSeed(s.m_RngSeed,s.m_RngSeed+235,s.m_RS);
else
CHighQualityRand::HQRndRandomize(s.m_RS);
}
//--- Compute XXT for algorithms which need it
if(!s.m_AreBasisAndSolverValid)
{
//--- check
if(!CAp::Assert(appendlen==0,__FUNCTION__+": integrity check failed / 34cx6"))
return;
switch(s.m_AlgoType)
{
case 2:
//--- Compute X*X^T for direct algorithm.
//--- Quite straightforward, no subtle optimizations.
s.m_XXT=matrix<double>::Zeros(winw,winw);
windowstotal=0;
for(i=0; i<s.m_NSequences; i++)
windowstotal+=MathMax(s.m_SequenceIdx[i+1]-s.m_SequenceIdx[i]-winw+1,0);
//--- check
if(!CAp::Assert(windowstotal>0,__FUNCTION__+": integrity check in UpdateBasis() failed / 76t34"))
return;
UpdateXXTPrepare(s,windowstotal,winw,s.m_MemoryLimit);
for(i=0; i<s.m_NSequences; i++)
{
for(j=0; j<MathMax(s.m_SequenceIdx[i+1]-s.m_SequenceIdx[i]-winw+1,0); j++)
UpdateXXTSend(s,s.m_SequenceData,s.m_SequenceIdx[i]+j,s.m_XXT);
}
UpdateXXTFinalize(s,s.m_XXT);
break;
case 3:
//--- Compute X*X^T for real-time algorithm:
//--- * prepare queue of windows to merge into XXT
//--- * shuffle queue in order to avoid time-related biases in algorithm
//--- * dequeue first chunk
s.m_XXT.Resize(winw,winw);
windowstotal=0;
for(i=0; i<s.m_NSequences; i++)
windowstotal+=MathMax(s.m_SequenceIdx[i+1]-s.m_SequenceIdx[i]-winw+1,0);
//--- check
if(!CAp::Assert(windowstotal>0,__FUNCTION__+": integrity check in UpdateBasis() failed / 76t34"))
return;
CApServ::IVectorSetLengthAtLeast(s.m_Rtqueue,windowstotal);
dstoffs=0;
for(i=0; i<s.m_NSequences; i++)
for(j=0; j<MathMax(s.m_SequenceIdx[i+1]-s.m_SequenceIdx[i]-winw+1,0); j++)
{
srcoffs=s.m_SequenceIdx[i]+j;
s.m_Rtqueue.Set(dstoffs,srcoffs);
dstoffs++;
}
//--- check
if(!CAp::Assert(dstoffs==windowstotal,__FUNCTION__+": integrity check in UpdateBasis() failed / fh45f"))
return;
if(s.m_RtPowerUp>1)
//--- Shuffle queue, it helps to avoid time-related bias in algorithm
for(i=0; i<windowstotal; i++)
{
j=i+CHighQualityRand::HQRndUniformI(s.m_RS,windowstotal-i);
s.m_Rtqueue.Swap(i,j);
}
s.m_RtqueueCnt=windowstotal;
s.m_RtqueueChunk=1;
s.m_RtqueueChunk=MathMax(s.m_RtqueueChunk,s.m_RtqueueCnt/s.m_RtPowerUp);
s.m_RtqueueChunk=MathMax(s.m_RtqueueChunk,2*s.m_TopK);
RealtimeDequeue(s,0.0,MathMin(s.m_RtqueueChunk,s.m_RtqueueCnt));
break;
}
}
//--- Handle possible updates for XXT:
//--- * check that append involves either last point of last sequence,
//--- or entire last sequence
//--- * if last sequence is shorter than window width, perform quick exit -
//--- we have nothing to update - no windows to insert into XXT
//--- * update XXT
if(appendlen>0)
{
//--- check
if(!CAp::Assert(s.m_AreBasisAndSolverValid,__FUNCTION__+": integrity check failed / 5gvz3"))
return;
if(!CAp::Assert(s.m_NSequences>=1,__FUNCTION__+": integrity check failed / 658ev"))
return;
if(!CAp::Assert(appendlen==1 || appendlen==s.m_SequenceIdx[s.m_NSequences]-s.m_SequenceIdx[s.m_NSequences-1]-winw+1,__FUNCTION__+": integrity check failed / sd3g7"))
return;
if(s.m_SequenceIdx[s.m_NSequences]-s.m_SequenceIdx[s.m_NSequences-1]<winw)
//--- Last sequence is too short, nothing to update
return;
if(s.m_AlgoType==2 || s.m_AlgoType==3)
{
if(appendlen>1)
{
//--- Long append, use GEMM for updates
UpdateXXTPrepare(s,appendlen,winw,s.m_MemoryLimit);
for(j=0; j<MathMax(s.m_SequenceIdx[s.m_NSequences]-s.m_SequenceIdx[s.m_NSequences-1]-winw+1,0); j++)
UpdateXXTSend(s,s.m_SequenceData,s.m_SequenceIdx[s.m_NSequences-1]+j,s.m_XXT);
UpdateXXTFinalize(s,s.m_XXT);
}
else
{
//--- Just one element is added, use rank-1 update
CAblas::RMatrixGer(winw,winw,s.m_XXT,0,0,1.0,s.m_SequenceData,s.m_SequenceIdx[s.m_NSequences]-winw,s.m_SequenceData,s.m_SequenceIdx[s.m_NSequences]-winw);
}
}
}
//--- Now, perform basis calculation - either full recalculation (AppendLen=0)
//--- or quick update (AppendLen>0).
switch(s.m_AlgoType)
{
case 1:
//--- Precomputed basis
if(winw!=s.m_PrecomputedWidth)
{
//--- Window width has changed, reset basis to zeros
s.m_NBasis=1;
s.m_Basis-matrix<double>::Zeros(winw,1);
s.m_SV=vector<double>::Zeros(1);
}
else
{
//--- OK, use precomputed basis
s.m_NBasis=s.m_PrecomputedNBasis;
s.m_Basis=s.m_PrecomputedBasis;
s.m_SV=vector<double>::Zeros(s.m_NBasis);
}
s.m_BasisT=s.m_Basis.Transpose()+0;
break;
case 2:
//--- Direct top-K algorithm
//--- Calculate eigenvectors with SMatrixEVD(), reorder by descending
//--- of magnitudes.
//--- Update is performed for invalid basis or for non-zero UpdateIts.
needevd=!s.m_AreBasisAndSolverValid;
needevd=needevd || updateits>=1.0;
needevd=needevd || CHighQualityRand::HQRndUniformR(s.m_RS)<(double)(updateits-(int)MathFloor(updateits));
if(needevd)
{
s.m_DbgCntEVD++;
s.m_NBasis=MathMin(winw,s.m_TopK);
if(!CEigenVDetect::SMatrixEVD(s.m_XXT,winw,1,true,s.m_SV,s.m_Basis))
{
CAp::Assert(false,__FUNCTION__+": SMatrixEVD failed");
return;
}
for(i=0; i<winw; i++)
{
k=winw-1-i;
if(i>=k)
break;
s.m_SV.Swap(i,k);
s.m_Basis.SwapCols(i,k);
}
for(i=0; i<=s.m_NBasis-1; i++)
s.m_SV.Set(i,MathSqrt(MathMax(s.m_SV[i],0.0)));
s.m_BasisT=s.m_Basis.Transpose()+0;
}
break;
case 3:
//--- Real-time top-K.
//--- Determine actual number of basis components, prepare subspace
//--- solver (either create from scratch or reuse).
//--- Update is always performed for invalid basis; for a valid basis
//--- it is performed with probability UpdateIts.
if(s.m_RtPowerUp==1)
subspaceits=s.m_DefaultSubspaceits;
else
subspaceits=3;
if(appendlen>0)
{
//--- check
if(!CAp::Assert(s.m_AreBasisAndSolverValid,__FUNCTION__+": integrity check in UpdateBasis() failed / srg6f"))
return;
if(!CAp::Assert(updateits>=0.0,__FUNCTION__+": integrity check in UpdateBasis() failed / srg4f"))
return;
subspaceits=(int)MathFloor(updateits);
if(CHighQualityRand::HQRndUniformR(s.m_RS)<(double)(updateits-(int)MathFloor(updateits)))
subspaceits++;
//--- check
if(!CAp::Assert(subspaceits>=0,__FUNCTION__+": integrity check in UpdateBasis() failed / srg9f"))
return;
//--- Dequeue pending dataset and merge it into XXT.
//--- Dequeuing is done only for appends, and only when we have
//--- non-empty queue.
if(s.m_RtqueueCnt>0)
RealtimeDequeue(s,1.0,MathMin(s.m_RtqueueChunk,s.m_RtqueueCnt));
}
//--- Now, proceed to solver
if(subspaceits>0)
{
if(appendlen==0)
{
s.m_NBasis=MathMin(winw,s.m_TopK);
CEigenVDetect::EigSubSpaceCreateBuf(winw,s.m_NBasis,s.m_Solver);
}
else
CEigenVDetect::EigSubSpaceSetWarmStart(s.m_Solver,true);
CEigenVDetect::EigSubSpaceSetCond(s.m_Solver,0.0,subspaceits);
//--- Perform initial basis estimation
s.m_DbgCntEVD++;
CEigenVDetect::EigSubSpaceOOCStart(s.m_Solver,0);
while(CEigenVDetect::EigSubSpaceOOCContinue(s.m_Solver))
{
CEigenVDetect::EigSubSpaceOOCGetRequestInfo(s.m_Solver,requesttype,requestsize);
//--- check
if(!CAp::Assert(requesttype==0,__FUNCTION__+": integrity check in UpdateBasis() failed / 346372"))
return;
CAblas::RMatrixGemm(winw,requestsize,winw,1.0,s.m_XXT,0,0,0,s.m_Solver.m_X,0,0,0,0.0,s.m_Solver.m_AX,0,0);
}
CEigenVDetect::EigSubSpaceOOCStop(s.m_Solver,s.m_SV,s.m_Basis,s.m_SolverRep);
s.m_SV.Clip(0,DBL_MAX);
s.m_SV=s.m_SV.Sqrt()+0;
s.m_BasisT=s.m_Basis.Transpose()+0;
}
break;
default:
CAp::Assert(false,__FUNCTION__+": integrity check in UpdateBasis() failed / dfgs34");
break;
}
//--- Update recurrent relation
CApServ::RVectorSetLengthAtLeast(s.m_ForecastA,MathMax(winw-1,1));
degeneraterecurrence=false;
if(winw>1)
{
//--- Non-degenerate case
s.m_Tmp0=s.m_BasisT.Col(winw-1)+0;
s.m_Tmp0.Resize(s.m_NBasis);
nu2=(s.m_Tmp0.Pow(2.0)+0).Sum();
if(nu2<(1.0-1000*CMath::m_machineepsilon))
CAblas::RMatrixGemVect(winw-1,s.m_NBasis,1/(1-nu2),s.m_BasisT,0,0,1,s.m_Tmp0,0,0.0,s.m_ForecastA,0);
else
degeneraterecurrence=true;
}
else
degeneraterecurrence=true;
if(degeneraterecurrence)
{
s.m_ForecastA.Fill(0.0);
s.m_ForecastA.Set(MathMax(winw-2,0),1.0);
}
//--- Set validity flag
s.m_AreBasisAndSolverValid=true;
}
//+------------------------------------------------------------------+
//| This function performs analysis using current basis. It assumes |
//| and checks that validity flag AreBasisAndSolverValid is set. |
//| INPUT PARAMETERS: |
//| S - model |
//| Data - array which holds data in elements [I0, I1): |
//| * right bound is not included. |
//| * I1 - I0 >= WindowWidth(assertion is performed). |
//| Trend - preallocated output array, large enough |
//| Noise - preallocated output array, large enough |
//| Offs - offset in Trend / Noise where result is stored; |
//| I1 - I0 elements are written starting at offset |
//| Offs. |
//| OUTPUT PARAMETERS: |
//| Trend, Noise - processing results |
//+------------------------------------------------------------------+
void CSSA::AnalyzeSequence(CSSAModel &s,CRowDouble &data,
int i0,int i1,
CRowDouble &trend,
CRowDouble &noise,
int offs)
{
//--- create variables
int winw=0;
int nwindows=0;
int cnt=0;
int batchstart=0;
int batchlimit=0;
int batchsize=0;
//--- check
if(!CAp::Assert(s.m_AreBasisAndSolverValid,__FUNCTION__+": integrity check failed / d84sz0"))
return;
if(!CAp::Assert(i1-i0>=s.m_WindowWidth,__FUNCTION__+": integrity check failed / d84sz1"))
return;
if(!CAp::Assert(s.m_NBasis>=1,__FUNCTION__+": integrity check failed / d84sz2"))
return;
//--- initialization
nwindows=i1-i0-s.m_WindowWidth+1;
winw=s.m_WindowWidth;
batchlimit=MathMax(nwindows,1);
if(s.m_MemoryLimit>0)
batchlimit=MathMin(batchlimit,MathMax(s.m_MemoryLimit/winw,4*winw));
//--- Zero-initialize trend and counts
cnt=i1-i0;
CApServ::IVectorSetLengthAtLeast(s.m_AseqCounts,cnt);
s.m_AseqCounts.Fill(0,0,cnt);
for(int i=0; i<cnt; i++)
trend.Set(offs+i,0.0);
//--- Reset temporaries if algorithm settings changed since last round
if(s.m_AseqTrajectory.Cols()!=winw)
s.m_AseqTrajectory.Resize(0,0);
if(s.m_AseqTbProduct.Cols()!=s.m_NBasis)
s.m_AseqTbProduct.Resize(0,0);
//--- Perform batch processing
CApServ::RMatrixSetLengthAtLeast(s.m_AseqTrajectory,batchlimit,winw);
CApServ::RMatrixSetLengthAtLeast(s.m_AseqTbProduct,batchlimit,s.m_NBasis);
batchsize=0;
batchstart=offs;
for(int i=0; i<nwindows; i++)
{
//--- Enqueue next row of trajectory matrix
if(batchsize==0)
batchstart=i;
for(int j=0; j<winw; j++)
s.m_AseqTrajectory.Set(batchsize,j,data[i0+i+j]);
batchsize++;
//--- Process batch
if(batchsize==batchlimit || i==nwindows-1)
{
//--- Project onto basis
CAblas::RMatrixGemm(batchsize,s.m_NBasis,winw,1.0,s.m_AseqTrajectory,0,0,0,s.m_BasisT,0,0,1,0.0,s.m_AseqTbProduct,0,0);
CAblas::RMatrixGemm(batchsize,winw,s.m_NBasis,1.0,s.m_AseqTbProduct,0,0,0,s.m_BasisT,0,0,0,0.0,s.m_AseqTrajectory,0,0);
//--- Hankelize
for(int k=0; k<batchsize; k++)
for(int j=0; j<winw; j++)
{
trend.Add(offs+batchstart+k+j,s.m_AseqTrajectory.Get(k,j));
s.m_AseqCounts.Set(batchstart+k+j,s.m_AseqCounts[batchstart+k+j]+1);
}
//--- Reset batch size
batchsize=0;
}
}
for(int i=0; i<cnt; i++)
{
trend.Mul(offs+i,1.0/s.m_AseqCounts[i]);
noise.Set(offs+i,data[i0+i]-trend[offs+i]);
}
}
//+------------------------------------------------------------------+
//| This function performs averaged forecasting. It assumes that |
//| basis is already built, everything is valid and checked. See |
//| comments on similar public functions to find out more about |
//| averaged predictions. |
//| INPUT PARAMETERS: |
//| S - model |
//| Data - array which holds data in elements [I0, I1): |
//| * right bound is not included. |
//| * I1 - I0 >= WindowWidth(assertion is performed)|
//| M - number of sliding windows to combine, M >= 1. |
//| If your dataset has less than M sliding windows,|
//| this parameter will be silently reduced. |
//| ForecastLen - number of ticks to predict, ForecastLen >= 1 |
//| Trend - preallocated output array, large enough |
//| Offs - offset in Trend where result is stored; |
//| I1 - I0 elements are written starting at offset |
//| Offs. |
//| OUTPUT PARAMETERS: |
//| Trend - array[ForecastLen], forecasted trend |
//+------------------------------------------------------------------+
void CSSA::ForecastAvgSequence(CSSAModel &s,CRowDouble &data,
int i0,int i1,int m,
int forecastlen,
bool smooth,
CRowDouble &trend,
int offs)
{
int winw=s.m_WindowWidth;
//--- check
if(!CAp::Assert(s.m_AreBasisAndSolverValid,__FUNCTION__+": integrity check failed / d84sz0"))
return;
if(!CAp::Assert(i1-i0-s.m_WindowWidth+1>=m,__FUNCTION__+": integrity check failed / d84sz1"))
return;
if(!CAp::Assert(s.m_NBasis>=1,__FUNCTION__+": integrity check failed / d84sz2"))
return;
if(!CAp::Assert(s.m_WindowWidth>=2,__FUNCTION__+": integrity check failed / 5tgdg5"))
return;
if(!CAp::Assert(s.m_WindowWidth>s.m_NBasis,__FUNCTION__+": integrity check failed / d5g56w"))
return;
//--- Prepare M synchronized predictions for the last known tick
//--- (last one is an actual value of the trend, previous M-1 predictions
//--- are predictions from differently positioned sliding windows).
CApServ::RMatrixSetLengthAtLeast(s.m_FcTrendM,m,winw);
CApServ::RVectorSetLengthAtLeast(s.m_Tmp0,MathMax(m,s.m_NBasis));
CApServ::RVectorSetLengthAtLeast(s.m_Tmp1,winw);
for(int k=0; k<m; k++)
{
//--- Perform prediction for rows [0,K-1]
CAblas::RMatrixGemVect(k,winw-1,1.0,s.m_FcTrendM,0,1,0,s.m_ForecastA,0,0.0,s.m_Tmp0,0);
for(int i=0; i<k; i++)
{
for(int j=1; j<winw; j++)
s.m_FcTrendM.Set(i,j-1,s.m_FcTrendM.Get(i,j));
s.m_FcTrendM.Set(i,winw-1,s.m_Tmp0[i]);
}
//--- Perform trend extraction for row K, add it to dataset
if(smooth)
{
CAblas::RMatrixGemVect(s.m_NBasis,winw,1.0,s.m_BasisT,0,0,0,data,i1-winw-(m-1-k),0.0,s.m_Tmp0,0);
CAblas::RMatrixGemVect(s.m_WindowWidth,s.m_NBasis,1.0,s.m_Basis,0,0,0,s.m_Tmp0,0,0.0,s.m_Tmp1,0);
if(s.m_FcTrendM.Cols()==s.m_Tmp1.Size())
s.m_FcTrendM.Row(k,s.m_Tmp1);
else
for(int j=0; j<winw; j++)
s.m_FcTrendM.Set(k,j,s.m_Tmp1[j]);
}
else
for(int j=0; j<winw; j++)
s.m_FcTrendM.Set(k,j,data[i1-winw-(m-1-k)+j]);
}
//--- Now we have M synchronized predictions of the sequence state at the last
//--- know moment (last "prediction" is just a copy of the trend). Let's start
//--- batch prediction!
for(int k=0; k<forecastlen; k++)
{
CAblas::RMatrixGemVect(m,winw-1,1.0,s.m_FcTrendM,0,1,0,s.m_ForecastA,0,0.0,s.m_Tmp0,0);
trend.Set(offs+k,0.0);
for(int i=0; i<=m-1; i++)
{
for(int j=1; j<winw; j++)
s.m_FcTrendM.Set(i,j-1,s.m_FcTrendM.Get(i,j));
s.m_FcTrendM.Set(i,winw-1,s.m_Tmp0[i]);
trend.Add(offs+k,s.m_Tmp0[i]);
}
trend.Mul(offs+k,1.0/m);
}
}
//+------------------------------------------------------------------+
//| This function extracts updates from real-time queue and applies |
//| them to the S.XXT matrix. XXT is premultiplied by Beta, which can|
//| be 0.0 for initial creation, 1.0 for subsequent updates, or even |
//| within(0, 1) for some kind of updates with decay. |
//| INPUT PARAMETERS: |
//| S - model |
//| Beta - >= 0, coefficient to premultiply XXT |
//| Cnt - 0 < Cnt <= S.RTQueueCnt, number of updates |
//| to extract from the end of the queue |
//| OUTPUT PARAMETERS: |
//| S - S.XXT updated, S.RTQueueCnt decreased |
//+------------------------------------------------------------------+
void CSSA::RealtimeDequeue(CSSAModel &s,double beta,int cnt)
{
int winw=s.m_WindowWidth;
//--- check
if(!CAp::Assert(cnt>0,__FUNCTION__+": RealTimeDequeue() integrity check failed / 43tdv"))
return;
if(!CAp::Assert(MathIsValidNumber(beta) && beta>=0.0,__FUNCTION__+": RealTimeDequeue() integrity check failed / 5gdg6"))
return;
if(!CAp::Assert(cnt<=s.m_RtqueueCnt,__FUNCTION__+": RealTimeDequeue() integrity check failed / 547yh"))
return;
if(!CAp::Assert(s.m_XXT.Cols()>=s.m_WindowWidth,__FUNCTION__+": RealTimeDequeue() integrity check failed / 54bf4"))
return;
if(!CAp::Assert(s.m_XXT.Rows()>=s.m_WindowWidth,__FUNCTION__+": RealTimeDequeue() integrity check failed / 9gdfn"))
return;
//--- Premultiply XXT by Beta
if(beta!=0.0)
s.m_XXT*=beta;
else
s.m_XXT.Fill(0,winw,winw);
//--- Dequeue
UpdateXXTPrepare(s,cnt,winw,s.m_MemoryLimit);
for(int i=0; i<cnt; i++)
{
UpdateXXTSend(s,s.m_SequenceData,s.m_Rtqueue[s.m_RtqueueCnt-1],s.m_XXT);
s.m_RtqueueCnt--;
}
UpdateXXTFinalize(s,s.m_XXT);
}
//+------------------------------------------------------------------+
//| This function prepares batch buffer for XXT update. The idea is |
//| that we send a stream of "XXT += u*u'" updates, and we want to |
//| package them into one big matrix update U*U', applied with SYRK()|
//| kernel, but U can consume too much memory, so we want to |
//| transparently divide it into few smaller chunks. |
//| This set of functions solves this problem: |
//| * UpdateXXTPrepare() prepares temporary buffers |
//| * UpdateXXTSend() sends next u to the buffer, |
//| possibly initiating next SYRK() |
//| * UpdateXXTFinalize() performs last SYRK() update |
//| INPUT PARAMETERS: |
//| S - model, only fields with UX prefix are used |
//| UpdateSize - number of updates |
//| WindowWidth - window width, > 0 |
//| MemoryLimit - memory limit, non-positive value means no limit |
//| OUTPUT PARAMETERS: |
//| S - UX temporaries updated |
//+------------------------------------------------------------------+
void CSSA::UpdateXXTPrepare(CSSAModel &s,int updatesize,int windowwidth,
int memorylimit)
{
//--- check
if(!CAp::Assert(windowwidth>0,__FUNCTION__+": WinW<=0"))
return;
s.m_UxBatchLimit=MathMax(updatesize,1);
if(memorylimit>0)
s.m_UxBatchLimit=MathMin(s.m_UxBatchLimit,MathMax(memorylimit/windowwidth,4*windowwidth));
s.m_UxBatchWidth=windowwidth;
s.m_UxBatchSize=0;
if(s.m_UxBatch.Cols()!=windowwidth)
s.m_UxBatch.Resize(0,0);
CApServ::RMatrixSetLengthAtLeast(s.m_UxBatch,s.m_UxBatchLimit,windowwidth);
}
//+------------------------------------------------------------------+
//| This function sends update u*u' to the batch buffer. |
//| INPUT PARAMETERS: |
//| S - model, only fields with UX prefix are used |
//| U - WindowWidth - sized update, starts at I0 |
//| I0 - starting position for update |
//| OUTPUT PARAMETERS: |
//| S - UX temporaries updated |
//| XXT - array[WindowWidth, WindowWidth], in the middle |
//| of update. All intermediate updates are applied |
//| to the upper triangle. |
//+------------------------------------------------------------------+
void CSSA::UpdateXXTSend(CSSAModel &s,CRowDouble &u,int i0,
CMatrixDouble &xxt)
{
//--- check
if(!CAp::Assert(i0+s.m_UxBatchWidth-1<u.Size(),__FUNCTION__+": incorrect U size"))
return;
if(!CAp::Assert(s.m_UxBatchSize>=0,__FUNCTION__+": integrity check failure"))
return;
if(!CAp::Assert(s.m_UxBatchSize<=s.m_UxBatchLimit,__FUNCTION__+": integrity check failure"))
return;
if(!CAp::Assert(s.m_UxBatchLimit>=1,__FUNCTION__+": integrity check failure"))
return;
//--- Send pending batch if full
if(s.m_UxBatchSize==s.m_UxBatchLimit)
{
CAblas::RMatrixSyrk(s.m_UxBatchWidth,s.m_UxBatchSize,1.0,s.m_UxBatch,0,0,2,1.0,xxt,0,0,true);
s.m_UxBatchSize=0;
}
//--- Append update to batch
int i1_=i0;
for(int i_=0; i_<s.m_UxBatchWidth; i_++)
s.m_UxBatch.Set(s.m_UxBatchSize,i_,u[i_+i1_]);
s.m_UxBatchSize++;
}
//+------------------------------------------------------------------+
//| This function finalizes batch buffer. Call it after the last |
//| update. |
//| INPUT PARAMETERS: |
//| S - model, only fields with UX prefix are used |
//| OUTPUT PARAMETERS: |
//| S - UX temporaries updated |
//| XXT - array[WindowWidth, WindowWidth], updated with |
//| all previous updates, both triangles of the |
//| symmetric matrix are present. |
//+------------------------------------------------------------------+
void CSSA::UpdateXXTFinalize(CSSAModel &s,CMatrixDouble &xxt)
{
//--- check
if(!CAp::Assert(s.m_UxBatchSize>=0,__FUNCTION__+": integrity check failure"))
return;
if(!CAp::Assert(s.m_UxBatchSize<=s.m_UxBatchLimit,__FUNCTION__+": integrity check failure"))
return;
if(!CAp::Assert(s.m_UxBatchLimit>=1,__FUNCTION__+": integrity check failure"))
return;
//---
if(s.m_UxBatchSize>0)
{
CAblas::RMatrixSyrk(s.m_UxBatchWidth,s.m_UxBatchSize,1.0,s.m_UxBatch,0,0,2,1.0,s.m_XXT,0,0,true);
s.m_UxBatchSize=0;
}
CAblas::RMatrixEnforceSymmetricity(s.m_XXT,s.m_UxBatchWidth,true);
}
//+------------------------------------------------------------------+
//| Buffer object which is used to perform various requests (usually |
//| model inference) in the multithreaded mode (multiple threads |
//| working with same KNN object). |
//| This object should be created with KNNCreateBuffer(). |
//+------------------------------------------------------------------+
struct CKNNBuffer
{
CKDTreeRequestBuffer m_treebuf;
CRowDouble m_x;
CRowDouble m_y;
CRowInt m_tags;
CMatrixDouble m_xy;
//---
CKNNBuffer(void) {}
~CKNNBuffer(void) {}
void Copy(const CKNNBuffer &obj);
//--- overloading
void operator=(const CKNNBuffer &obj) { Copy(obj); }
};
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CKNNBuffer::Copy(const CKNNBuffer &obj)
{
m_treebuf=obj.m_treebuf;
m_x=obj.m_x;
m_y=obj.m_y;
m_tags=obj.m_tags;
m_xy=obj.m_xy;
}
//+------------------------------------------------------------------+
//| A KNN builder object; this object encapsulates dataset and all |
//| related settings, it is used to create an actual instance of KNN |
//| model. |
//+------------------------------------------------------------------+
struct CKNNBuilder
{
int m_dstype;
int m_npoints;
int m_nvars;
bool m_iscls;
int m_nout;
CMatrixDouble m_dsdata;
CRowDouble m_dsrval;
CRowInt m_dsival;
int m_knnnrm;
//---
CKNNBuilder(void);
~CKNNBuilder(void) {}
void Copy(const CKNNBuilder &obj);
//--- overloading
void operator=(const CKNNBuilder &obj) { Copy(obj); }
};
//+------------------------------------------------------------------+
//| Constructor |
//+------------------------------------------------------------------+
CKNNBuilder::CKNNBuilder(void)
{
m_dstype=0;
m_npoints=0;
m_nvars=0;
m_iscls=false;
m_nout=0;
m_knnnrm=0;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CKNNBuilder::Copy(const CKNNBuilder &obj)
{
m_dstype=obj.m_dstype;
m_npoints=obj.m_npoints;
m_nvars=obj.m_nvars;
m_iscls=obj.m_iscls;
m_nout=obj.m_nout;
m_dsdata=obj.m_dsdata;
m_dsrval=obj.m_dsrval;
m_dsival=obj.m_dsival;
m_knnnrm=obj.m_knnnrm;
}
//+------------------------------------------------------------------+
//| KNN model, can be used for classification or regression |
//+------------------------------------------------------------------+
struct CKNNModel
{
int m_nvars;
int m_nout;
int m_k;
double m_eps;
bool m_iscls;
bool m_isdummy;
CKDTree m_tree;
CKNNBuffer m_buffer;
//---
CKNNModel(void);
~CKNNModel(void) {}
//---
void Copy(const CKNNModel &obj);
//--- overloading
void operator=(const CKNNModel &obj) { Copy(obj); }
};
//+------------------------------------------------------------------+
//| Constructor |
//+------------------------------------------------------------------+
CKNNModel::CKNNModel(void)
{
m_nvars=0;
m_nout=0;
m_k=0;
m_eps=0;
m_iscls=false;
m_isdummy=false;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CKNNModel::Copy(const CKNNModel &obj)
{
m_nvars=obj.m_nvars;
m_nout=obj.m_nout;
m_k=obj.m_k;
m_eps=obj.m_eps;
m_iscls=obj.m_iscls;
m_isdummy=obj.m_isdummy;
m_tree=obj.m_tree;
m_buffer=obj.m_buffer;
}
//+------------------------------------------------------------------+
//| KNN training report. |
//| Following fields store training set errors: |
//| * relclserror - fraction of misclassified cases, [0,1] |
//| * avgce - average cross-entropy in bits per symbol |
//| * rmserror - root-mean-square error |
//| * avgerror - average error |
//| * avgrelerror - average relative error |
//| For classification problems: |
//| * RMS, AVG and AVGREL errors are calculated for posterior |
//| probabilities |
//| For regression problems: |
//| * RELCLS and AVGCE errors are zero |
//+------------------------------------------------------------------+
struct CKNNReport
{
double m_RelCLSError;
double m_AvgCE;
double m_RMSError;
double m_AvgError;
double m_AvgRelError;
//---
CKNNReport(void) { ZeroMemory(this); }
~CKNNReport(void) {}
//---
void Copy(const CKNNReport &obj);
//--- overloading
void operator=(const CKNNReport &obj) { Copy(obj); }
};
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CKNNReport::Copy(const CKNNReport &obj)
{
m_RelCLSError=obj.m_RelCLSError;
m_AvgCE=obj.m_AvgCE;
m_RMSError=obj.m_RMSError;
m_AvgError=obj.m_AvgError;
m_AvgRelError=obj.m_AvgRelError;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
class CKNN
{
public:
//--- constant
static const int m_knnfirstversion;
static void KNNCreateBuffer(CKNNModel &model,CKNNBuffer &buf);
static void KNNBuilderCreate(CKNNBuilder &s);
static void KNNBuilderSetDatasetReg(CKNNBuilder &s,CMatrixDouble &xy,int npoints,int nvars,int nout);
static void KNNBuilderSetDatasetCLS(CKNNBuilder &s,CMatrixDouble &xy,int npoints,int nvars,int nclasses);
static void KNNBuilderSetNorm(CKNNBuilder &s,int nrmtype);
static void KNNBuilderBuildKNNModel(CKNNBuilder &s,int k,double eps,CKNNModel &model,CKNNReport &rep);
static void KNNRewriteKEps(CKNNModel &model,int k,double eps);
static void KNNProcess(CKNNModel &model,CRowDouble &x,CRowDouble &y);
static double KNNProcess0(CKNNModel &model,CRowDouble &x);
static int KNNClassify(CKNNModel &model,CRowDouble &x);
static void KNNProcessI(CKNNModel &model,CRowDouble &x,CRowDouble &y);
static void KNNTsProcess(CKNNModel &model,CKNNBuffer &buf,CRowDouble &x,CRowDouble &y);
static double KNNRelClsError(CKNNModel &model,CMatrixDouble &xy,int npoints);
static double KNNAvgCE(CKNNModel &model,CMatrixDouble &xy,int npoints);
static double KNNRMSError(CKNNModel &model,CMatrixDouble &xy,int npoints);
static double KNNAvgError(CKNNModel &model,CMatrixDouble &xy,int npoints);
static double CKNN::KNNAvgRelError(CKNNModel &model,CMatrixDouble &xy,int npoints);
static void KNNAllErrors(CKNNModel &model,CMatrixDouble &xy,int npoints,CKNNReport &rep);
static void CKNN::KNNAlloc(CSerializer &s,CKNNModel &model);
static void KNNSerialize(CSerializer &s,CKNNModel &model);
static void KNNUnserialize(CSerializer &s,CKNNModel &model);
private:
static void ClearReport(CKNNReport &rep);
static void ProcessInternal(CKNNModel &model,CKNNBuffer &buf);
};
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
const int CKNN::m_knnfirstversion=0;
//+------------------------------------------------------------------+
//| This function creates buffer structure which can be used to |
//| perform parallel KNN requests. |
//| KNN subpackage provides two sets of computing functions - ones |
//| which use internal buffer of KNN model (these functions are |
//| single-threaded because they use same buffer, which can not |
//| shared between threads), and ones which use external buffer. |
//| This function is used to initialize external buffer. |
//| INPUT PARAMETERS |
//| Model - KNN model which is associated with newly created|
//| buffer |
//| OUTPUT PARAMETERS |
//| Buf - external buffer. |
//| IMPORTANT: buffer object should be used only with model which was|
//| used to initialize buffer. Any attempt to use buffer|
//| with different object is dangerous - you may get |
//| integrity check failure (exception) because sizes of |
//| internal arrays do not fit to dimensions of the model |
//| structure. |
//+------------------------------------------------------------------+
void CKNN::KNNCreateBuffer(CKNNModel &model,CKNNBuffer &buf)
{
if(!model.m_isdummy)
CNearestNeighbor::KDTreeCreateRequestBuffer(model.m_tree,buf.m_treebuf);
buf.m_x.Resize(model.m_nvars);
buf.m_y.Resize(model.m_nout);
}
//+------------------------------------------------------------------+
//| This subroutine creates KNNBuilder object which is used to train |
//| KNN models. |
//| By default, new builder stores empty dataset and some reasonable |
//| default settings. At the very least, you should specify dataset |
//| prior to building KNN model. You can also tweak settings of the |
//| model construction algorithm (recommended, although default |
//| settings should work well). |
//| Following actions are mandatory: |
//| * calling knnbuildersetdataset() to specify dataset |
//| * calling knnbuilderbuildknnmodel() to build KNN model using |
//| current dataset and default settings |
//| Additionally, you may call: |
//| * KNNBuilderSetNorm() to change norm being used |
//| INPUT PARAMETERS: |
//| none |
//| OUTPUT PARAMETERS: |
//| S - KNN builder |
//+------------------------------------------------------------------+
void CKNN::KNNBuilderCreate(CKNNBuilder &s)
{
//--- Empty dataset
s.m_dstype=-1;
s.m_npoints=0;
s.m_nvars=0;
s.m_iscls=false;
s.m_nout=1;
//--- Default training settings
s.m_knnnrm=2;
}
//+------------------------------------------------------------------+
//| Specifies regression problem (one or more continuous output |
//| variables are predicted). There also exists "classification" |
//| version of this function. |
//| This subroutine adds dense dataset to the internal storage of the|
//| builder object. Specifying your dataset in the dense format means|
//| that the dense version of the KNN construction algorithm will be |
//| invoked. |
//| INPUT PARAMETERS: |
//| S - KNN builder object |
//| XY - array[NPoints,NVars+NOut] (note: actual size can|
//| be larger, only leading part is used anyway), |
//| dataset: |
//| * first NVars elements of each row store values |
//| of the independent variables |
//| * next NOut elements store values of the |
//| dependent variables |
//| NPoints - number of rows in the dataset, NPoints>=1 |
//| NVars - number of independent variables, NVars>=1 |
//| NOut - number of dependent variables, NOut>=1 |
//| OUTPUT PARAMETERS: |
//| S - KNN builder |
//+------------------------------------------------------------------+
void CKNN::KNNBuilderSetDatasetReg(CKNNBuilder &s,CMatrixDouble &xy,
int npoints,int nvars,int nout)
{
//--- Check parameters
if(!CAp::Assert(npoints>=1,__FUNCTION__+": npoints<1"))
return;
if(!CAp::Assert(nvars>=1,__FUNCTION__+": nvars<1"))
return;
if(!CAp::Assert(nout>=1,__FUNCTION__+": nout<1"))
return;
if(!CAp::Assert(xy.Rows()>=npoints,__FUNCTION__+": rows(xy)<npoints"))
return;
if(!CAp::Assert(xy.Cols()>=nvars+nout,__FUNCTION__+": cols(xy)<nvars+nout"))
return;
if(!CAp::Assert(CApServ::IsFiniteMatrix(xy,npoints,nvars+nout),__FUNCTION__+": xy parameter contains INFs or NANs"))
return;
//--- Set dataset
s.m_dstype=0;
s.m_iscls=false;
s.m_npoints=npoints;
s.m_nvars=nvars;
s.m_nout=nout;
ulong parts[]={nvars,nout};
matrix<double> splited[];
xy.Split(parts,1,splited);
s.m_dsdata=splited[0];
splited[1].Reshape(1,npoints*nout);
s.m_dsrval=splited[1].Row(0);
}
//+------------------------------------------------------------------+
//| Specifies classification problem (two or more classes are |
//| predicted). There also exists "regression" version of this |
//| function. |
//| This subroutine adds dense dataset to the internal storage of the|
//| builder object. Specifying your dataset in the dense format means|
//| that the dense version of the KNN construction algorithm will be |
//| invoked. |
//| INPUT PARAMETERS: |
//| S - KNN builder object |
//| XY - array[NPoints, NVars + 1] (note: actual size can be|
//| larger, only leading part is used anyway), dataset:|
//| * first NVars elements of each row store values of |
//| the independent variables |
//| * next element stores class index, in [0, NClasses)|
//| NPoints - number of rows in the dataset, NPoints >= 1 |
//| NVars - number of independent variables, NVars >= 1 |
//| NClasses - number of classes, NClasses >= 2 |
//| OUTPUT PARAMETERS: |
//| S - KNN builder |
//+------------------------------------------------------------------+
void CKNN::KNNBuilderSetDatasetCLS(CKNNBuilder &s,
CMatrixDouble &xy,
int npoints,int nvars,int nclasses)
{
//--- Check parameters
if(!CAp::Assert(npoints>=1,__FUNCTION__+": npoints<1"))
return;
if(!CAp::Assert(nvars>=1,__FUNCTION__+": nvars<1"))
return;
if(!CAp::Assert(nclasses>=2,__FUNCTION__+": nclasses<2"))
return;
if(!CAp::Assert(xy.Rows()>=npoints,__FUNCTION__+": rows(xy)<npoints"))
return;
if(!CAp::Assert(xy.Cols()>=nvars+1,__FUNCTION__+": cols(xy)<nvars+1"))
return;
if(!CAp::Assert(CApServ::IsFiniteMatrix(xy,npoints,nvars+1),__FUNCTION__+": xy parameter contains INFs or NANs"))
return;
vector<double> y=xy.Col(nvars);
if(!CAp::Assert(y.Min()>=0 && y.Max()<nclasses,__FUNCTION__+": last column of xy contains invalid class number"))
return;
//--- Set dataset
s.m_iscls= true;
s.m_dstype= 0;
s.m_npoints= npoints;
s.m_nvars= nvars;
s.m_nout= nclasses;
ulong parts[]={nvars,nclasses};
matrix<double> splited[];
xy.Split(parts,1,splited);
s.m_dsdata=splited[0];
s.m_dsival.Resize(npoints);
for(int i=0; i<npoints; i++)
s.m_dsival.Set(i,(int)MathRound(y[i]));
}
//+------------------------------------------------------------------+
//| This function sets norm type used for neighbor search. |
//| INPUT PARAMETERS: |
//| S - decision forest builder object |
//| NormType - norm type: |
//| * 0 inf-norm |
//| * 1 1-norm |
//| * 2 Euclidean norm(default) |
//| OUTPUT PARAMETERS: |
//| S - decision forest builder |
//+------------------------------------------------------------------+
void CKNN::KNNBuilderSetNorm(CKNNBuilder &s,int nrmtype)
{
//--- check
if(!CAp::Assert(nrmtype==0 || nrmtype==1 || nrmtype==2,__FUNCTION__+": unexpected norm type"))
return;
s.m_knnnrm=nrmtype;
}
//+------------------------------------------------------------------+
//| This subroutine builds KNN model according to current settings, |
//| using dataset internally stored in the builder object. |
//| The model being built performs inference using Eps-approximate |
//| K nearest neighbors search algorithm, with: |
//| *K=1, Eps=0 corresponding to the "nearest neighbor |
//| algorithm" |
//| *K>1, Eps=0 corresponding to the "K nearest neighbors |
//| algorithm" |
//| *K>=1, Eps>0 corresponding to "approximate nearest |
//| neighbors algorithm" |
//| An approximate KNN is a good option for high-dimensional datasets|
//| (exact KNN works slowly when dimensions count grows). |
//| An ALGLIB implementation of kd-trees is used to perform k-nn |
//| searches. |
//| INPUT PARAMETERS: |
//| S - KNN builder object |
//| K - number of neighbors to search for, K >= 1 |
//| Eps - approximation factor: |
//| * Eps = 0 means that exact kNN search is performed |
//| * Eps > 0 means that(1 + Eps) - approximate search |
//| is performed |
//| OUTPUT PARAMETERS: |
//| Model - KNN model |
//| Rep - report |
//+------------------------------------------------------------------+
void CKNN::KNNBuilderBuildKNNModel(CKNNBuilder &s,int k,double eps,
CKNNModel &model,CKNNReport &rep)
{
//--- create variables
int npoints=s.m_npoints;
int nvars=s.m_nvars;
int nout=s.m_nout;
bool iscls=s.m_iscls;
CMatrixDouble xy;
CRowInt tags;
//--- Check settings
if(!CAp::Assert(k>=1,__FUNCTION__+": k<1"))
return;
if(!CAp::Assert(MathIsValidNumber(eps) && eps>=0.0,__FUNCTION__+": eps<0"))
return;
//--- Prepare output
ClearReport(rep);
model.m_nvars=nvars;
model.m_nout=nout;
model.m_iscls=iscls;
model.m_k=k;
model.m_eps=eps;
model.m_isdummy=false;
//--- Quick exit for empty dataset
if(s.m_dstype==-1)
{
model.m_isdummy=true;
return;
}
//--- Build kd-tree
if(iscls)
{
xy=s.m_dsdata;
xy.Resize(npoints,nvars+1);
xy.Col(nvars,s.m_dsival);
tags=s.m_dsival;
CNearestNeighbor::KDTreeBuildTagged(xy,tags,npoints,nvars,0,s.m_knnnrm,model.m_tree);
}
else
{
xy=s.m_dsdata;
xy.Resize(npoints,nvars+nout);
for(int i=0; i<npoints; i++)
for(int j=0; j<nout; j++)
xy.Set(i,nvars+j,s.m_dsrval[i*nout+j]);
CNearestNeighbor::KDTreeBuild(xy,npoints,nvars,nout,s.m_knnnrm,model.m_tree);
}
//--- Build buffer
KNNCreateBuffer(model,model.m_buffer);
//--- Report
KNNAllErrors(model,xy,npoints,rep);
}
//+------------------------------------------------------------------+
//| Changing search settings of KNN model. |
//| K and EPS parameters of KNN(AKNN) search are specified during |
//| model construction. However, plain KNN algorithm with Euclidean |
//| distance allows you to change them at any moment. |
//| NOTE: future versions of KNN model may support advanced versions |
//| of KNN, such as NCA or LMNN. It is possible that such |
//| algorithms won't allow you to change search settings on the|
//| fly. If you call this function for an algorithm which does |
//| not support on-the-fly changes, it will throw an exception|
//| INPUT PARAMETERS: |
//| Model - KNN model |
//| K - K >= 1, neighbors count |
//| EPS - accuracy of the EPS-approximate NN search. Set |
//| to 0.0, if you want to perform "classic" KNN |
//| search. Specify larger values if you need to |
//| speed-up high-dimensional KNN queries. |
//| OUTPUT PARAMETERS: |
//| nothing on success, exception on failure |
//+------------------------------------------------------------------+
void CKNN::KNNRewriteKEps(CKNNModel &model,int k,double eps)
{
//--- check
if(!CAp::Assert(k>=1,__FUNCTION__+": k<1"))
return;
if(!CAp::Assert(MathIsValidNumber(eps) && eps>=0.0,__FUNCTION__+": eps<0"))
return;
//--- change values
model.m_k=k;
model.m_eps=eps;
}
//+------------------------------------------------------------------+
//| Inference using KNN model. |
//| See also KNNProcess0(), KNNProcessI() and KNNClassify() for |
//| options with a bit more convenient interface. |
//| INPUT PARAMETERS: |
//| Model - KNN model |
//| X - input vector, array[0..NVars - 1]. |
//| Y - possible preallocated buffer. Reused if long enough|
//| OUTPUT PARAMETERS: |
//| Y - result. Regression estimate when solving regression|
//| task, vector of posterior probabilities for |
//| classification task. |
//+------------------------------------------------------------------+
void CKNN::KNNProcess(CKNNModel &model,CRowDouble &x,CRowDouble &y)
{
KNNTsProcess(model,model.m_buffer,x,y);
}
//+------------------------------------------------------------------+
//| This function returns first component of the inferred vector |
//| (i.e.one with index #0). |
//| It is a convenience wrapper for KNNProcess() intended for either:|
//| * 1 - dimensional regression problems |
//| * 2 - class classification problems |
//| In the former case this function returns inference result as |
//| scalar, which is definitely more convenient that wrapping it as |
//| vector. In the latter case it returns probability of object |
//| belonging to class #0. |
//| If you call it for anything different from two cases above, it |
//| will work as defined, i.e. return y[0], although it is of less |
//| use in such cases. |
//| INPUT PARAMETERS: |
//| Model - KNN model |
//| X - input vector, array[0..NVars - 1]. |
//| RESULT: |
//| Y[0] |
//+------------------------------------------------------------------+
double CKNN::KNNProcess0(CKNNModel &model,CRowDouble &x)
{
//--- copy
model.m_buffer.m_x=x;
//--- function call
ProcessInternal(model,model.m_buffer);
//--- return result
return(model.m_buffer.m_y[0]);
}
//+------------------------------------------------------------------+
//| This function returns most probable class number for an input X.|
//| It is same as calling KNNProcess(model, x, y), then determining |
//| i = argmax(y[i]) and returning i. |
//| A class number in [0, NOut) range in returned for classification |
//| problems, -1 is returned when this function is called for |
//| regression problems. |
//| INPUT PARAMETERS: |
//| Model - KNN model |
//| X - input vector, array[0..NVars - 1]. |
//| RESULT: |
//| class number, -1 for regression tasks |
//+------------------------------------------------------------------+
int CKNN::KNNClassify(CKNNModel &model,CRowDouble &x)
{
//--- check
if(!model.m_iscls)
return (-1);
//--- function call
ProcessInternal(model,model.m_buffer);
//--- return result
return ((int)model.m_buffer.m_y.ArgMax());
}
//+------------------------------------------------------------------+
//| 'interactive' variant of KNNProcess() which support constructs |
//| like "y = KNNProcessI(model,x)" and interactive mode of the |
//| interpreter. |
//| This function allocates new array on each call, so it is |
//| significantly slower than its 'non-interactive' counterpart, but |
//| it is more convenient when you call it from command line. |
//+------------------------------------------------------------------+
void CKNN::KNNProcessI(CKNNModel &model,CRowDouble &x,CRowDouble &y)
{
y.Resize(0);
KNNProcess(model,x,y);
}
//+------------------------------------------------------------------+
//| Thread - safe procesing using external buffer for temporaries. |
//| This function is thread-safe(i.e. you can use same KNN model from|
//| multiple threads) as long as you use different buffer objects for|
//| different threads. |
//| INPUT PARAMETERS: |
//| Model - KNN model |
//| Buf - buffer object, must be allocated specifically for |
//| this model with KNNCreateBuffer(). |
//| X - input vector, array[NVars] |
//| OUTPUT PARAMETERS: |
//| Y - result, array[NOut]. Regression estimate when |
//| solving regression task, vector of posterior |
//| probabilities for a classification task. |
//+------------------------------------------------------------------+
void CKNN::KNNTsProcess(CKNNModel &model,CKNNBuffer &buf,CRowDouble &x,
CRowDouble &y)
{
int nout=model.m_nout;
//--- copy
buf.m_x=x;
ProcessInternal(model,buf);
//--- return result
if(y.Size()<=nout)
y=buf.m_y;
else
for(int i=0;i<nout;i++)
y.Set(i,buf.m_y[i]);
}
//+------------------------------------------------------------------+
//| Relative classification error on the test set |
//| INPUT PARAMETERS: |
//| Model - KNN model |
//| XY - test set |
//| NPoints - test set size |
//| RESULT: |
//| percent of incorrectly classified cases. |
//| Zero if model solves regression task. |
//| NOTE: if you need several different kinds of error metrics, it is|
//| better to use KNNAllErrors() which computes all error |
//| metric with just one pass over dataset. |
//+------------------------------------------------------------------+
double CKNN::KNNRelClsError(CKNNModel &model,CMatrixDouble &xy,int npoints)
{
CKNNReport rep;
//--- function call
KNNAllErrors(model,xy,npoints,rep);
//--- return result
return(rep.m_RelCLSError);
}
//+------------------------------------------------------------------+
//| Average cross-entropy (in bits per element) on the test set |
//| INPUT PARAMETERS: |
//| Model - KNN model |
//| XY - test set |
//| NPoints - test set size |
//| RESULT: |
//| CrossEntropy / NPoints. |
//| Zero if model solves regression task. |
//| NOTE: the cross-entropy metric is too unstable when used |
//| to evaluate KNN models (such models can report exactly |
//| zero probabilities), so we do not recommend using it. |
//| NOTE: if you need several different kinds of error metrics, it |
//| is better to use KNNAllErrors() which computes all error |
//| metric with just one pass over dataset. |
//+------------------------------------------------------------------+
double CKNN::KNNAvgCE(CKNNModel &model,CMatrixDouble &xy,int npoints)
{
CKNNReport rep;
//--- function call
KNNAllErrors(model,xy,npoints,rep);
//--- return result
return(rep.m_AvgCE);
}
//+------------------------------------------------------------------+
//| RMS error on the test set. |
//| Its meaning for regression task is obvious. As for classification|
//| problems, RMS error means error when estimating posterior |
//| probabilities. |
//| INPUT PARAMETERS: |
//| Model - KNN model |
//| XY - test set |
//| NPoints - test set size |
//| RESULT: |
//| root mean square error. |
//| NOTE: if you need several different kinds of error metrics, it |
//| is better to use KNNAllErrors() which computes all error |
//| metric with just one pass over dataset. |
//+------------------------------------------------------------------+
double CKNN::KNNRMSError(CKNNModel &model,CMatrixDouble &xy,int npoints)
{
CKNNReport rep;
//--- function call
KNNAllErrors(model,xy,npoints,rep);
//--- return result
return(rep.m_RMSError);
}
//+------------------------------------------------------------------+
//| Average error on the test set |
//| Its meaning for regression task is obvious. As for classification|
//| problems, average error means error when estimating posterior |
//| probabilities. |
//| INPUT PARAMETERS: |
//| Model - KNN model |
//| XY - test set |
//| NPoints - test set size |
//| RESULT: |
//| average error |
//| NOTE: if you need several different kinds of error metrics, it |
//| is better to use KNNAllErrors() which computes all error |
//| metric with just one pass over dataset. |
//+------------------------------------------------------------------+
double CKNN::KNNAvgError(CKNNModel &model,CMatrixDouble &xy,int npoints)
{
CKNNReport rep;
//--- function call
KNNAllErrors(model,xy,npoints,rep);
//--- return result
return(rep.m_AvgError);
}
//+------------------------------------------------------------------+
//| Average relative error on the test set |
//| Its meaning for regression task is obvious. As for classification|
//| problems, average relative error means error when estimating |
//| posterior probabilities. |
//| INPUT PARAMETERS: |
//| Model - KNN model |
//| XY - test set |
//| NPoints - test set size |
//| RESULT: |
//| average relative error |
//| NOTE: if you need several different kinds of error metrics, it |
//| is better to use KNNAllErrors() which computes all error |
//| metric with just one pass over dataset. |
//+------------------------------------------------------------------+
double CKNN::KNNAvgRelError(CKNNModel &model,CMatrixDouble &xy,int npoints)
{
CKNNReport rep;
//--- function call
KNNAllErrors(model,xy,npoints,rep);
//--- return result
return(rep.m_AvgRelError);
}
//+------------------------------------------------------------------+
//| Calculates all kinds of errors for the model in one call. |
//| INPUT PARAMETERS: |
//| Model - KNN model |
//| XY - test set: |
//| * one row per point |
//| * first NVars columns store independent variables |
//| * depending on problem type: |
//| * next column stores class number |
//| in [0, NClasses) - for classification |
//| problems |
//| * next NOut columns store dependent |
//| variables - for regression problems |
//| NPoints - test set size, NPoints >= 0 |
//| OUTPUT PARAMETERS: |
//| Rep - following fields are loaded with errors for both |
//| regression and classification models: |
//| * rep.RMSError - RMS error for the output |
//| * rep.AvgError - average error |
//| * rep.AvgRelError - average relative error |
//| following fields are set only for classification |
//| models, zero for regression ones: |
//| * relclserror - relative classification error, |
//| in [0, 1] |
//| * avgce - average cross-entropy in bits per |
//| dataset entry |
//| NOTE: the cross-entropy metric is too unstable when used to |
//| evaluate KNN models (such models can report exactly zero |
//| probabilities), so we do not recommend using it. |
//+------------------------------------------------------------------+
void CKNN::KNNAllErrors(CKNNModel &model,CMatrixDouble &xy,
int npoints,CKNNReport &rep)
{
//--- Clean up report
ClearReport(rep);
//--- Quick exit if needed
if(model.m_isdummy || npoints==0)
return;
//--- create variables
CKNNBuffer buf;
CRowDouble desiredy;
CRowDouble errbuf;
int ny=0;
int j=0;
int nvars=model.m_nvars;
int nout=model.m_nout;
bool iscls=model.m_iscls;
//---
if(iscls)
ny=1;
else
ny=nout;
//--- Check input
if(!CAp::Assert(npoints>=0,__FUNCTION__+": npoints<0"))
return;
if(!CAp::Assert(xy.Rows()>=npoints,__FUNCTION__+": rows(xy)<npoints"))
return;
if(!CAp::Assert(xy.Cols()>=nvars+ny,__FUNCTION__+": cols(xy)<nvars+nout"))
return;
if(!CAp::Assert(CApServ::IsFiniteMatrix(xy,npoints,nvars+ny),__FUNCTION__+": xy parameter contains INFs or NANs"))
return;
//--- Process using local buffer
KNNCreateBuffer(model,buf);
if(iscls)
CBdSS::DSErrAllocate(nout,errbuf);
else
CBdSS::DSErrAllocate(-nout,errbuf);
desiredy.Resize(ny);
ulong parts[]={nvars,nout};
matrix<double> splitted[];
//--- check
xy.Split(parts,1,splitted);
for(int i=0; i<npoints; i++)
{
buf.m_x=splitted[0].Row(i);
desiredy=splitted[1].Row(i);
if(iscls)
{
j=(int)MathRound(desiredy[0]);
//--- check
if(!CAp::Assert(j>=0 && j<nout,__FUNCTION__+": one of the class labels is not in [0,NClasses)"))
return;
}
ProcessInternal(model,buf);
CBdSS::DSErrAccumulate(errbuf,buf.m_y,desiredy);
}
CBdSS::DSErrFinish(errbuf);
//--- Extract results
if(iscls)
{
rep.m_RelCLSError=errbuf[0];
rep.m_AvgCE=errbuf[1];
}
rep.m_RMSError=errbuf[2];
rep.m_AvgError=errbuf[3];
rep.m_AvgRelError=errbuf[4];
}
//+------------------------------------------------------------------+
//| Serializer: allocation |
//+------------------------------------------------------------------+
void CKNN::KNNAlloc(CSerializer &s,CKNNModel &model)
{
s.Alloc_Entry();
s.Alloc_Entry();
s.Alloc_Entry();
s.Alloc_Entry();
s.Alloc_Entry();
s.Alloc_Entry();
s.Alloc_Entry();
s.Alloc_Entry();
if(!model.m_isdummy)
CNearestNeighbor::KDTreeAlloc(s,model.m_tree);
}
//+------------------------------------------------------------------+
//| Serializer: serialization |
//+------------------------------------------------------------------+
void CKNN::KNNSerialize(CSerializer &s,CKNNModel &model)
{
s.Serialize_Int(CSCodes::GetKNNSerializationCode());
s.Serialize_Int(m_knnfirstversion);
s.Serialize_Int(model.m_nvars);
s.Serialize_Int(model.m_nout);
s.Serialize_Int(model.m_k);
s.Serialize_Double(model.m_eps);
s.Serialize_Bool(model.m_iscls);
s.Serialize_Bool(model.m_isdummy);
if(!model.m_isdummy)
CNearestNeighbor::KDTreeSerialize(s,model.m_tree);
}
//+------------------------------------------------------------------+
//| Serializer: unserialization |
//+------------------------------------------------------------------+
void CKNN::KNNUnserialize(CSerializer &s,CKNNModel &model)
{
//--- check correctness of header
int i0=s.Unserialize_Int();
if(!CAp::Assert(i0==CSCodes::GetKNNSerializationCode(),__FUNCTION__+": stream header corrupted"))
return;
int i1=s.Unserialize_Int();
if(!CAp::Assert(i1==m_knnfirstversion,__FUNCTION__+": stream header corrupted"))
return;
//--- Unserialize data
model.m_nvars=s.Unserialize_Int();
model.m_nout=s.Unserialize_Int();
model.m_k=s.Unserialize_Int();
model.m_eps=s.Unserialize_Double();
model.m_iscls=s.Unserialize_Bool();
model.m_isdummy=s.Unserialize_Bool();
if(!model.m_isdummy)
CNearestNeighbor::KDTreeUnserialize(s,model.m_tree);
//--- Prepare local buffer
KNNCreateBuffer(model,model.m_buffer);
}
//+------------------------------------------------------------------+
//| Sets report fields to their default values |
//+------------------------------------------------------------------+
void CKNN::ClearReport(CKNNReport &rep)
{
rep.m_RelCLSError=0;
rep.m_AvgCE=0;
rep.m_RMSError=0;
rep.m_AvgError=0;
rep.m_AvgRelError=0;
}
//+------------------------------------------------------------------+
//| This function processes buf.X and stores result to buf.Y |
//| INPUT PARAMETERS: |
//| Model - KNN model |
//| Buf - processing buffer. |
//| IMPORTANT: buffer object should be used only with model which was|
//| used to initialize buffer. Any attempt to use buffer |
//| with different object is dangerous - you may get |
//| integrity check failure (exception) because sizes of |
//| internal arrays do not fit to dimensions of the model |
//| structure. |
//+------------------------------------------------------------------+
void CKNN::ProcessInternal(CKNNModel &model,CKNNBuffer &buf)
{
//--- Quick exit if needed
buf.m_y.Fill(0);
if(model.m_isdummy)
return;
//--- create variables
int nncnt=0;
double v=0;
int nvars=model.m_nvars;
int nout=model.m_nout;
bool iscls=model.m_iscls;
//--- Perform request, average results
nncnt=CNearestNeighbor::KDTreeTsQueryAKNN(model.m_tree,buf.m_treebuf,buf.m_x,model.m_k,true,model.m_eps);
v=1/CApServ::Coalesce(nncnt,1);
if(iscls)
{
CNearestNeighbor::KDTreeTsQueryResultsTags(model.m_tree,buf.m_treebuf,buf.m_tags);
for(int i=0; i<nncnt; i++)
{
int j=buf.m_tags[i];
buf.m_y.Add(j,v);
}
}
else
{
CNearestNeighbor::KDTreeTsQueryResultsXY(model.m_tree,buf.m_treebuf,buf.m_xy);
for(int i=0; i<nncnt; i++)
for(int j=0; j<nout; j++)
buf.m_y.Add(j,v*buf.m_xy.Get(i,nvars+j));
}
}
//+------------------------------------------------------------------+
Reference in New Issue View Git Blame Copy Permalink