//+------------------------------------------------------------------+
//|                                                 dataanalysis.mqh |
//|            Copyright 2003-2012 Sergey Bochkanov (ALGLIB project) |
//|                   Copyright 2012-2017, MetaQuotes Software Corp. |
//|                                             https://www.mql5.com |
//+------------------------------------------------------------------+
//| Implementation of ALGLIB library in MetaQuotes Language 5        |
//|                                                                  |
//| The features of the library include:                             |
//| - Linear algebra (direct algorithms, EVD, SVD)                   |
//| - Solving systems of linear and non-linear equations             |
//| - Interpolation                                                  |
//| - Optimization                                                   |
//| - FFT (Fast Fourier Transform)                                   |
//| - Numerical integration                                          |
//| - Linear and nonlinear least-squares fitting                     |
//| - Ordinary differential equations                                |
//| - Computation of special functions                               |
//| - Descriptive statistics and hypothesis testing                  |
//| - Data analysis - classification, regression                     |
//| - Implementing linear algebra algorithms, interpolation, etc.    |
//|   in high-precision arithmetic (using MPFR)                      |
//|                                                                  |
//| This file is free software; you can redistribute it and/or       |
//| modify it under the terms of the GNU General Public License as   |
//| published by the Free Software Foundation (www.fsf.org); either  |
//| version 2 of the License, or (at your option) any later version. |
//|                                                                  |
//| This program is distributed in the hope that it will be useful,  |
//| but WITHOUT ANY WARRANTY; without even the implied warranty of   |
//| MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the     |
//| GNU General Public License for more details.                     |
//+------------------------------------------------------------------+
#include "ap.mqh"
#include "optimization.mqh"
#include "statistics.mqh"
#include "solvers.mqh"
//+------------------------------------------------------------------+
//| Auxiliary class for CBdSS                                        |
//+------------------------------------------------------------------+
class CCVReport
  {
public:
   double            m_relclserror;
   double            m_avgce;
   double            m_rmserror;
   double            m_avgerror;
   double            m_avgrelerror;

                     CCVReport(void);
                    ~CCVReport(void);
  };
//+------------------------------------------------------------------+
//| Constructor without parameters                                   |
//+------------------------------------------------------------------+
CCVReport::CCVReport(void)
  {

  }
//+------------------------------------------------------------------+
//| Destructor                                                       |
//+------------------------------------------------------------------+
CCVReport::~CCVReport(void)
  {

  }
//+------------------------------------------------------------------+
//| Data analysis                                                    |
//+------------------------------------------------------------------+
class CBdSS
  {
private:
   //--- private methods
   static double     XLnY(const double x,const double y);
   static double     GetCV(int &cnt[],const int nc);
   static void       TieAddC(int &c[],int &ties[],const int ntie,const int nc,int &cnt[]);
   static void       TieSubC(int &c[],int &ties[],const int ntie,const int nc,int &cnt[]);
public:
   //--- constructor, destructor
                     CBdSS(void);
                    ~CBdSS(void);
   //--- public methods
   static void       DSErrAllocate(const int nclasses,double &buf[]);
   static void       DSErrAccumulate(double &buf[],double &y[],double &desiredy[]);
   static void       DSErrFinish(double &buf[]);
   static void       DSNormalize(CMatrixDouble &xy,const int npoints,const int nvars,int &info,double &means[],double &sigmas[]);
   static void       DSNormalizeC(CMatrixDouble &xy,const int npoints,const int nvars,int &info,double &means[],double &sigmas[]);
   static double     DSGetMeanMindIstance(CMatrixDouble &xy,const int npoints,const int nvars);
   static void       DSTie(double &a[],const int n,int &ties[],int &tiecount,int &p1[],int &p2[]);
   static void       DSTieFastI(double &a[],int &b[],const int n,int &ties[],int &tiecount,double &bufr[],int &bufi[]);
   static void       DSOptimalSplit2(double &ca[],int &cc[],const int n,int &info,double &threshold,double &pal,double &pbl,double &par,double &pbr,double &cve);
   static void       DSOptimalSplit2Fast(double &a[],int &c[],int &tiesbuf[],int &cntbuf[],double &bufr[],int &bufi[],const int n,const int nc,double alpha,int &info,double &threshold,double &rms,double &cvrms);
   static void       DSSplitK(double &ca[],int &cc[],const int n,const int nc,int kmax,int &info,double &thresholds[],int &ni,double &cve);
   static void       DSOptimalSplitK(double &ca[],int &cc[],const int n,const int nc,int kmax,int &info,double &thresholds[],int &ni,double &cve);
  };
//+------------------------------------------------------------------+
//| Constructor without parameters                                   |
//+------------------------------------------------------------------+
CBdSS::CBdSS(void)
  {

  }
//+------------------------------------------------------------------+
//| Destructor                                                       |
//+------------------------------------------------------------------+
CBdSS::~CBdSS(void)
  {

  }
//+------------------------------------------------------------------+
//| This set of routines (DSErrAllocate, DSErrAccumulate,            |
//| DSErrFinish) calculates different error functions (classification| 
//| error, cross-entropy, rms, avg, avg.rel errors).                 |
//| 1. DSErrAllocate prepares buffer.                                |
//| 2. DSErrAccumulate accumulates individual errors:                |
//|     * Y contains predicted output (posterior probabilities for   |
//|       classification)                                            |
//|     * DesiredY contains desired output (class number for         |
//|       classification)                                            |
//| 3. DSErrFinish outputs results:                                  |
//|    * Buf[0] contains relative classification error (zero for     |
//|      regression tasks)                                           |
//|    * Buf[1] contains avg. cross-entropy (zero for regression     |
//|      tasks)                                                      |
//|    * Buf[2] contains rms error (regression, classification)      |
//|    * Buf[3] contains average error (regression, classification)  |
//|    * Buf[4] contains average relative error (regression,         |
//|      classification)                                             |
//| NOTES(1):                                                        |
//|     "NClasses>0" means that we have classification task.         |
//|     "NClasses<0" means regression task with -NClasses real       |
//|     outputs.                                                     |
//| NOTES(2):                                                        |
//|     rms. avg, avg.rel errors for classification tasks are        |
//|     interpreted as errors in posterior probabilities with        |
//|     respect to probabilities given by training/test set.         |
//+------------------------------------------------------------------+
static void CBdSS::DSErrAllocate(const int nclasses,double &buf[])
  {
//--- allocation
   ArrayResizeAL(buf,8);
//--- initialization
   buf[0]=0;
   buf[1]=0;
   buf[2]=0;
   buf[3]=0;
   buf[4]=0;
   buf[5]=nclasses;
   buf[6]=0;
   buf[7]=0;
  }
//+------------------------------------------------------------------+
//| See DSErrAllocate for comments on this routine.                  |
//+------------------------------------------------------------------+
static void CBdSS::DSErrAccumulate(double &buf[],double &y[],double &desiredy[])
  {
//--- create variables
   int    nclasses=0;
   int    nout=0;
   int    offs=0;
   int    mmax=0;
   int    rmax=0;
   int    j=0;
   double v=0;
   double ev=0;
//--- initialization
   offs=5;
   nclasses=(int)MathRound(buf[offs]);
//--- check
   if(nclasses>0)
     {
      //--- Classification
      rmax=(int)MathRound(desiredy[0]);
      mmax=0;
      //--- initialization
      for(j=1;j<=nclasses-1;j++)
        {
         //--- check
         if(y[j]>y[mmax])
            mmax=j;
        }
      //--- check
      if(mmax!=rmax)
         buf[0]=buf[0]+1;
      //--- check
      if(y[rmax]>0.0)
         buf[1]=buf[1]-MathLog(y[rmax]);
      else
         buf[1]=buf[1]+MathLog(CMath::m_maxrealnumber);
      //--- calculation
      for(j=0;j<=nclasses-1;j++)
        {
         v=y[j];
         //--- check
         if(j==rmax)
            ev=1;
         else
            ev=0;
         //--- change values
         buf[2]=buf[2]+CMath::Sqr(v-ev);
         buf[3]=buf[3]+MathAbs(v-ev);
         //--- check
         if(ev!=0.0)
           {
            buf[4]=buf[4]+MathAbs((v-ev)/ev);
            buf[offs+2]=buf[offs+2]+1;
           }
        }
      //--- change value
      buf[offs+1]=buf[offs+1]+1;
     }
   else
     {
      //--- Regression
      nout=-nclasses;
      rmax=0;
      //--- initialization
      for(j=1;j<=nout-1;j++)
        {
         //--- check
         if(desiredy[j]>desiredy[rmax])
            rmax=j;
        }
      //--- initialization
      mmax=0;
      for(j=1;j<=nout-1;j++)
        {
         //--- check
         if(y[j]>y[mmax])
            mmax=j;
        }
      //--- check
      if(mmax!=rmax)
         buf[0]=buf[0]+1;
      //--- calculation
      for(j=0;j<=nout-1;j++)
        {
         //--- change values
         v=y[j];
         ev=desiredy[j];
         buf[2]=buf[2]+CMath::Sqr(v-ev);
         buf[3]=buf[3]+MathAbs(v-ev);
         //--- check
         if(ev!=0.0)
           {
            buf[4]=buf[4]+MathAbs((v-ev)/ev);
            buf[offs+2]=buf[offs+2]+1;
           }
        }
      //--- change value
      buf[offs+1]=buf[offs+1]+1;
     }
  }
//+------------------------------------------------------------------+
//| See DSErrAllocate for comments on this routine.                  |
//+------------------------------------------------------------------+
static void CBdSS::DSErrFinish(double &buf[])
  {
//--- create variables
   int nout=0;
   int offs=0;
//--- initialization
   offs=5;
   nout=(int)(MathAbs((int)MathRound(buf[offs])));
//--- check
   if(buf[offs+1]!=0.0)
     {
      //--- change values
      buf[0]=buf[0]/buf[offs+1];
      buf[1]=buf[1]/buf[offs+1];
      buf[2]=MathSqrt(buf[2]/(nout*buf[offs+1]));
      buf[3]=buf[3]/(nout*buf[offs+1]);
     }
//--- check
   if(buf[offs+2]!=0.0)
      buf[4]=buf[4]/buf[offs+2];
  }
//+------------------------------------------------------------------+
//| Normalize                                                        |
//+------------------------------------------------------------------+
static void CBdSS::DSNormalize(CMatrixDouble &xy,const int npoints,
                               const int nvars,int &info,double &means[],
                               double &sigmas[])
  {
//--- create variables
   int    i=0;
   int    j=0;
   double mean=0;
   double variance=0;
   double skewness=0;
   double kurtosis=0;
   int    i_=0;
//--- create array
   double tmp[];
//--- initialization
   info=0;
//--- Test parameters
   if(npoints<=0 || nvars<1)
     {
      info=-1;
      return;
     }
//--- change value
   info=1;
//--- Standartization
   ArrayResizeAL(means,nvars);
   ArrayResizeAL(sigmas,nvars);
   ArrayResizeAL(tmp,npoints);
//--- calculation
   for(j=0;j<=nvars-1;j++)
     {
      //--- copy
      for(i_=0;i_<=npoints-1;i_++)
         tmp[i_]=xy[i_][j];
      //--- function call
      CBaseStat::SampleMoments(tmp,npoints,mean,variance,skewness,kurtosis);
      //--- change values
      means[j]=mean;
      sigmas[j]=MathSqrt(variance);
      //--- check
      if(sigmas[j]==0.0)
         sigmas[j]=1;
      //--- change values
      for(i=0;i<=npoints-1;i++)
         xy[i].Set(j,(xy[i][j]-means[j])/sigmas[j]);
     }
  }
//+------------------------------------------------------------------+
//| Normalize                                                        |
//+------------------------------------------------------------------+
static void CBdSS::DSNormalizeC(CMatrixDouble &xy,const int npoints,
                                const int nvars,int &info,double &means[],
                                double &sigmas[])
  {
//--- create variables
   int    j=0;
   double mean=0;
   double variance=0;
   double skewness=0;
   double kurtosis=0;
   int    i_=0;
//--- create array
   double tmp[];
//--- initialization
   info=0;
//--- Test parameters
   if(npoints<=0 || nvars<1)
     {
      info=-1;
      return;
     }
//--- change value
   info=1;
//--- Standartization
   ArrayResizeAL(means,nvars);
   ArrayResizeAL(sigmas,nvars);
   ArrayResizeAL(tmp,npoints);
   for(j=0;j<=nvars-1;j++)
     {
      //--- copy
      for(i_=0;i_<=npoints-1;i_++)
         tmp[i_]=xy[i_][j];
      //--- function call
      CBaseStat::SampleMoments(tmp,npoints,mean,variance,skewness,kurtosis);
      //--- change values
      means[j]=mean;
      sigmas[j]=MathSqrt(variance);
      //--- check
      if(sigmas[j]==0.0)
         sigmas[j]=1;
     }
  }
//+------------------------------------------------------------------+
//| Method                                                           |
//+------------------------------------------------------------------+
static double CBdSS::DSGetMeanMindIstance(CMatrixDouble &xy,const int npoints,
                                          const int nvars)
  {
//--- create variables
   double result=0;
   int    i=0;
   int    j=0;
   double v=0;
   int    i_=0;
//--- creating arrays
   double tmp[];
   double tmp2[];
//--- Test parameters
   if(npoints<=0 || nvars<1)
      return(0);
//--- Process
   ArrayResizeAL(tmp,npoints);
   for(i=0;i<=npoints-1;i++)
      tmp[i]=CMath::m_maxrealnumber;
//--- allocation
   ArrayResizeAL(tmp2,nvars);
   for(i=0;i<=npoints-1;i++)
     {
      for(j=i+1;j<=npoints-1;j++)
        {
         //--- calculation
         for(i_=0;i_<=nvars-1;i_++)
            tmp2[i_]=xy[i][i_];
         for(i_=0;i_<=nvars-1;i_++)
            tmp2[i_]=tmp2[i_]-xy[j][i_];
         v=0.0;
         for(i_=0;i_<=nvars-1;i_++)
            v+=tmp2[i_]*tmp2[i_];
         //--- change values
         v=MathSqrt(v);
         tmp[i]=MathMin(tmp[i],v);
         tmp[j]=MathMin(tmp[j],v);
        }
     }
//--- get result
   result=0;
   for(i=0;i<=npoints-1;i++)
      result=result+tmp[i]/npoints;
//--- return result
   return(result);
  }
//+------------------------------------------------------------------+
//| Method                                                           |
//+------------------------------------------------------------------+
static void CBdSS::DSTie(double &a[],const int n,int &ties[],int &tiecount,
                         int &p1[],int &p2[])
  {
//--- create variables
   int i=0;
   int k=0;
//--- create array
   int tmp[];
//--- initialization
   tiecount=0;
//--- Special case
   if(n<=0)
     {
      tiecount=0;
      return;
     }
//--- Sort A
   CTSort::TagSort(a,n,p1,p2);
//--- Process ties
   tiecount=1;
   for(i=1;i<=n-1;i++)
     {
      //--- check
      if(a[i]!=a[i-1])
         tiecount=tiecount+1;
     }
//--- allocation
   ArrayResizeAL(ties,tiecount+1);
//--- change values
   ties[0]=0;
   k=1;
//--- calculation
   for(i=1;i<=n-1;i++)
     {
      //--- check
      if(a[i]!=a[i-1])
        {
         ties[k]=i;
         k=k+1;
        }
     }
//--- change value
   ties[tiecount]=n;
  }
//+------------------------------------------------------------------+
//| Method                                                           |
//+------------------------------------------------------------------+
static void CBdSS::DSTieFastI(double &a[],int &b[],const int n,int &ties[],
                              int &tiecount,double &bufr[],int &bufi[])
  {
//--- create variables
   int i=0;
   int k=0;
//--- create array
   int tmp[];
//--- initialization
   tiecount=0;
//--- Special case
   if(n<=0)
     {
      tiecount=0;
      return;
     }
//--- Sort A
   CTSort::TagSortFastI(a,b,bufr,bufi,n);
//--- Process ties
   ties[0]=0;
   k=1;
//--- calculation
   for(i=1;i<=n-1;i++)
     {
      //--- check
      if(a[i]!=a[i-1])
        {
         ties[k]=i;
         k=k+1;
        }
     }
//--- change values
   ties[k]=n;
   tiecount=k;
  }
//+------------------------------------------------------------------+
//| Optimal binary classification                                    |
//| Algorithms finds optimal (=with minimal cross-entropy) binary    |
//| partition.                                                       |
//| Internal subroutine.                                             |
//| INPUT PARAMETERS:                                                |
//|     A       -   array[0..N-1], variable                          |
//|     C       -   array[0..N-1], class numbers (0 or 1).           |
//|     N       -   array size                                       |
//| OUTPUT PARAMETERS:                                               |
//|     Info    -   completetion code:                               |
//|                 * -3, all values of A[] are same (partition is   |
//|                   impossible)                                    |
//|                 * -2, one of C[] is incorrect (<0, >1)           |
//|                 * -1, incorrect pararemets were passed (N<=0).   |
//|                 *  1, OK                                         |
//|     Threshold-  partiton boundary. Left part contains values     |
//|                 which are strictly less than Threshold. Right    |
//|                 part contains values which are greater than or   |
//|                 equal to Threshold.                              |
//|     PAL, PBL-   probabilities P(0|v<Threshold) and               |
//|                 P(1|v<Threshold)                                 |
//|     PAR, PBR-   probabilities P(0|v>=Threshold) and              |
//|                 P(1|v>=Threshold)                                |
//|     CVE     -   cross-validation estimate of cross-entropy       |
//+------------------------------------------------------------------+
static void CBdSS::DSOptimalSplit2(double &ca[],int &cc[],const int n,
                                   int &info,double &threshold,double &pal,
                                   double &pbl,double &par,double &pbr,
                                   double &cve)
  {
//--- create variables
   int    i=0;
   int    t=0;
   double s=0;
   int    tiecount=0;
   int    k=0;
   int    koptimal=0;
   double pak=0;
   double pbk=0;
   double cvoptimal=0;
   double cv=0;
//--- creating arrays
   int    ties[];
   int    p1[];
   int    p2[];
   double a[];
   int    c[];
//--- copy
   ArrayCopy(a,ca);
   ArrayCopy(c,cc);
//--- initialization
   info=0;
   threshold=0;
   pal=0;
   pbl=0;
   par=0;
   pbr=0;
   cve=0;
//--- Test for errors in inputs
   if(n<=0)
     {
      info=-1;
      return;
     }
   for(i=0;i<=n-1;i++)
     {
      //--- check
      if(c[i]!=0 && c[i]!=1)
        {
         info=-2;
         return;
        }
     }
//--- change value
   info=1;
//--- Tie
   DSTie(a,n,ties,tiecount,p1,p2);
//--- swap
   for(i=0;i<=n-1;i++)
     {
      //--- check
      if(p2[i]!=i)
        {
         t=c[i];
         c[i]=c[p2[i]];
         c[p2[i]]=t;
        }
     }
//--- Special case: number of ties is 1.
//--- NOTE: we assume that P[i][j] equals to 0 or 1,
//---       intermediate values are not allowed.
   if(tiecount==1)
     {
      info=-3;
      return;
     }
//--- General case,number of ties > 1
//--- NOTE: we assume that P[i][j] equals to 0 or 1,
//---       intermediate values are not allowed.
   pal=0;
   pbl=0;
   par=0;
   pbr=0;
   for(i=0;i<=n-1;i++)
     {
      //--- check
      if(c[i]==0)
         par=par+1;
      //--- check
      if(c[i]==1)
         pbr=pbr+1;
     }
//--- change values
   koptimal=-1;
   cvoptimal=CMath::m_maxrealnumber;
   for(k=0;k<=tiecount-2;k++)
     {
      //--- first,obtain information about K-th tie which is
      //--- moved from R-part to L-part
      pak=0;
      pbk=0;
      for(i=ties[k];i<=ties[k+1]-1;i++)
        {
         //--- check
         if(c[i]==0)
            pak=pak+1;
         //--- check
         if(c[i]==1)
            pbk=pbk+1;
        }
      //--- Calculate cross-validation CE
      cv=0;
      cv=cv-XLnY(pal+pak,(pal+pak)/(pal+pak+pbl+pbk+1));
      cv=cv-XLnY(pbl+pbk,(pbl+pbk)/(pal+pak+1+pbl+pbk));
      cv=cv-XLnY(par-pak,(par-pak)/(par-pak+pbr-pbk+1));
      cv=cv-XLnY(pbr-pbk,(pbr-pbk)/(par-pak+1+pbr-pbk));
      //--- Compare with best
      if(cv<cvoptimal)
        {
         cvoptimal=cv;
         koptimal=k;
        }
      //--- update
      pal=pal+pak;
      pbl=pbl+pbk;
      par=par-pak;
      pbr=pbr-pbk;
     }
//--- change values
   cve=cvoptimal;
   threshold=0.5*(a[ties[koptimal]]+a[ties[koptimal+1]]);
   pal=0;
   pbl=0;
   par=0;
   pbr=0;
   for(i=0;i<=n-1;i++)
     {
      //--- check
      if(a[i]<threshold)
        {
         //--- check
         if(c[i]==0)
            pal=pal+1;
         else
            pbl=pbl+1;
        }
      else
        {
         //--- check
         if(c[i]==0)
            par=par+1;
         else
            pbr=pbr+1;
        }
     }
//--- change values
   s=pal+pbl;
   pal=pal/s;
   pbl=pbl/s;
   s=par+pbr;
   par=par/s;
   pbr=pbr/s;
  }
//+------------------------------------------------------------------+
//| Optimal partition, internal subroutine. Fast version.            |
//| Accepts:                                                         |
//|     A       array[0..N-1]       array of attributes array[0..N-1]|
//|     C       array[0..N-1]       array of class labels            |
//|     TiesBuf array[0..N]         temporaries (ties)               |
//|     CntBuf  array[0..2*NC-1]    temporaries (counts)             |
//|     Alpha                       centering factor (0<=alpha<=1,   |
//|                                 recommended value - 0.05)        |
//|     BufR    array[0..N-1]       temporaries                      |
//|     BufI    array[0..N-1]       temporaries                      |
//| Output:                                                          |
//|     Info    error code (">0"=OK, "<0"=bad)                       |
//|     RMS     training set RMS error                               |
//|     CVRMS   leave-one-out RMS error                              |
//| Note:                                                            |
//|     content of all arrays is changed by subroutine;              |
//|     it doesn't allocate temporaries.                             |
//+------------------------------------------------------------------+
static void CBdSS::DSOptimalSplit2Fast(double &a[],int &c[],int &tiesbuf[],
                                       int &cntbuf[],double &bufr[],int &bufi[],
                                       const int n,const int nc,double alpha,
                                       int &info,double &threshold,
                                       double &rms,double &cvrms)
  {
//--- create variables
   int    i=0;
   int    k=0;
   int    cl=0;
   int    tiecount=0;
   double cbest=0;
   double cc=0;
   int    koptimal=0;
   int    sl=0;
   int    sr=0;
   double v=0;
   double w=0;
   double x=0;
//--- initialization
   info=0;
   threshold=0;
   rms=0;
   cvrms=0;
//--- Test for errors in inputs
   if(n<=0 || nc<2)
     {
      info=-1;
      return;
     }
   for(i=0;i<=n-1;i++)
     {
      //--- check
      if(c[i]<0 || c[i]>=nc)
        {
         info=-2;
         return;
        }
     }
//--- change value
   info=1;
//--- Tie
   DSTieFastI(a,c,n,tiesbuf,tiecount,bufr,bufi);
//--- Special case: number of ties is 1.
   if(tiecount==1)
     {
      info=-3;
      return;
     }
//--- General case,number of ties > 1
   for(i=0;i<=2*nc-1;i++)
      cntbuf[i]=0;
   for(i=0;i<=n-1;i++)
      cntbuf[nc+c[i]]=cntbuf[nc+c[i]]+1;
//--- change values
   koptimal=-1;
   threshold=a[n-1];
   cbest=CMath::m_maxrealnumber;
   sl=0;
   sr=n;
//--- calculation
   for(k=0;k<=tiecount-2;k++)
     {
      //--- first,move Kth tie from right to left
      for(i=tiesbuf[k];i<=tiesbuf[k+1]-1;i++)
        {
         cl=c[i];
         cntbuf[cl]=cntbuf[cl]+1;
         cntbuf[nc+cl]=cntbuf[nc+cl]-1;
        }
      sl=sl+(tiesbuf[k+1]-tiesbuf[k]);
      sr=sr-(tiesbuf[k+1]-tiesbuf[k]);
      //--- Calculate RMS error
      v=0;
      for(i=0;i<=nc-1;i++)
        {
         w=cntbuf[i];
         v=v+w*CMath::Sqr(w/sl-1);
         v=v+(sl-w)*CMath::Sqr(w/sl);
         w=cntbuf[nc+i];
         v=v+w*CMath::Sqr(w/sr-1);
         v=v+(sr-w)*CMath::Sqr(w/sr);
        }
      //--- change value
      v=MathSqrt(v/(nc*n));
      //--- Compare with best
      x=(double)(2*sl)/(double)(sl+sr)-1;
      cc=v*(1-alpha+alpha*CMath::Sqr(x));
      //--- check
      if(cc<cbest)
        {
         //--- store split
         rms=v;
         koptimal=k;
         cbest=cc;
         //--- calculate CVRMS error
         cvrms=0;
         for(i=0;i<=nc-1;i++)
           {
            //--- check
            if(sl>1)
              {
               w=cntbuf[i];
               cvrms=cvrms+w*CMath::Sqr((w-1)/(sl-1)-1);
               cvrms=cvrms+(sl-w)*CMath::Sqr(w/(sl-1));
              }
            else
              {
               w=cntbuf[i];
               cvrms=cvrms+w*CMath::Sqr(1.0/(double)nc-1);
               cvrms=cvrms+(sl-w)*CMath::Sqr(1.0/(double)nc);
              }
            //--- check
            if(sr>1)
              {
               w=cntbuf[nc+i];
               cvrms=cvrms+w*CMath::Sqr((w-1)/(sr-1)-1);
               cvrms=cvrms+(sr-w)*CMath::Sqr(w/(sr-1));
              }
            else
              {
               w=cntbuf[nc+i];
               cvrms=cvrms+w*CMath::Sqr(1.0/(double)nc-1);
               cvrms=cvrms+(sr-w)*CMath::Sqr(1.0/(double)nc);
              }
           }
         //--- change value
         cvrms=MathSqrt(cvrms/(nc*n));
        }
     }
//--- Calculate threshold.
//--- Code is a bit complicated because there can be such
//--- numbers that 0.5(A+B) equals to A or B (if A-B=epsilon)
   threshold=0.5*(a[tiesbuf[koptimal]]+a[tiesbuf[koptimal+1]]);
//--- check
   if(threshold<=a[tiesbuf[koptimal]])
      threshold=a[tiesbuf[koptimal+1]];
  }
//+------------------------------------------------------------------+
//| Automatic non-optimal discretization, internal subroutine.       |
//+------------------------------------------------------------------+
static void CBdSS::DSSplitK(double &ca[],int &cc[],const int n,const int nc,
                            int kmax,int &info,double &thresholds[],int &ni,
                            double &cve)
  {
//--- create variables
   int    i=0;
   int    j=0;
   int    j1=0;
   int    k=0;
   int    tiecount=0;
   double v2=0;
   int    bestk=0;
   double bestcve=0;
   double curcve=0;
//--- creating arrays
   int    ties[];
   int    p1[];
   int    p2[];
   int    cnt[];
   int    bestsizes[];
   int    cursizes[];
   double a[];
   int    c[];
//--- copy
   ArrayCopy(a,ca);
   ArrayCopy(c,cc);
//--- initialization
   info=0;
   ni=0;
   cve=0;
//--- Test for errors in inputs
   if((n<=0 || nc<2) || kmax<2)
     {
      info=-1;
      return;
     }
   for(i=0;i<=n-1;i++)
     {
      //--- check
      if(c[i]<0 || c[i]>=nc)
        {
         info=-2;
         return;
        }
     }
//--- change value
   info=1;
//--- Tie
   DSTie(a,n,ties,tiecount,p1,p2);
//--- swap
   for(i=0;i<=n-1;i++)
     {
      //--- check
      if(p2[i]!=i)
        {
         k=c[i];
         c[i]=c[p2[i]];
         c[p2[i]]=k;
        }
     }
//--- Special cases
   if(tiecount==1)
     {
      info=-3;
      return;
     }
//--- General case:
//--- 0. allocate arrays
   kmax=MathMin(kmax,tiecount);
//--- allocation
   ArrayResizeAL(bestsizes,kmax);
   ArrayResizeAL(cursizes,kmax);
   ArrayResizeAL(cnt,nc);
//--- General case:
//--- 1. prepare "weak" solution (two subintervals,divided at median)
   v2=CMath::m_maxrealnumber;
   j=-1;
   for(i=1;i<=tiecount-1;i++)
     {
      //--- check
      if(MathAbs(ties[i]-0.5*(n-1))<v2)
        {
         v2=MathAbs(ties[i]-0.5*n);
         j=i;
        }
     }
//--- check
   if(!CAp::Assert(j>0,__FUNCTION__+": internal error #1!"))
      return;
//--- change values
   bestk=2;
   bestsizes[0]=ties[j];
   bestsizes[1]=n-j;
   bestcve=0;
//--- calculation
   for(i=0;i<=nc-1;i++)
      cnt[i]=0;
   for(i=0;i<=j-1;i++)
      TieAddC(c,ties,i,nc,cnt);
   bestcve=bestcve+GetCV(cnt,nc);
//--- calculation
   for(i=0;i<=nc-1;i++)
      cnt[i]=0;
   for(i=j;i<=tiecount-1;i++)
      TieAddC(c,ties,i,nc,cnt);
   bestcve=bestcve+GetCV(cnt,nc);
//--- General case:
//--- 2. Use greedy algorithm to find sub-optimal split in O(KMax*N) time
   for(k=2;k<=kmax;k++)
     {
      //--- Prepare greedy K-interval split
      for(i=0;i<=k-1;i++)
         cursizes[i]=0;
      //--- change values
      i=0;
      j=0;
      //--- cycle
      while(j<=tiecount-1 && i<=k-1)
        {
         //--- Rule: I-th bin is empty,fill it
         if(cursizes[i]==0)
           {
            cursizes[i]=ties[j+1]-ties[j];
            j=j+1;
            continue;
           }
         //--- Rule: (K-1-I) bins left,(K-1-I) ties left (1 tie per bin);next bin
         if(tiecount-j==k-1-i)
           {
            i=i+1;
            continue;
           }
         //--- Rule: last bin,always place in current
         if(i==k-1)
           {
            cursizes[i]=cursizes[i]+ties[j+1]-ties[j];
            j=j+1;
            continue;
           }
         //--- Place J-th tie in I-th bin,or leave for I+1-th bin.
         if(MathAbs(cursizes[i]+ties[j+1]-ties[j]-(double)n/(double)k)<MathAbs(cursizes[i]-(double)n/(double)k))
           {
            cursizes[i]=cursizes[i]+ties[j+1]-ties[j];
            j=j+1;
           }
         else
            i=i+1;
        }
      //--- check
      if(!CAp::Assert(cursizes[k-1]!=0 && j==tiecount,__FUNCTION__+": internal error #1"))
         return;
      //--- Calculate CVE
      curcve=0;
      j=0;
      for(i=0;i<=k-1;i++)
        {
         //--- calculation
         for(j1=0;j1<=nc-1;j1++)
            cnt[j1]=0;
         for(j1=j;j1<=j+cursizes[i]-1;j1++)
            cnt[c[j1]]=cnt[c[j1]]+1;
         curcve=curcve+GetCV(cnt,nc);
         j=j+cursizes[i];
        }
      //--- Choose best variant
      if(curcve<bestcve)
        {
         for(i=0;i<=k-1;i++)
            bestsizes[i]=cursizes[i];
         bestcve=curcve;
         bestk=k;
        }
     }
//--- Transform from sizes to thresholds
   cve=bestcve;
   ni=bestk;
//--- allocation
   ArrayResizeAL(thresholds,ni-1);
   j=bestsizes[0];
//--- calculation
   for(i=1;i<=bestk-1;i++)
     {
      thresholds[i-1]=0.5*(a[j-1]+a[j]);
      j=j+bestsizes[i];
     }
  }
//+------------------------------------------------------------------+
//| Automatic optimal discretization, internal subroutine.           |
//+------------------------------------------------------------------+
static void CBdSS::DSOptimalSplitK(double &ca[],int &cc[],const int n,
                                   const int nc,int kmax,int &info,
                                   double &thresholds[],int &ni,double &cve)
  {
//--- create variables
   int    i=0;
   int    j=0;
   int    s=0;
   int    jl=0;
   int    jr=0;
   double v2=0;
   int    tiecount=0;
   double cvtemp=0;
   int    k=0;
   int    koptimal=0;
   double cvoptimal=0;
//--- creating arrays
   int    ties[];
   int    p1[];
   int    p2[];
   int    cnt[];
   int    cnt2[];
   double a[];
   int    c[];
//--- create matrix
   CMatrixDouble cv;
   CMatrixInt    splits;
//--- copy
   ArrayCopy(a,ca);
   ArrayCopy(c,cc);
//--- initialization
   info=0;
   ni=0;
   cve=0;
//--- Test for errors in inputs
   if((n<=0 || nc<2) || kmax<2)
     {
      info=-1;
      return;
     }
   for(i=0;i<=n-1;i++)
     {
      //--- check
      if(c[i]<0 || c[i]>=nc)
        {
         info=-2;
         return;
        }
     }
//--- change value
   info=1;
//--- Tie
   DSTie(a,n,ties,tiecount,p1,p2);
//--- swap
   for(i=0;i<=n-1;i++)
     {
      //--- check
      if(p2[i]!=i)
        {
         k=c[i];
         c[i]=c[p2[i]];
         c[p2[i]]=k;
        }
     }
//--- Special cases
   if(tiecount==1)
     {
      info=-3;
      return;
     }
//--- General case
//--- Use dynamic programming to find best split in O(KMax*NC*TieCount^2) time
   kmax=MathMin(kmax,tiecount);
//--- allocation
   cv.Resize(kmax,tiecount);
   splits.Resize(kmax,tiecount);
   ArrayResizeAL(cnt,nc);
   ArrayResizeAL(cnt2,nc);
//--- calculation
   for(j=0;j<=nc-1;j++)
      cnt[j]=0;
   for(j=0;j<=tiecount-1;j++)
     {
      TieAddC(c,ties,j,nc,cnt);
      splits[0].Set(j,0);
      cv[0].Set(j,GetCV(cnt,nc));
     }
   for(k=1;k<=kmax-1;k++)
     {
      for(j=0;j<=nc-1;j++)
         cnt[j]=0;
      //--- Subtask size J in [K..TieCount-1]:
      //--- optimal K-splitting on ties from 0-th to J-th.
      for(j=k;j<=tiecount-1;j++)
        {
         //--- Update Cnt - let it contain classes of ties from K-th to J-th
         TieAddC(c,ties,j,nc,cnt);
         //--- Search for optimal split point S in [K..J]
         for(i=0;i<=nc-1;i++)
            cnt2[i]=cnt[i];
         cv[k].Set(j,cv[k-1][j-1]+GetCV(cnt2,nc));
         splits[k].Set(j,j);
         //--- calculation
         for(s=k+1;s<=j;s++)
           {
            //--- Update Cnt2 - let it contain classes of ties from S-th to J-th
            TieSubC(c,ties,s-1,nc,cnt2);
            //--- Calculate CVE
            cvtemp=cv[k-1][s-1]+GetCV(cnt2,nc);
            //--- check
            if(cvtemp<cv[k][j])
              {
               cv[k].Set(j,cvtemp);
               splits[k].Set(j,s);
              }
           }
        }
     }
//--- Choose best partition,output result
   koptimal=-1;
   cvoptimal=CMath::m_maxrealnumber;
   for(k=0;k<=kmax-1;k++)
     {
      //--- check
      if(cv[k][tiecount-1]<cvoptimal)
        {
         cvoptimal=cv[k][tiecount-1];
         koptimal=k;
        }
     }
//--- check
   if(!CAp::Assert(koptimal>=0,__FUNCTION__+": internal error #1!"))
      return;
//--- check
   if(koptimal==0)
     {
      //--- Special case: best partition is one big interval.
      //--- Even 2-partition is not better.
      //--- This is possible when dealing with "weak" predictor variables.
      //--- Make binary split as close to the median as possible.
      v2=CMath::m_maxrealnumber;
      j=-1;
      for(i=1;i<=tiecount-1;i++)
        {
         //--- check
         if(MathAbs(ties[i]-0.5*(n-1))<v2)
           {
            v2=MathAbs(ties[i]-0.5*(n-1));
            j=i;
           }
        }
      //--- check
      if(!CAp::Assert(j>0,__FUNCTION__+": internal error #2!"))
         return;
      //--- allocation
      ArrayResizeAL(thresholds,1);
      //--- change values
      thresholds[0]=0.5*(a[ties[j-1]]+a[ties[j]]);
      ni=2;
      cve=0;
      //--- calculation
      for(i=0;i<=nc-1;i++)
         cnt[i]=0;
      for(i=0;i<=j-1;i++)
         TieAddC(c,ties,i,nc,cnt);
      cve=cve+GetCV(cnt,nc);
      for(i=0;i<=nc-1;i++)
         cnt[i]=0;
      for(i=j;i<=tiecount-1;i++)
         TieAddC(c,ties,i,nc,cnt);
      cve=cve+GetCV(cnt,nc);
     }
   else
     {
      //--- General case: 2 or more intervals
      ArrayResizeAL(thresholds,koptimal);
      ni=koptimal+1;
      cve=cv[koptimal][tiecount-1];
      jl=splits[koptimal][tiecount-1];
      jr=tiecount-1;
      //--- calculation
      for(k=koptimal;k>=1;k--)
        {
         thresholds[k-1]=0.5*(a[ties[jl-1]]+a[ties[jl]]);
         jr=jl-1;
         jl=splits[k-1][jl-1];
        }
     }
  }
//+------------------------------------------------------------------+
//| Internal function                                                |
//+------------------------------------------------------------------+
static double CBdSS::XLnY(const double x,const double y)
  {
//--- check
   if(x==0.0)
      return(0);
//--- return result
   return(x*MathLog(y));
  }
//+------------------------------------------------------------------+
//| Internal function,                                               |
//| returns number of samples of class I in Cnt[I]                   |
//+------------------------------------------------------------------+
static double CBdSS::GetCV(int &cnt[],const int nc)
  {
//--- create variables
   double result=0;
   int    i=0;
   double s=0;
//--- calculation
   s=0;
   for(i=0;i<=nc-1;i++)
      s=s+cnt[i];
//--- get result
   result=0;
   for(i=0;i<=nc-1;i++)
      result=result-XLnY(cnt[i],cnt[i]/(s+nc-1));
//--- return result
   return(result);
  }
//+------------------------------------------------------------------+
//| Internal function,adds number of samples of class I in tie NTie  |
//| to Cnt[I]                                                        |
//+------------------------------------------------------------------+
static void CBdSS::TieAddC(int &c[],int &ties[],const int ntie,const int nc,
                           int &cnt[])
  {
//--- create a variable
   int i=0;
//--- calculation
   for(i=ties[ntie];i<=ties[ntie+1]-1;i++)
      cnt[c[i]]=cnt[c[i]]+1;
  }
//+------------------------------------------------------------------+
//| Internal function,subtracts number of samples of class I in tie  |
//| NTie to Cnt[I]                                                   |
//+------------------------------------------------------------------+
static void CBdSS::TieSubC(int &c[],int &ties[],const int ntie,const int nc,
                           int &cnt[])
  {
//--- create a variable
   int i=0;
//--- calculation
   for(i=ties[ntie];i<=ties[ntie+1]-1;i++)
      cnt[c[i]]=cnt[c[i]]-1;
  }
//+------------------------------------------------------------------+
//| Auxiliary class for CDForest                                     |
//+------------------------------------------------------------------+
class CDecisionForest
  {
public:
   int               m_nvars;
   int               m_nclasses;
   int               m_ntrees;
   int               m_bufsize;
   double            m_trees[];
   //--- constructor, destructor
                     CDecisionForest(void);
                    ~CDecisionForest(void);
   //--- copy
   void              Copy(CDecisionForest &obj);
  };
//+------------------------------------------------------------------+
//| Constructor without parameters                                   |
//+------------------------------------------------------------------+
CDecisionForest::CDecisionForest(void)
  {

  }
//+------------------------------------------------------------------+
//| Destructor                                                       |
//+------------------------------------------------------------------+
CDecisionForest::~CDecisionForest(void)
  {

  }
//+------------------------------------------------------------------+
//| Copy                                                             |
//+------------------------------------------------------------------+
void CDecisionForest::Copy(CDecisionForest &obj)
  {
//--- copy variables
   m_nvars=obj.m_nvars;
   m_nclasses=obj.m_nclasses;
   m_ntrees=obj.m_ntrees;
   m_bufsize=obj.m_bufsize;
//--- copy array
   ArrayCopy(m_trees,obj.m_trees);
  }
//+------------------------------------------------------------------+
//| This class is a shell for class CDecisionForest                  |
//+------------------------------------------------------------------+
class CDecisionForestShell
  {
private:
   CDecisionForest   m_innerobj;
public:
   //--- constructors, destructor
                     CDecisionForestShell(void);
                     CDecisionForestShell(CDecisionForest &obj);
                    ~CDecisionForestShell(void);
   //--- method
   CDecisionForest *GetInnerObj(void);
  };
//+------------------------------------------------------------------+
//| Constructor without parameters                                   |
//+------------------------------------------------------------------+
CDecisionForestShell::CDecisionForestShell(void)
  {

  }
//+------------------------------------------------------------------+
//| Copy constructor                                                 |
//+------------------------------------------------------------------+
CDecisionForestShell::CDecisionForestShell(CDecisionForest &obj)
  {
//--- copy
   m_innerobj.Copy(obj);
  }
//+------------------------------------------------------------------+
//| Destructor                                                       |
//+------------------------------------------------------------------+
CDecisionForestShell::~CDecisionForestShell(void)
  {

  }
//+------------------------------------------------------------------+
//| Return object of class                                           |
//+------------------------------------------------------------------+
CDecisionForest *CDecisionForestShell::GetInnerObj(void)
  {
//--- return result
   return(GetPointer(m_innerobj));
  }
//+------------------------------------------------------------------+
//| Auxiliary class for CDForest                                     |
//+------------------------------------------------------------------+
class CDFReport
  {
public:
   //--- variables
   double            m_relclserror;
   double            m_avgce;
   double            m_rmserror;
   double            m_avgerror;
   double            m_avgrelerror;
   double            m_oobrelclserror;
   double            m_oobavgce;
   double            m_oobrmserror;
   double            m_oobavgerror;
   double            m_oobavgrelerror;
   //--- constructor, destructor
                     CDFReport(void);
                    ~CDFReport(void);
   //--- copy
   void              Copy(CDFReport &obj);
  };
//+------------------------------------------------------------------+
//| Constructor without parameters                                   |
//+------------------------------------------------------------------+
CDFReport::CDFReport(void)
  {

  }
//+------------------------------------------------------------------+
//| Destructor                                                       |
//+------------------------------------------------------------------+
CDFReport::~CDFReport(void)
  {

  }
//+------------------------------------------------------------------+
//| Copy                                                             |
//+------------------------------------------------------------------+
void CDFReport::Copy(CDFReport &obj)
  {
//--- copy variables
   m_relclserror=obj.m_relclserror;
   m_avgce=obj.m_avgce;
   m_rmserror=obj.m_rmserror;
   m_avgerror=obj.m_avgerror;
   m_avgrelerror=obj.m_avgrelerror;
   m_oobrelclserror=obj.m_oobrelclserror;
   m_oobavgce=obj.m_oobavgce;
   m_oobrmserror=obj.m_oobrmserror;
   m_oobavgerror=obj.m_oobavgerror;
   m_oobavgrelerror=obj.m_oobavgrelerror;
  }
//+------------------------------------------------------------------+
//| This class is a shell for class CDFReport                        |
//+------------------------------------------------------------------+
class CDFReportShell
  {
private:
   CDFReport         m_innerobj;
public:
   //--- constructors, destructor
                     CDFReportShell(void);
                     CDFReportShell(CDFReport &obj);
                    ~CDFReportShell(void);
   //--- methods
   double            GetRelClsError(void);
   void              SetRelClsError(const double d);
   double            GetAvgCE(void);
   void              SetAvgCE(const double d);
   double            GetRMSError(void);
   void              SetRMSError(const double d);
   double            GetAvgError(void);
   void              SetAvgError(const double d);
   double            GetAvgRelError(void);
   void              SetAvgRelError(const double d);
   double            GetOOBRelClsError(void);
   void              SetOOBRelClsError(const double d);
   double            GetOOBAvgCE(void);
   void              SetOOBAvgCE(const double d);
   double            GetOOBRMSError(void);
   void              SetOOBRMSError(const double d);
   double            GetOOBAvgError(void);
   void              SetOOBAvgError(const double d);
   double            GetOOBAvgRelError(void);
   void              SetOOBAvgRelError(const double d);
   CDFReport        *GetInnerObj(void);
  };
//+------------------------------------------------------------------+
//| Constructor without parameters                                   |
//+------------------------------------------------------------------+
CDFReportShell::CDFReportShell(void)
  {

  }
//+------------------------------------------------------------------+
//| Copy constructor                                                 |
//+------------------------------------------------------------------+
CDFReportShell::CDFReportShell(CDFReport &obj)
  {
//--- copy
   m_innerobj.Copy(obj);
  }
//+------------------------------------------------------------------+
//| Destructor                                                       |
//+------------------------------------------------------------------+
CDFReportShell::~CDFReportShell(void)
  {

  }
//+------------------------------------------------------------------+
//| Returns the value of the variable relclserror                    |
//+------------------------------------------------------------------+
double CDFReportShell::GetRelClsError(void)
  {
//--- return result
   return(m_innerobj.m_relclserror);
  }
//+------------------------------------------------------------------+
//| Changing the value of the variable relclserror                   |
//+------------------------------------------------------------------+
void CDFReportShell::SetRelClsError(const double d)
  {
//--- change value
   m_innerobj.m_relclserror=d;
  }
//+------------------------------------------------------------------+
//| Returns the value of the variable avgce                          |
//+------------------------------------------------------------------+
double CDFReportShell::GetAvgCE(void)
  {
//--- return result
   return(m_innerobj.m_avgce);
  }
//+------------------------------------------------------------------+
//| Changing the value of the variable avgce                         |
//+------------------------------------------------------------------+
void CDFReportShell::SetAvgCE(const double d)
  {
//--- change value
   m_innerobj.m_avgce=d;
  }
//+------------------------------------------------------------------+
//| Returns the value of the variable rmserror                       |
//+------------------------------------------------------------------+
double CDFReportShell::GetRMSError(void)
  {
//--- return result
   return(m_innerobj.m_rmserror);
  }
//+------------------------------------------------------------------+
//| Changing the value of the variable rmserror                      |
//+------------------------------------------------------------------+
void CDFReportShell::SetRMSError(const double d)
  {
//--- change value
   m_innerobj.m_rmserror=d;
  }
//+------------------------------------------------------------------+
//| Returns the value of the variable avgerror                       |
//+------------------------------------------------------------------+
double CDFReportShell::GetAvgError(void)
  {
//--- return result
   return(m_innerobj.m_avgerror);
  }
//+------------------------------------------------------------------+
//| Changing the value of the variable avgerror                      |
//+------------------------------------------------------------------+
void CDFReportShell::SetAvgError(const double d)
  {
//--- change value
   m_innerobj.m_avgerror=d;
  }
//+------------------------------------------------------------------+
//| Returns the value of the variable avgrelerror                    |
//+------------------------------------------------------------------+
double CDFReportShell::GetAvgRelError(void)
  {
//--- return result
   return(m_innerobj.m_avgrelerror);
  }
//+------------------------------------------------------------------+
//| Changing the value of the variable avgrelerror                   |
//+------------------------------------------------------------------+
void CDFReportShell::SetAvgRelError(const double d)
  {
//--- change value
   m_innerobj.m_avgrelerror=d;
  }
//+------------------------------------------------------------------+
//| Returns the value of the variable oobrelclserror                 |
//+------------------------------------------------------------------+
double CDFReportShell::GetOOBRelClsError(void)
  {
//--- return result
   return(m_innerobj.m_oobrelclserror);
  }
//+------------------------------------------------------------------+
//| Changing the value of the variable oobrelclserror                |
//+------------------------------------------------------------------+
void CDFReportShell::SetOOBRelClsError(const double d)
  {
//--- change value
   m_innerobj.m_oobrelclserror=d;
  }
//+------------------------------------------------------------------+
//| Returns the value of the variable oobavgce                       |
//+------------------------------------------------------------------+
double CDFReportShell::GetOOBAvgCE(void)
  {
//--- return result
   return(m_innerobj.m_oobavgce);
  }
//+------------------------------------------------------------------+
//| Changing the value of the variable oobavgce                      |
//+------------------------------------------------------------------+
void CDFReportShell::SetOOBAvgCE(const double d)
  {
//--- change value
   m_innerobj.m_oobavgce=d;
  }
//+------------------------------------------------------------------+
//| Returns the value of the variable oobrmserror                    |
//+------------------------------------------------------------------+
double CDFReportShell::GetOOBRMSError(void)
  {
//--- return result
   return(m_innerobj.m_oobrmserror);
  }
//+------------------------------------------------------------------+
//| Changing the value of the variable oobrmserror                   |
//+------------------------------------------------------------------+
void CDFReportShell::SetOOBRMSError(const double d)
  {
//--- change value
   m_innerobj.m_oobrmserror=d;
  }
//+------------------------------------------------------------------+
//| Returns the value of the variable oobavgerror                    |
//+------------------------------------------------------------------+
double CDFReportShell::GetOOBAvgError(void)
  {
//--- return result
   return(m_innerobj.m_oobavgerror);
  }
//+------------------------------------------------------------------+
//| Changing the value of the variable oobavgerror                   |
//+------------------------------------------------------------------+
void CDFReportShell::SetOOBAvgError(const double d)
  {
//--- change value
   m_innerobj.m_oobavgerror=d;
  }
//+------------------------------------------------------------------+
//| Returns the value of the variable oobavgrelerror                 |
//+------------------------------------------------------------------+
double CDFReportShell::GetOOBAvgRelError(void)
  {
//--- return result
   return(m_innerobj.m_oobavgrelerror);
  }
//+------------------------------------------------------------------+
//| Changing the value of the variable oobavgrelerror                |
//+------------------------------------------------------------------+
void CDFReportShell::SetOOBAvgRelError(const double d)
  {
//--- change value
   m_innerobj.m_oobavgrelerror=d;
  }
//+------------------------------------------------------------------+
//| Return object of class                                           |
//+------------------------------------------------------------------+
CDFReport *CDFReportShell::GetInnerObj(void)
  {
//--- return result
   return(GetPointer(m_innerobj));
  }
//+------------------------------------------------------------------+
//| Auxiliary class for CDForest                                     |
//+------------------------------------------------------------------+
class CDFInternalBuffers
  {
public:
   //--- arrays
   double            m_treebuf[];
   int               m_idxbuf[];
   double            m_tmpbufr[];
   double            m_tmpbufr2[];
   int               m_tmpbufi[];
   int               m_classibuf[];
   double            m_sortrbuf[];
   double            m_sortrbuf2[];
   int               m_sortibuf[];
   int               m_varpool[];
   bool              m_evsbin[];
   double            m_evssplits[];
   //--- constructor, destructor
                     CDFInternalBuffers(void);
                    ~CDFInternalBuffers(void);
  };
//+------------------------------------------------------------------+
//| Constructor without parameters                                   |
//+------------------------------------------------------------------+
CDFInternalBuffers::CDFInternalBuffers(void)
  {

  }
//+------------------------------------------------------------------+
//| Destructor                                                       |
//+------------------------------------------------------------------+
CDFInternalBuffers::~CDFInternalBuffers(void)
  {

  }
//+------------------------------------------------------------------+
//| Decision forest class                                            |
//+------------------------------------------------------------------+
class CDForest
  {
private:
   //--- private methods
   static int        DFClsError(CDecisionForest &df,CMatrixDouble &xy,const int npoints);
   static void       DFProcessInternal(CDecisionForest &df,const int offs,double &x[],double &y[]);
   static void       DFBuildTree(CMatrixDouble &xy,const int npoints,const int nvars,const int nclasses,const int nfeatures,const int nvarsinpool,const int flags,CDFInternalBuffers &bufs);
   static void       DFBuildTreeRec(CMatrixDouble &xy,const int npoints,const int nvars,const int nclasses,const int nfeatures,int nvarsinpool,const int flags,int &numprocessed,const int idx1,const int idx2,CDFInternalBuffers &bufs);
   static void       DFSplitC(double &x[],int &c[],int &cntbuf[],const int n,const int nc,const int flags,int &info,double &threshold,double &e,double &sortrbuf[],int &sortibuf[]);
   static void       DFSplitR(double &x[],double &y[],const int n,const int flags,int &info,double &threshold,double &e,double &sortrbuf[],double &sortrbuf2[]);
public:
   //--- class constants
   static const int  m_innernodewidth;
   static const int  m_leafnodewidth;
   static const int  m_dfusestrongsplits;
   static const int  m_dfuseevs;
   static const int  m_dffirstversion;
   //--- constructor, destructor
                     CDForest(void);
                    ~CDForest(void);
   //--- public methods
   static void       DFBuildRandomDecisionForest(CMatrixDouble &xy,const int npoints,const int nvars,const int nclasses,const int ntrees,const double r,int &info,CDecisionForest &df,CDFReport &rep);
   static void       DFBuildRandomDecisionForestX1(CMatrixDouble &xy,const int npoints,const int nvars,const int nclasses,const int ntrees,const int nrndvars,const double r,int &info,CDecisionForest &df,CDFReport &rep);
   static void       DFBuildInternal(CMatrixDouble &xy,const int npoints,const int nvars,const int nclasses,const int ntrees,const int samplesize,const int nfeatures,const int flags,int &info,CDecisionForest &df,CDFReport &rep);
   static void       DFProcess(CDecisionForest &df,double &x[],double &y[]);
   static void       DFProcessI(CDecisionForest &df,double &x[],double &y[]);
   static double     DFRelClsError(CDecisionForest &df,CMatrixDouble &xy,const int npoints);
   static double     DFAvgCE(CDecisionForest &df,CMatrixDouble &xy,const int npoints);
   static double     DFRMSError(CDecisionForest &df,CMatrixDouble &xy,const int npoints);
   static double     DFAvgError(CDecisionForest &df,CMatrixDouble &xy,const int npoints);
   static double     DFAvgRelError(CDecisionForest &df,CMatrixDouble &xy,const int npoints);
   static void       DFCopy(CDecisionForest &df1,CDecisionForest &df2);
   static void       DFAlloc(CSerializer &s,CDecisionForest &forest);
   static void       DFSerialize(CSerializer &s,CDecisionForest &forest);
   static void       DFUnserialize(CSerializer &s,CDecisionForest &forest);
  };
//+------------------------------------------------------------------+
//| Initialize constants                                             |
//+------------------------------------------------------------------+
const int CDForest::m_innernodewidth=3;
const int CDForest::m_leafnodewidth=2;
const int CDForest::m_dfusestrongsplits=1;
const int CDForest::m_dfuseevs=2;
const int CDForest::m_dffirstversion=0;
//+------------------------------------------------------------------+
//| Constructor without parameters                                   |
//+------------------------------------------------------------------+
CDForest::CDForest(void)
  {

  }
//+------------------------------------------------------------------+
//| Destructor                                                       |
//+------------------------------------------------------------------+
CDForest::~CDForest(void)
  {

  }
//+------------------------------------------------------------------+
//| This subroutine builds random decision forest.                   |
//| INPUT PARAMETERS:                                                |
//|     XY          -   training set                                 |
//|     NPoints     -   training set size, NPoints>=1                |
//|     NVars       -   number of independent variables, NVars>=1    |
//|     NClasses    -   task type:                                   |
//|                     * NClasses=1 - regression task with one      |
//|                                    dependent variable            |
//|                     * NClasses>1 - classification task with      |
//|                                    NClasses classes.             |
//|     NTrees      -   number of trees in a forest, NTrees>=1.      |
//|                     recommended values: 50-100.                  |
//|     R           -   percent of a training set used to build      |
//|                     individual trees. 0<R<=1.                    |
//|                     recommended values: 0.1 <= R <= 0.66.        |
//| OUTPUT PARAMETERS:                                               |
//|     Info        -   return code:                                 |
//|                     * -2, if there is a point with class number  |
//|                           outside of [0..NClasses-1].            |
//|                     * -1, if incorrect parameters was passed     |
//|                           (NPoints<1, NVars<1, NClasses<1,       |
//|                           NTrees<1, R<=0 or R>1).                |
//|                     *  1, if task has been solved                |
//|     DF          -   model built                                  |
//|     Rep         -   training report, contains error on a training|
//|                     set and out-of-bag estimates of              |
//|                     generalization error.                        |
//+------------------------------------------------------------------+
static void CDForest::DFBuildRandomDecisionForest(CMatrixDouble &xy,
                                                  const int npoints,
                                                  const int nvars,
                                                  const int nclasses,
                                                  const int ntrees,
                                                  const double r,int &info,
                                                  CDecisionForest &df,
                                                  CDFReport &rep)
  {
//--- create a variable
   int samplesize=0;
//--- initialization
   info=0;
//--- check
   if(r<=0.0 || r>1.0)
     {
      info=-1;
      return;
     }
//--- calculation
   samplesize=(int)(MathMax((int)MathRound(r*npoints),1));
//--- function call
   DFBuildInternal(xy,npoints,nvars,nclasses,ntrees,samplesize,(int)(MathMax(nvars/2,1)),m_dfusestrongsplits+m_dfuseevs,info,df,rep);
  }
//+------------------------------------------------------------------+
//| This subroutine builds random decision forest.                   |
//| This function gives ability to tune number of variables used when|
//| choosing best split.                                             |
//| INPUT PARAMETERS:                                                |
//|     XY          -   training set                                 |
//|     NPoints     -   training set size, NPoints>=1                |
//|     NVars       -   number of independent variables, NVars>=1    |
//|     NClasses    -   task type:                                   |
//|                     * NClasses=1 - regression task with one      |
//|                                    dependent variable            |
//|                     * NClasses>1 - classification task with      |
//|                                    NClasses classes.             |
//|     NTrees      -   number of trees in a forest, NTrees>=1.      |
//|                     recommended values: 50-100.                  |
//|     NRndVars    -   number of variables used when choosing best  |
//|                     split                                        |
//|     R           -   percent of a training set used to build      |
//|                     individual trees. 0<R<=1.                    |
//|                     recommended values: 0.1 <= R <= 0.66.        |
//| OUTPUT PARAMETERS:                                               |
//|     Info        -   return code:                                 |
//|                     * -2, if there is a point with class number  |
//|                           outside of [0..NClasses-1].            |
//|                     * -1, if incorrect parameters was passed     |
//|                           (NPoints<1, NVars<1, NClasses<1,       |
//|                           NTrees<1, R<=0 or R>1).                |
//|                     *  1, if task has been solved                |
//|     DF          -   model built                                  |
//|     Rep         -   training report, contains error on a training| 
//|                     set and out-of-bag estimates of              |
//|                     generalization error.                        |
//+------------------------------------------------------------------+
static void CDForest::DFBuildRandomDecisionForestX1(CMatrixDouble &xy,
                                                    const int npoints,
                                                    const int nvars,
                                                    const int nclasses,
                                                    const int ntrees,
                                                    const int nrndvars,
                                                    const double r,int &info,
                                                    CDecisionForest &df,
                                                    CDFReport &rep)
  {
//--- create a variable
   int samplesize=0;
//--- initialization
   info=0;
//--- check
   if(r<=0.0 || r>1.0)
     {
      info=-1;
      return;
     }
//--- check
   if(nrndvars<=0 || nrndvars>nvars)
     {
      info=-1;
      return;
     }
//--- calculation
   samplesize=(int)(MathMax((int)MathRound(r*npoints),1));
//--- function call
   DFBuildInternal(xy,npoints,nvars,nclasses,ntrees,samplesize,nrndvars,m_dfusestrongsplits+m_dfuseevs,info,df,rep);
  }
//+------------------------------------------------------------------+
//| Class method                                                     |
//+------------------------------------------------------------------+
static void CDForest::DFBuildInternal(CMatrixDouble &xy,const int npoints,
                                      const int nvars,const int nclasses,
                                      const int ntrees,const int samplesize,
                                      const int nfeatures,const int flags,
                                      int &info,CDecisionForest &df,
                                      CDFReport &rep)
  {
//--- create variables
   int    i=0;
   int    j=0;
   int    k=0;
   int    tmpi=0;
   int    lasttreeoffs=0;
   int    offs=0;
   int    ooboffs=0;
   int    treesize=0;
   int    nvarsinpool=0;
   bool   useevs;
   int    oobcnt=0;
   int    oobrelcnt=0;
   double v=0;
   double vmin=0;
   double vmax=0;
   bool   bflag;
   int    i_=0;
   int    i1_=0;
//--- creating arrays
   int    permbuf[];
   double oobbuf[];
   int    oobcntbuf[];
   double x[];
   double y[];
//--- create matrix
   CMatrixDouble xys;
//--- create object of class
   CDFInternalBuffers bufs;
//--- initialization
   info=0;
//--- Test for inputs
   if(npoints<1 || samplesize<1 || samplesize>npoints || nvars<1 || nclasses<1 || ntrees<1 || nfeatures<1)
     {
      info=-1;
      return;
     }
//--- check
   if(nclasses>1)
     {
      for(i=0;i<=npoints-1;i++)
        {
         //--- check
         if((int)MathRound(xy[i][nvars])<0 || (int)MathRound(xy[i][nvars])>=nclasses)
           {
            info=-2;
            return;
           }
        }
     }
//--- change value
   info=1;
//--- Flags
   useevs=flags/m_dfuseevs%2!=0;
//--- Allocate data,prepare header
   treesize=1+m_innernodewidth*(samplesize-1)+m_leafnodewidth*samplesize;
//--- allocation
   ArrayResizeAL(permbuf,npoints);
   ArrayResizeAL(bufs.m_treebuf,treesize);
   ArrayResizeAL(bufs.m_idxbuf,npoints);
   ArrayResizeAL(bufs.m_tmpbufr,npoints);
   ArrayResizeAL(bufs.m_tmpbufr2,npoints);
   ArrayResizeAL(bufs.m_tmpbufi,npoints);
   ArrayResizeAL(bufs.m_sortrbuf,npoints);
   ArrayResizeAL(bufs.m_sortrbuf2,npoints);
   ArrayResizeAL(bufs.m_sortibuf,npoints);
   ArrayResizeAL(bufs.m_varpool,nvars);
   ArrayResizeAL(bufs.m_evsbin,nvars);
   ArrayResizeAL(bufs.m_evssplits,nvars);
   ArrayResizeAL(bufs.m_classibuf,2*nclasses);
   ArrayResizeAL(oobbuf,nclasses*npoints);
   ArrayResizeAL(oobcntbuf,npoints);
   ArrayResizeAL(df.m_trees,ntrees*treesize);
   xys.Resize(samplesize,nvars+1);
   ArrayResizeAL(x,nvars);
   ArrayResizeAL(y,nclasses);
//--- initialization
   for(i=0;i<=npoints-1;i++)
      permbuf[i]=i;
   for(i=0;i<=npoints*nclasses-1;i++)
      oobbuf[i]=0;
   for(i=0;i<=npoints-1;i++)
      oobcntbuf[i]=0;
//--- Prepare variable pool and EVS (extended variable selection/splitting) buffers
//--- (whether EVS is turned on or not):
//--- 1. detect binary variables and pre-calculate splits for them
//--- 2. detect variables with non-distinct values and exclude them from pool
   for(i=0;i<=nvars-1;i++)
      bufs.m_varpool[i]=i;
   nvarsinpool=nvars;
//--- check
   if(useevs)
     {
      for(j=0;j<=nvars-1;j++)
        {
         vmin=xy[0][j];
         vmax=vmin;
         //--- calculation
         for(i=0;i<=npoints-1;i++)
           {
            v=xy[i][j];
            vmin=MathMin(vmin,v);
            vmax=MathMax(vmax,v);
           }
         //--- check
         if(vmin==vmax)
           {
            //--- exclude variable from pool
            bufs.m_varpool[j]=bufs.m_varpool[nvarsinpool-1];
            bufs.m_varpool[nvarsinpool-1]=-1;
            nvarsinpool=nvarsinpool-1;
            continue;
           }
         //--- change value
         bflag=false;
         for(i=0;i<=npoints-1;i++)
           {
            v=xy[i][j];
            //--- check
            if(v!=vmin&&v!=vmax)
              {
               bflag=true;
               break;
              }
           }
         //--- check
         if(bflag)
           {
            //--- non-binary variable
            bufs.m_evsbin[j]=false;
           }
         else
           {
            //--- Prepare
            bufs.m_evsbin[j]=true;
            bufs.m_evssplits[j]=0.5*(vmin+vmax);
            //--- check
            if(bufs.m_evssplits[j]<=vmin)
               bufs.m_evssplits[j]=vmax;
           }
        }
     }
//--- RANDOM FOREST FORMAT
//--- W[0]         -   size of array
//--- W[1]         -   version number
//--- W[2]         -   NVars
//--- W[3]         -   NClasses (1 for regression)
//--- W[4]         -   NTrees
//--- W[5]         -   trees offset
//--- TREE FORMAT
//--- W[Offs]      -   size of sub-array
//---     node info:
//--- W[K+0]       -   variable number        (-1 for leaf mode)
//--- W[K+1]       -   threshold              (class/value for leaf node)
//--- W[K+2]       -   ">=" branch index      (absent for leaf node)
   df.m_nvars=nvars;
   df.m_nclasses=nclasses;
   df.m_ntrees=ntrees;
//--- Build forest
   offs=0;
   for(i=0;i<=ntrees-1;i++)
     {
      //--- Prepare sample
      for(k=0;k<=samplesize-1;k++)
        {
         //--- calculation
         j=k+CMath::RandomInteger(npoints-k);
         tmpi=permbuf[k];
         permbuf[k]=permbuf[j];
         permbuf[j]=tmpi;
         j=permbuf[k];
         for(i_=0;i_<=nvars;i_++)
            xys[k].Set(i_,xy[j][i_]);
        }
      //--- build tree,copy
      DFBuildTree(xys,samplesize,nvars,nclasses,nfeatures,nvarsinpool,flags,bufs);
      //--- calculation
      j=(int)MathRound(bufs.m_treebuf[0]);
      i1_=-offs;
      for(i_=offs;i_<=offs+j-1;i_++)
         df.m_trees[i_]=bufs.m_treebuf[i_+i1_];
      lasttreeoffs=offs;
      offs=offs+j;
      //--- OOB estimates
      for(k=samplesize;k<=npoints-1;k++)
        {
         for(j=0;j<=nclasses-1;j++)
            y[j]=0;
         j=permbuf[k];
         for(i_=0;i_<=nvars-1;i_++)
            x[i_]=xy[j][i_];
         //--- function call
         DFProcessInternal(df,lasttreeoffs,x,y);
         //--- calculation
         i1_=-j*nclasses;
         for(i_=j*nclasses;i_<=(j+1)*nclasses-1;i_++)
            oobbuf[i_]=oobbuf[i_]+y[i_+i1_];
         oobcntbuf[j]=oobcntbuf[j]+1;
        }
     }
   df.m_bufsize=offs;
//--- Normalize OOB results
   for(i=0;i<=npoints-1;i++)
     {
      //--- check
      if(oobcntbuf[i]!=0)
        {
         v=1.0/(double)oobcntbuf[i];
         for(i_=i*nclasses;i_<=i*nclasses+nclasses-1;i_++)
            oobbuf[i_]=v*oobbuf[i_];
        }
     }
//--- Calculate training set estimates
   rep.m_relclserror=DFRelClsError(df,xy,npoints);
   rep.m_avgce=DFAvgCE(df,xy,npoints);
   rep.m_rmserror=DFRMSError(df,xy,npoints);
   rep.m_avgerror=DFAvgError(df,xy,npoints);
   rep.m_avgrelerror=DFAvgRelError(df,xy,npoints);
//--- Calculate OOB estimates.
   rep.m_oobrelclserror=0;
   rep.m_oobavgce=0;
   rep.m_oobrmserror=0;
   rep.m_oobavgerror=0;
   rep.m_oobavgrelerror=0;
   oobcnt=0;
   oobrelcnt=0;
   for(i=0;i<=npoints-1;i++)
     {
      //--- check
      if(oobcntbuf[i]!=0)
        {
         ooboffs=i*nclasses;
         //--- check
         if(nclasses>1)
           {
            //--- classification-specific code
            k=(int)MathRound(xy[i][nvars]);
            tmpi=0;
            for(j=1;j<=nclasses-1;j++)
              {
               //--- check
               if(oobbuf[ooboffs+j]>oobbuf[ooboffs+tmpi])
                  tmpi=j;
              }
            //--- check
            if(tmpi!=k)
               rep.m_oobrelclserror=rep.m_oobrelclserror+1;
            //--- check
            if(oobbuf[ooboffs+k]!=0.0)
               rep.m_oobavgce=rep.m_oobavgce-MathLog(oobbuf[ooboffs+k]);
            else
               rep.m_oobavgce=rep.m_oobavgce-MathLog(CMath::m_minrealnumber);
            //--- calculation
            for(j=0;j<=nclasses-1;j++)
              {
               //--- check
               if(j==k)
                 {
                  rep.m_oobrmserror=rep.m_oobrmserror+CMath::Sqr(oobbuf[ooboffs+j]-1);
                  rep.m_oobavgerror=rep.m_oobavgerror+MathAbs(oobbuf[ooboffs+j]-1);
                  rep.m_oobavgrelerror=rep.m_oobavgrelerror+MathAbs(oobbuf[ooboffs+j]-1);
                  oobrelcnt=oobrelcnt+1;
                 }
               else
                 {
                  rep.m_oobrmserror=rep.m_oobrmserror+CMath::Sqr(oobbuf[ooboffs+j]);
                  rep.m_oobavgerror=rep.m_oobavgerror+MathAbs(oobbuf[ooboffs+j]);
                 }
              }
           }
         else
           {
            //--- regression-specific code
            rep.m_oobrmserror=rep.m_oobrmserror+CMath::Sqr(oobbuf[ooboffs]-xy[i][nvars]);
            rep.m_oobavgerror=rep.m_oobavgerror+MathAbs(oobbuf[ooboffs]-xy[i][nvars]);
            //--- check
            if(xy[i][nvars]!=0.0)
              {
               rep.m_oobavgrelerror=rep.m_oobavgrelerror+MathAbs((oobbuf[ooboffs]-xy[i][nvars])/xy[i][nvars]);
               oobrelcnt=oobrelcnt+1;
              }
           }
         //--- update OOB estimates count.
         oobcnt=oobcnt+1;
        }
     }
//--- check
   if(oobcnt>0)
     {
      //--- change values
      rep.m_oobrelclserror=rep.m_oobrelclserror/oobcnt;
      rep.m_oobavgce=rep.m_oobavgce/oobcnt;
      rep.m_oobrmserror=MathSqrt(rep.m_oobrmserror/(oobcnt*nclasses));
      rep.m_oobavgerror=rep.m_oobavgerror/(oobcnt*nclasses);
      //--- check
      if(oobrelcnt>0)
         rep.m_oobavgrelerror=rep.m_oobavgrelerror/oobrelcnt;
     }
  }
//+------------------------------------------------------------------+
//| Procesing                                                        |
//| INPUT PARAMETERS:                                                |
//|     DF      -   decision forest model                            |
//|     X       -   input vector,  array[0..NVars-1].                |
//| OUTPUT PARAMETERS:                                               |
//|     Y       -   result. Regression estimate when solving         |
//|                 regression task, vector of posterior             |
//|                 probabilities for classification task.           |
//| See also DFProcessI.                                             |
//+------------------------------------------------------------------+
static void CDForest::DFProcess(CDecisionForest &df,double &x[],double &y[])
  {
//--- create variables
   int    offs=0;
   int    i=0;
   double v=0;
   int    i_=0;
//--- Proceed
   if(CAp::Len(y)<df.m_nclasses)
      ArrayResizeAL(y,df.m_nclasses);
//--- initialization
   offs=0;
   for(i=0;i<=df.m_nclasses-1;i++)
      y[i]=0;
   for(i=0;i<=df.m_ntrees-1;i++)
     {
      //--- Process basic tree
      DFProcessInternal(df,offs,x,y);
      //--- Next tree
      offs=offs+(int)MathRound(df.m_trees[offs]);
     }
//--- calculation
   v=1.0/(double)df.m_ntrees;
   for(i_=0;i_<=df.m_nclasses-1;i_++)
      y[i_]=v*y[i_];
  }
//+------------------------------------------------------------------+
//| 'interactive' variant of DFProcess for languages like Python     |
//| which support constructs like "Y = DFProcessI(DF,X)" and         |
//| interactive mode of interpreter                                  |
//| This function allocates new array on each call, so it is         |
//| significantly slower than its 'non-interactive' counterpart, but |
//| it is more convenient when you call it from command line.        |
//+------------------------------------------------------------------+
static void CDForest::DFProcessI(CDecisionForest &df,double &x[],double &y[])
  {
//--- function call
   DFProcess(df,x,y);
  }
//+------------------------------------------------------------------+
//| Relative classification error on the test set                    |
//| INPUT PARAMETERS:                                                |
//|     DF      -   decision forest model                            |
//|     XY      -   test set                                         |
//|     NPoints -   test set size                                    |
//| RESULT:                                                          |
//|     percent of incorrectly classified cases.                     |
//|     Zero if model solves regression task.                        |
//+------------------------------------------------------------------+
static double CDForest::DFRelClsError(CDecisionForest &df,CMatrixDouble &xy,
                                      const int npoints)
  {
//--- return result
   return((double)DFClsError(df,xy,npoints)/(double)npoints);
  }
//+------------------------------------------------------------------+
//| Average cross-entropy (in bits per element) on the test set      |
//| INPUT PARAMETERS:                                                |
//|     DF      -   decision forest model                            |
//|     XY      -   test set                                         |
//|     NPoints -   test set size                                    |
//| RESULT:                                                          |
//|     CrossEntropy/(NPoints*LN(2)).                                |
//|     Zero if model solves regression task.                        |
//+------------------------------------------------------------------+
static double CDForest::DFAvgCE(CDecisionForest &df,CMatrixDouble &xy,
                                const int npoints)
  {
//--- create variables
   double result=0;
   int    i=0;
   int    j=0;
   int    k=0;
   int    tmpi=0;
   int    i_=0;
//--- creating arrays
   double x[];
   double y[];
//--- allocation
   ArrayResizeAL(x,df.m_nvars);
   ArrayResizeAL(y,df.m_nclasses);
//--- initialization
   result=0;
   for(i=0;i<=npoints-1;i++)
     {
      for(i_=0;i_<=df.m_nvars-1;i_++)
         x[i_]=xy[i][i_];
      //--- function call
      DFProcess(df,x,y);
      //--- check
      if(df.m_nclasses>1)
        {
         //--- classification-specific code
         k=(int)MathRound(xy[i][df.m_nvars]);
         tmpi=0;
         for(j=1;j<=df.m_nclasses-1;j++)
           {
            //--- check
            if(y[j]>(double)(y[tmpi]))
               tmpi=j;
           }
         //--- check
         if(y[k]!=0.0)
            result=result-MathLog(y[k]);
         else
            result=result-MathLog(CMath::m_minrealnumber);
        }
     }
//--- return result
   return(result/npoints);
  }
//+------------------------------------------------------------------+
//| RMS error on the test set                                        |
//| INPUT PARAMETERS:                                                |
//|     DF      -   decision forest model                            |
//|     XY      -   test set                                         |
//|     NPoints -   test set size                                    |
//| RESULT:                                                          |
//|     root mean square error.                                      |
//|     Its meaning for regression task is obvious. As for           |
//|     classification task,RMS error means error when estimating    |
//|     posterior probabilities.                                     |
//+------------------------------------------------------------------+
static double CDForest::DFRMSError(CDecisionForest &df,CMatrixDouble &xy,
                                   const int npoints)
  {
//--- create variables
   double result=0;
   int    i=0;
   int    j=0;
   int    k=0;
   int    tmpi=0;
   int    i_=0;
//--- creating arrays
   double x[];
   double y[];
//--- allocation
   ArrayResizeAL(x,df.m_nvars);
   ArrayResizeAL(y,df.m_nclasses);
//--- initialization
   result=0;
   for(i=0;i<=npoints-1;i++)
     {
      for(i_=0;i_<=df.m_nvars-1;i_++)
         x[i_]=xy[i][i_];
      //--- function call
      DFProcess(df,x,y);
      //--- check
      if(df.m_nclasses>1)
        {
         //--- classification-specific code
         k=(int)MathRound(xy[i][df.m_nvars]);
         tmpi=0;
         for(j=1;j<=df.m_nclasses-1;j++)
           {
            //--- check
            if(y[j]>y[tmpi])
               tmpi=j;
           }
         for(j=0;j<=df.m_nclasses-1;j++)
           {
            //--- check
            if(j==k)
               result=result+CMath::Sqr(y[j]-1);
            else
               result=result+CMath::Sqr(y[j]);
           }
        }
      else
        {
         //--- regression-specific code
         result=result+CMath::Sqr(y[0]-xy[i][df.m_nvars]);
        }
     }
//--- return result
   return(MathSqrt(result/(npoints*df.m_nclasses)));
  }
//+------------------------------------------------------------------+
//| Average error on the test set                                    |
//| INPUT PARAMETERS:                                                |
//|     DF      -   decision forest model                            |
//|     XY      -   test set                                         |
//|     NPoints -   test set size                                    |
//| RESULT:                                                          |
//|     Its meaning for regression task is obvious. As for           |
//|     classification task, it means average error when estimating  |
//|     posterior probabilities.                                     |
//+------------------------------------------------------------------+
static double CDForest::DFAvgError(CDecisionForest &df,CMatrixDouble &xy,
                                   const int npoints)
  {
//--- create variables
   double result=0;
   int    i=0;
   int    j=0;
   int    k=0;
   int    i_=0;
//--- creating arrays
   double x[];
   double y[];
//--- allocation
   ArrayResizeAL(x,df.m_nvars);
   ArrayResizeAL(y,df.m_nclasses);
//--- initialization
   result=0;
   for(i=0;i<=npoints-1;i++)
     {
      //--- copy
      for(i_=0;i_<=df.m_nvars-1;i_++)
         x[i_]=xy[i][i_];
      //--- function call
      DFProcess(df,x,y);
      //--- check
      if(df.m_nclasses>1)
        {
         //--- classification-specific code
         k=(int)MathRound(xy[i][df.m_nvars]);
         for(j=0;j<=df.m_nclasses-1;j++)
           {
            //--- check
            if(j==k)
               result=result+MathAbs(y[j]-1);
            else
               result=result+MathAbs(y[j]);
           }
        }
      else
        {
         //--- regression-specific code
         result=result+MathAbs(y[0]-xy[i][df.m_nvars]);
        }
     }
//--- return result
   return(result/(npoints*df.m_nclasses));
  }
//+------------------------------------------------------------------+
//| Average relative error on the test set                           |
//| INPUT PARAMETERS:                                                |
//|     DF      -   decision forest model                            |
//|     XY      -   test set                                         |
//|     NPoints -   test set size                                    |
//| RESULT:                                                          |
//|     Its meaning for regression task is obvious. As for           |
//|     classification task, it means average relative error when    |
//|     estimating posterior probability of belonging to the correct |
//|     class.                                                       |
//+------------------------------------------------------------------+
static double CDForest::DFAvgRelError(CDecisionForest &df,CMatrixDouble &xy,
                                      const int npoints)
  {
//--- create variables
   double result=0;
   int    relcnt=0;
   int    i=0;
   int    j=0;
   int    k=0;
   int    i_=0;
//--- creating arrays
   double x[];
   double y[];
//--- allocation
   ArrayResizeAL(x,df.m_nvars);
   ArrayResizeAL(y,df.m_nclasses);
//--- initialization
   result=0;
   relcnt=0;
   for(i=0;i<=npoints-1;i++)
     {
      //--- copy
      for(i_=0;i_<=df.m_nvars-1;i_++)
         x[i_]=xy[i][i_];
      //--- function call
      DFProcess(df,x,y);
      //--- check
      if(df.m_nclasses>1)
        {
         //--- classification-specific code
         k=(int)MathRound(xy[i][df.m_nvars]);
         for(j=0;j<=df.m_nclasses-1;j++)
           {
            //--- check
            if(j==k)
              {
               result=result+MathAbs(y[j]-1);
               relcnt=relcnt+1;
              }
           }
        }
      else
        {
         //--- regression-specific code
         if(xy[i][df.m_nvars]!=0.0)
           {
            result=result+MathAbs((y[0]-xy[i][df.m_nvars])/xy[i][df.m_nvars]);
            relcnt=relcnt+1;
           }
        }
     }
//--- check
   if(relcnt>0)
      result=result/relcnt;
//--- return result
   return(result);
  }
//+------------------------------------------------------------------+
//| Copying of DecisionForest strucure                               |
//| INPUT PARAMETERS:                                                |
//|     DF1 -   original                                             |
//| OUTPUT PARAMETERS:                                               |
//|     DF2 -   copy                                                 |
//+------------------------------------------------------------------+
static void CDForest::DFCopy(CDecisionForest &df1,CDecisionForest &df2)
  {
//--- create a variable
   int i_=0;
//--- change values
   df2.m_nvars=df1.m_nvars;
   df2.m_nclasses=df1.m_nclasses;
   df2.m_ntrees=df1.m_ntrees;
   df2.m_bufsize=df1.m_bufsize;
//--- allocation
   ArrayResizeAL(df2.m_trees,df1.m_bufsize);
//--- copy
   for(i_=0;i_<=df1.m_bufsize-1;i_++)
      df2.m_trees[i_]=df1.m_trees[i_];
  }
//+------------------------------------------------------------------+
//| Serializer: allocation                                           |
//+------------------------------------------------------------------+
static void CDForest::DFAlloc(CSerializer &s,CDecisionForest &forest)
  {
//--- preparation to serialize
   s.Alloc_Entry();
   s.Alloc_Entry();
   s.Alloc_Entry();
   s.Alloc_Entry();
   s.Alloc_Entry();
   s.Alloc_Entry();
//--- function call
   CApServ::AllocRealArray(s,forest.m_trees,forest.m_bufsize);
  }
//+------------------------------------------------------------------+
//| Serializer: serialization                                        |
//+------------------------------------------------------------------+
static void CDForest::DFSerialize(CSerializer &s,CDecisionForest &forest)
  {
//--- serializetion
   s.Serialize_Int(CSCodes::GetRDFSerializationCode());
   s.Serialize_Int(m_dffirstversion);
   s.Serialize_Int(forest.m_nvars);
   s.Serialize_Int(forest.m_nclasses);
   s.Serialize_Int(forest.m_ntrees);
   s.Serialize_Int(forest.m_bufsize);
//--- function call
   CApServ::SerializeRealArray(s,forest.m_trees,forest.m_bufsize);
  }
//+------------------------------------------------------------------+
//| Serializer: unserialization                                      |
//+------------------------------------------------------------------+
static void CDForest::DFUnserialize(CSerializer &s,CDecisionForest &forest)
  {
//--- create variables
   int i0=0;
   int i1=0;
//--- check correctness of header
   i0=s.Unserialize_Int();
//--- check
   if(!CAp::Assert(i0==CSCodes::GetRDFSerializationCode(),__FUNCTION__+": stream header corrupted"))
      return;
//--- unserializetion
   i1=s.Unserialize_Int();
//--- check
   if(!CAp::Assert(i1==m_dffirstversion,__FUNCTION__+": stream header corrupted"))
      return;
//--- Unserialize data
   forest.m_nvars=s.Unserialize_Int();
   forest.m_nclasses=s.Unserialize_Int();
   forest.m_ntrees=s.Unserialize_Int();
   forest.m_bufsize=s.Unserialize_Int();
//--- function call
   CApServ::UnserializeRealArray(s,forest.m_trees);
  }
//+------------------------------------------------------------------+
//| Classification error                                             |
//+------------------------------------------------------------------+
static int CDForest::DFClsError(CDecisionForest &df,CMatrixDouble &xy,
                                const int npoints)
  {
//--- create variables
   int result=0;
   int i=0;
   int j=0;
   int k=0;
   int tmpi=0;
   int i_=0;
//--- creating arrays
   double x[];
   double y[];
//--- check
   if(df.m_nclasses<=1)
      return(0);
//--- allocation
   ArrayResizeAL(x,df.m_nvars);
   ArrayResizeAL(y,df.m_nclasses);
//--- initialization
   result=0;
   for(i=0;i<=npoints-1;i++)
     {
      //--- copy
      for(i_=0;i_<=df.m_nvars-1;i_++)
         x[i_]=xy[i][i_];
      //--- function call
      DFProcess(df,x,y);
      //--- change values
      k=(int)MathRound(xy[i][df.m_nvars]);
      tmpi=0;
      for(j=1;j<=df.m_nclasses-1;j++)
        {
         //--- check
         if(y[j]>(double)(y[tmpi]))
            tmpi=j;
        }
      //--- check
      if(tmpi!=k)
         result=result+1;
     }
//--- return result
   return(result);
  }
//+------------------------------------------------------------------+
//| Internal subroutine for processing one decision tree starting at |
//| Offs                                                             |
//+------------------------------------------------------------------+
static void CDForest::DFProcessInternal(CDecisionForest &df,const int offs,
                                        double &x[],double &y[])
  {
//--- create variables
   int k=0;
   int idx=0;
//--- Set pointer to the root
   k=offs+1;
//--- Navigate through the tree
   while(true)
     {
      //--- check
      if(df.m_trees[k]==-1.0)
        {
         //--- check
         if(df.m_nclasses==1)
            y[0]=y[0]+df.m_trees[k+1];
         else
           {
            idx=(int)MathRound(df.m_trees[k+1]);
            y[idx]=y[idx]+1;
           }
         //--- break the cycle
         break;
        }
      //--- check
      if(x[(int)MathRound(df.m_trees[k])]<df.m_trees[k+1])
         k=k+m_innernodewidth;
      else
         k=offs+(int)MathRound(df.m_trees[k+2]);
     }
  }
//+------------------------------------------------------------------+
//| Builds one decision tree. Just a wrapper for the DFBuildTreeRec. |
//+------------------------------------------------------------------+
static void CDForest::DFBuildTree(CMatrixDouble &xy,const int npoints,
                                  const int nvars,const int nclasses,
                                  const int nfeatures,const int nvarsinpool,
                                  const int flags,CDFInternalBuffers &bufs)
  {
//--- create variables
   int numprocessed=0;
   int i=0;
//--- check
   if(!CAp::Assert(npoints>0))
      return;
//--- Prepare IdxBuf. It stores indices of the training set elements.
//--- When training set is being split,contents of IdxBuf is
//--- correspondingly reordered so we can know which elements belong
//--- to which branch of decision tree.
   for(i=0;i<=npoints-1;i++)
      bufs.m_idxbuf[i]=i;
//--- Recursive procedure
   numprocessed=1;
//--- function call
   DFBuildTreeRec(xy,npoints,nvars,nclasses,nfeatures,nvarsinpool,flags,numprocessed,0,npoints-1,bufs);
//--- change values
   bufs.m_treebuf[0]=numprocessed;
  }
//+------------------------------------------------------------------+
//| Builds one decision tree (internal recursive subroutine)         |
//| Parameters:                                                      |
//|     TreeBuf     -   large enough array,at least TreeSize         |
//|     IdxBuf      -   at least NPoints elements                    |
//|     TmpBufR     -   at least NPoints                             |
//|     TmpBufR2    -   at least NPoints                             |
//|     TmpBufI     -   at least NPoints                             |
//|     TmpBufI2    -   at least NPoints+1                           |
//+------------------------------------------------------------------+
static void CDForest::DFBuildTreeRec(CMatrixDouble &xy,const int npoints,
                                     const int nvars,const int nclasses,
                                     const int nfeatures,int nvarsinpool,
                                     const int flags,int &numprocessed,
                                     const int idx1,const int idx2,
                                     CDFInternalBuffers &bufs)
  {
//--- create variables
   int    i=0;
   int    j=0;
   int    k=0;
   bool   bflag;
   int    i1=0;
   int    i2=0;
   int    info=0;
   double sl=0;
   double sr=0;
   double w=0;
   int    idxbest=0;
   double ebest=0;
   double tbest=0;
   int    varcur=0;
   double s=0;
   double v=0;
   double v1=0;
   double v2=0;
   double threshold=0;
   int    oldnp=0;
   double currms=0;
   bool   useevs;
//--- these initializers are not really necessary,
//--- but without them compiler complains about uninitialized locals
   tbest=0;
//--- Prepare
   if(!CAp::Assert(npoints>0))
      return;
//--- check
   if(!CAp::Assert(idx2>=idx1))
      return;
   useevs=flags/m_dfuseevs%2!=0;
//--- Leaf node
   if(idx2==idx1)
     {
      bufs.m_treebuf[numprocessed]=-1;
      bufs.m_treebuf[numprocessed+1]=xy[bufs.m_idxbuf[idx1]][nvars];
      numprocessed=numprocessed+m_leafnodewidth;
      //--- exit the function
      return;
     }
//--- Non-leaf node.
//--- Select random variable,prepare split:
//--- 1. prepare default solution - no splitting,class at random
//--- 2. investigate possible splits,compare with default/best
   idxbest=-1;
//--- check
   if(nclasses>1)
     {
      //--- default solution for classification
      for(i=0;i<=nclasses-1;i++)
         bufs.m_classibuf[i]=0;
      s=idx2-idx1+1;
      for(i=idx1;i<=idx2;i++)
        {
         j=(int)MathRound(xy[bufs.m_idxbuf[i]][nvars]);
         bufs.m_classibuf[j]=bufs.m_classibuf[j]+1;
        }
      //--- calculation
      ebest=0;
      for(i=0;i<=nclasses-1;i++)
         ebest=ebest+bufs.m_classibuf[i]*CMath::Sqr(1-bufs.m_classibuf[i]/s)+(s-bufs.m_classibuf[i])*CMath::Sqr(bufs.m_classibuf[i]/s);
      ebest=MathSqrt(ebest/(nclasses*(idx2-idx1+1)));
     }
   else
     {
      //--- default solution for regression
      v=0;
      for(i=idx1;i<=idx2;i++)
         v=v+xy[bufs.m_idxbuf[i]][nvars];
      v=v/(idx2-idx1+1);
      //--- calculation
      ebest=0;
      for(i=idx1;i<=idx2;i++)
         ebest=ebest+CMath::Sqr(xy[bufs.m_idxbuf[i]][nvars]-v);
      ebest=MathSqrt(ebest/(idx2-idx1+1));
     }
//--- change value
   i=0;
//--- cycle
   while(i<=MathMin(nfeatures,nvarsinpool)-1)
     {
      //--- select variables from pool
      j=i+CMath::RandomInteger(nvarsinpool-i);
      k=bufs.m_varpool[i];
      bufs.m_varpool[i]=bufs.m_varpool[j];
      bufs.m_varpool[j]=k;
      varcur=bufs.m_varpool[i];
      //--- load variable values to working array
      //--- apply EVS preprocessing: if all variable values are same,
      //--- variable is excluded from pool.
      //--- This is necessary for binary pre-splits (see later) to work.
      for(j=idx1;j<=idx2;j++)
         bufs.m_tmpbufr[j-idx1]=xy[bufs.m_idxbuf[j]][varcur];
      //--- check
      if(useevs)
        {
         bflag=false;
         v=bufs.m_tmpbufr[0];
         for(j=0;j<=idx2-idx1;j++)
           {
            //--- check
            if(bufs.m_tmpbufr[j]!=v)
              {
               bflag=true;
               break;
              }
           }
         //--- check
         if(!bflag)
           {
            //--- exclude variable from pool,
            //--- go to the next iteration.
            //--- I is not increased.
            k=bufs.m_varpool[i];
            bufs.m_varpool[i]=bufs.m_varpool[nvarsinpool-1];
            bufs.m_varpool[nvarsinpool-1]=k;
            nvarsinpool=nvarsinpool-1;
            continue;
           }
        }
      //--- load labels to working array
      if(nclasses>1)
        {
         for(j=idx1;j<=idx2;j++)
            bufs.m_tmpbufi[j-idx1]=(int)MathRound(xy[bufs.m_idxbuf[j]][nvars]);
        }
      else
        {
         for(j=idx1;j<=idx2;j++)
            bufs.m_tmpbufr2[j-idx1]=xy[bufs.m_idxbuf[j]][nvars];
        }
      //--- calculate split
      if(useevs && bufs.m_evsbin[varcur])
        {
         //--- Pre-calculated splits for binary variables.
         //--- Threshold is already known,just calculate RMS error
         threshold=bufs.m_evssplits[varcur];
         //--- check
         if(nclasses>1)
           {
            //--- classification-specific code
            for(j=0;j<=2*nclasses-1;j++)
               bufs.m_classibuf[j]=0;
            //--- change values
            sl=0;
            sr=0;
            //--- calculation
            for(j=0;j<=idx2-idx1;j++)
              {
               k=bufs.m_tmpbufi[j];
               //--- check
               if(bufs.m_tmpbufr[j]<threshold)
                 {
                  bufs.m_classibuf[k]=bufs.m_classibuf[k]+1;
                  sl=sl+1;
                 }
               else
                 {
                  bufs.m_classibuf[k+nclasses]=bufs.m_classibuf[k+nclasses]+1;
                  sr=sr+1;
                 }
              }
            //--- check
            if(!CAp::Assert(sl!=0.0 && sr!=0.0,__FUNCTION__+": something strange!"))
               return;
            //--- change values
            currms=0;
            //--- calculation
            for(j=0;j<=nclasses-1;j++)
              {
               w=bufs.m_classibuf[j];
               currms=currms+w*CMath::Sqr(w/sl-1);
               currms=currms+(sl-w)*CMath::Sqr(w/sl);
               w=bufs.m_classibuf[nclasses+j];
               currms=currms+w*CMath::Sqr(w/sr-1);
               currms=currms+(sr-w)*CMath::Sqr(w/sr);
              }
            currms=MathSqrt(currms/(nclasses*(idx2-idx1+1)));
           }
         else
           {
            //--- regression-specific code
            sl=0;
            sr=0;
            v1=0;
            v2=0;
            //--- calculation
            for(j=0;j<=idx2-idx1;j++)
              {
               //--- check
               if(bufs.m_tmpbufr[j]<threshold)
                 {
                  v1=v1+bufs.m_tmpbufr2[j];
                  sl=sl+1;
                 }
               else
                 {
                  v2=v2+bufs.m_tmpbufr2[j];
                  sr=sr+1;
                 }
              }
            //--- check
            if(!CAp::Assert(sl!=0.0 && sr!=0.0,__FUNCTION__+": something strange!"))
               return;
            //--- change values
            v1=v1/sl;
            v2=v2/sr;
            currms=0;
            for(j=0;j<=idx2-idx1;j++)
              {
               //--- check
               if(bufs.m_tmpbufr[j]<threshold)
                  currms=currms+CMath::Sqr(v1-bufs.m_tmpbufr2[j]);
               else
                  currms=currms+CMath::Sqr(v2-bufs.m_tmpbufr2[j]);
              }
            currms=MathSqrt(currms/(idx2-idx1+1));
           }
         //--- change value
         info=1;
        }
      else
        {
         //--- Generic splits
         if(nclasses>1)
            DFSplitC(bufs.m_tmpbufr,bufs.m_tmpbufi,bufs.m_classibuf,idx2-idx1+1,nclasses,m_dfusestrongsplits,info,threshold,currms,bufs.m_sortrbuf,bufs.m_sortibuf);
         else
            DFSplitR(bufs.m_tmpbufr,bufs.m_tmpbufr2,idx2-idx1+1,m_dfusestrongsplits,info,threshold,currms,bufs.m_sortrbuf,bufs.m_sortrbuf2);
        }
      //--- check
      if(info>0)
        {
         //--- check
         if(currms<=ebest)
           {
            ebest=currms;
            idxbest=varcur;
            tbest=threshold;
           }
        }
      //--- Next iteration
      i=i+1;
     }
//--- to split or not to split
   if(idxbest<0)
     {
      //--- All values are same,cannot split.
      bufs.m_treebuf[numprocessed]=-1;
      //--- check
      if(nclasses>1)
        {
         //--- Select random class label (randomness allows us to
         //--- approximate distribution of the classes)
         bufs.m_treebuf[numprocessed+1]=(int)MathRound(xy[bufs.m_idxbuf[idx1+CMath::RandomInteger(idx2-idx1+1)]][nvars]);
        }
      else
        {
         //--- Select average (for regression task).
         v=0;
         for(i=idx1;i<=idx2;i++)
            v=v+xy[bufs.m_idxbuf[i]][nvars]/(idx2-idx1+1);
         bufs.m_treebuf[numprocessed+1]=v;
        }
      //--- change value
      numprocessed=numprocessed+m_leafnodewidth;
     }
   else
     {
      //--- we can split
      bufs.m_treebuf[numprocessed]=idxbest;
      bufs.m_treebuf[numprocessed+1]=tbest;
      i1=idx1;
      i2=idx2;
      //--- cycle
      while(i1<=i2)
        {
         //--- Reorder indices so that left partition is in [Idx1..I1-1],
         //--- and right partition is in [I2+1..Idx2]
         if(xy[bufs.m_idxbuf[i1]][idxbest]<tbest)
           {
            i1=i1+1;
            continue;
           }
         //--- check
         if(xy[bufs.m_idxbuf[i2]][idxbest]>=tbest)
           {
            i2=i2-1;
            continue;
           }
         //--- change values
         j=bufs.m_idxbuf[i1];
         bufs.m_idxbuf[i1]=bufs.m_idxbuf[i2];
         bufs.m_idxbuf[i2]=j;
         i1=i1+1;
         i2=i2-1;
        }
      //--- change values
      oldnp=numprocessed;
      numprocessed=numprocessed+m_innernodewidth;
      //--- function call
      DFBuildTreeRec(xy,npoints,nvars,nclasses,nfeatures,nvarsinpool,flags,numprocessed,idx1,i1-1,bufs);
      bufs.m_treebuf[oldnp+2]=numprocessed;
      //--- function call
      DFBuildTreeRec(xy,npoints,nvars,nclasses,nfeatures,nvarsinpool,flags,numprocessed,i2+1,idx2,bufs);
     }
  }
//+------------------------------------------------------------------+
//| Makes split on attribute                                         |
//+------------------------------------------------------------------+
static void CDForest::DFSplitC(double &x[],int &c[],int &cntbuf[],const int n,
                               const int nc,const int flags,int &info,
                               double &threshold,double &e,double &sortrbuf[],
                               int &sortibuf[])
  {
//--- create variables
   int    i=0;
   int    neq=0;
   int    nless=0;
   int    ngreater=0;
   int    q=0;
   int    qmin=0;
   int    qmax=0;
   int    qcnt=0;
   double cursplit=0;
   int    nleft=0;
   double v=0;
   double cure=0;
   double w=0;
   double sl=0;
   double sr=0;
//--- initialization
   info=0;
   threshold=0;
   e=0;
//--- function call
   CTSort::TagSortFastI(x,c,sortrbuf,sortibuf,n);
//--- change values
   e=CMath::m_maxrealnumber;
   threshold=0.5*(x[0]+x[n-1]);
   info=-3;
//--- check
   if(flags/m_dfusestrongsplits%2==0)
     {
      //--- weak splits,split at half
      qcnt=2;
      qmin=1;
      qmax=1;
     }
   else
     {
      //--- strong splits: choose best quartile
      qcnt=4;
      qmin=1;
      qmax=3;
     }
   for(q=qmin;q<=qmax;q++)
     {
      //--- change values
      cursplit=x[n*q/qcnt];
      neq=0;
      nless=0;
      ngreater=0;
      //--- calculation
      for(i=0;i<=n-1;i++)
        {
         //--- check
         if(x[i]<cursplit)
            nless=nless+1;
         //--- check
         if(x[i]==cursplit)
            neq=neq+1;
         //--- check
         if(x[i]>cursplit)
            ngreater=ngreater+1;
        }
      //--- check
      if(!CAp::Assert(neq!=0,__FUNCTION__+": NEq=0,something strange!!!"))
         return;
      //--- check
      if(nless!=0 || ngreater!=0)
        {
         //--- set threshold between two partitions, with
         //--- some tweaking to avoid problems with floating point
         //--- arithmetics.
         //--- The problem is that when you calculates C = 0.5*(A+B) there
         //--- can be no C which lies strictly between A and B (for example,
         //--- there is no floating point number which is
         //--- greater than 1 and less than 1+eps). In such situations
         //--- we choose right side as theshold (remember that
         //--- points which lie on threshold falls to the right side).
         if(nless<ngreater)
           {
            cursplit=0.5*(x[nless+neq-1]+x[nless+neq]);
            nleft=nless+neq;
            //--- check
            if(cursplit<=(double)(x[nless+neq-1]))
               cursplit=x[nless+neq];
           }
         else
           {
            cursplit=0.5*(x[nless-1]+x[nless]);
            nleft=nless;
            //--- check
            if(cursplit<=(double)(x[nless-1]))
               cursplit=x[nless];
           }
         //--- change value
         info=1;
         cure=0;
         //--- calculation
         for(i=0;i<=2*nc-1;i++)
            cntbuf[i]=0;
         for(i=0;i<=nleft-1;i++)
            cntbuf[c[i]]=cntbuf[c[i]]+1;
         for(i=nleft;i<=n-1;i++)
            cntbuf[nc+c[i]]=cntbuf[nc+c[i]]+1;
         //--- change values
         sl=nleft;
         sr=n-nleft;
         v=0;
         //--- calculation
         for(i=0;i<=nc-1;i++)
           {
            w=cntbuf[i];
            v=v+w*CMath::Sqr(w/sl-1);
            v=v+(sl-w)*CMath::Sqr(w/sl);
            w=cntbuf[nc+i];
            v=v+w*CMath::Sqr(w/sr-1);
            v=v+(sr-w)*CMath::Sqr(w/sr);
           }
         cure=MathSqrt(v/(nc*n));
         //--- check
         if(cure<e)
           {
            threshold=cursplit;
            e=cure;
           }
        }
     }
  }
//+------------------------------------------------------------------+
//| Makes split on attribute                                         |
//+------------------------------------------------------------------+
static void CDForest::DFSplitR(double &x[],double &y[],const int n,const int flags,
                               int &info,double &threshold,double &e,
                               double &sortrbuf[],double &sortrbuf2[])
  {
//--- create variables
   int    i=0;
   int    neq=0;
   int    nless=0;
   int    ngreater=0;
   int    q=0;
   int    qmin=0;
   int    qmax=0;
   int    qcnt=0;
   double cursplit=0;
   int    nleft=0;
   double v=0;
   double cure=0;
//--- initialization
   info=0;
   threshold=0;
   e=0;
//--- function call
   CTSort::TagSortFastR(x,y,sortrbuf,sortrbuf2,n);
//--- change values
   e=CMath::m_maxrealnumber;
   threshold=0.5*(x[0]+x[n-1]);
   info=-3;
//--- check
   if(flags/m_dfusestrongsplits%2==0)
     {
      //--- weak splits,split at half
      qcnt=2;
      qmin=1;
      qmax=1;
     }
   else
     {
      //--- strong splits: choose best quartile
      qcnt=4;
      qmin=1;
      qmax=3;
     }
//--- calculation
   for(q=qmin;q<=qmax;q++)
     {
      //--- change values
      cursplit=x[n*q/qcnt];
      neq=0;
      nless=0;
      ngreater=0;
      for(i=0;i<=n-1;i++)
        {
         //--- check
         if(x[i]<cursplit)
            nless=nless+1;
         //--- check
         if(x[i]==cursplit)
            neq=neq+1;
         //--- check
         if(x[i]>cursplit)
            ngreater=ngreater+1;
        }
      //--- check
      if(!CAp::Assert(neq!=0,__FUNCTION__+": NEq=0,something strange!!!"))
         return;
      //--- check
      if(nless!=0 || ngreater!=0)
        {
         //--- set threshold between two partitions, with
         //--- some tweaking to avoid problems with floating point
         //--- arithmetics.
         //--- The problem is that when you calculates C = 0.5*(A+B) there
         //--- can be no C which lies strictly between A and B (for example,
         //--- there is no floating point number which is
         //--- greater than 1 and less than 1+eps). In such situations
         //--- we choose right side as theshold (remember that
         //--- points which lie on threshold falls to the right side).
         if(nless<ngreater)
           {
            cursplit=0.5*(x[nless+neq-1]+x[nless+neq]);
            nleft=nless+neq;
            //--- check
            if(cursplit<=(double)(x[nless+neq-1]))
               cursplit=x[nless+neq];
           }
         else
           {
            cursplit=0.5*(x[nless-1]+x[nless]);
            nleft=nless;
            //--- check
            if(cursplit<=(double)(x[nless-1]))
               cursplit=x[nless];
           }
         //--- change value
         info=1;
         cure=0;
         v=0;
         //--- calculation
         for(i=0;i<=nleft-1;i++)
            v=v+y[i];
         v=v/nleft;
         for(i=0;i<=nleft-1;i++)
            cure=cure+CMath::Sqr(y[i]-v);
         v=0;
         for(i=nleft;i<=n-1;i++)
            v=v+y[i];
         v=v/(n-nleft);
         for(i=nleft;i<=n-1;i++)
            cure=cure+CMath::Sqr(y[i]-v);
         cure=MathSqrt(cure/n);
         //--- check
         if(cure<e)
           {
            threshold=cursplit;
            e=cure;
           }
        }
     }
  }
//+------------------------------------------------------------------+
//| Middle and clusterization                                        |
//+------------------------------------------------------------------+
class CKMeans
  {
private:
   //--- private method
   static bool       SelectCenterPP(CMatrixDouble &xy,const int npoints,const int nvars,CMatrixDouble &centers,bool &cbusycenters[],const int ccnt,double &d2[],double &p[],double &tmp[]);
public:
   //--- constructor, destructor
                     CKMeans(void);
                    ~CKMeans(void);
   //--- public method
   static void       KMeansGenerate(CMatrixDouble &xy,const int npoints,const int nvars,const int k,const int restarts,int &info,CMatrixDouble &c,int &xyc[]);
  };
//+------------------------------------------------------------------+
//| Constructor without parameters                                   |
//+------------------------------------------------------------------+
CKMeans::CKMeans(void)
  {

  }
//+------------------------------------------------------------------+
//| Destructor                                                       |
//+------------------------------------------------------------------+
CKMeans::~CKMeans(void)
  {

  }
//+------------------------------------------------------------------+
//| k-means++ clusterization                                         |
//| INPUT PARAMETERS:                                                |
//|     XY          -   dataset, array [0..NPoints-1,0..NVars-1].    |
//|     NPoints     -   dataset size, NPoints>=K                     |
//|     NVars       -   number of variables, NVars>=1                |
//|     K           -   desired number of clusters, K>=1             |
//|     Restarts    -   number of restarts, Restarts>=1              |
//| OUTPUT PARAMETERS:                                               |
//|     Info        -   return code:                                 |
//|                     * -3, if task is degenerate (number of       |
//|                           distinct points is less than K)        |
//|                     * -1, if incorrect                           |
//|                           NPoints/NFeatures/K/Restarts was passed|
//|                     *  1, if subroutine finished successfully    |
//|     C           -   array[0..NVars-1,0..K-1].matrix whose columns|
//|                     store cluster's centers                      |
//|     XYC         -   array[NPoints], which contains cluster       |
//|                     indexes                                      |
//+------------------------------------------------------------------+
static void CKMeans::KMeansGenerate(CMatrixDouble &xy,const int npoints,
                                    const int nvars,const int k,
                                    const int restarts,int &info,
                                    CMatrixDouble &c,int &xyc[])
  {
//--- create variables
   int    i=0;
   int    j=0;
   double e=0;
   double ebest=0;
   double v=0;
   int    cclosest=0;
   bool   waschanges;
   bool   zerosizeclusters;
   int    pass=0;
   int    i_=0;
   double dclosest=0;
//--- creating arrays
   int    xycbest[];
   double x[];
   double tmp[];
   double d2[];
   double p[];
   int    csizes[];
   bool   cbusy[];
   double work[];
//--- create matrix
   CMatrixDouble ct;
   CMatrixDouble ctbest;
//--- initialization
   info=0;
//--- Test parameters
   if(npoints<k || nvars<1 || k<1 || restarts<1)
     {
      info=-1;
      return;
     }
//--- TODO: special case K=1
//--- TODO: special case K=NPoints
   info=1;
//--- Multiple passes of k-means++ algorithm
   ct.Resize(k,nvars);
   ctbest.Resize(k,nvars);
   ArrayResizeAL(xyc,npoints);
   ArrayResizeAL(xycbest,npoints);
   ArrayResizeAL(d2,npoints);
   ArrayResizeAL(p,npoints);
   ArrayResizeAL(tmp,nvars);
   ArrayResizeAL(csizes,k);
   ArrayResizeAL(cbusy,k);
//--- change value
   ebest=CMath::m_maxrealnumber;
//--- calculation
   for(pass=1;pass<=restarts;pass++)
     {
      //--- Select initial centers  using k-means++ algorithm
      //--- 1. Choose first center at random
      //--- 2. Choose next centers using their distance from centers already chosen
      //--- Note that for performance reasons centers are stored in ROWS of CT,not
      //--- in columns. We'll transpose CT in the end and store it in the C.
      i=CMath::RandomInteger(npoints);
      for(i_=0;i_<=nvars-1;i_++)
         ct[0].Set(i_,xy[i][i_]);
      cbusy[0]=true;
      for(i=1;i<=k-1;i++)
         cbusy[i]=false;
      //--- check
      if(!SelectCenterPP(xy,npoints,nvars,ct,cbusy,k,d2,p,tmp))
        {
         info=-3;
         return;
        }
      //--- Update centers:
      //--- 2. update center positions
      for(i=0;i<=npoints-1;i++)
         xyc[i]=-1;
      //--- cycle
      while(true)
        {
         //--- fill XYC with center numbers
         waschanges=false;
         for(i=0;i<=npoints-1;i++)
           {
            //--- change values
            cclosest=-1;
            dclosest=CMath::m_maxrealnumber;
            for(j=0;j<=k-1;j++)
              {
               //--- calculation
               for(i_=0;i_<=nvars-1;i_++)
                  tmp[i_]=xy[i][i_];
               for(i_=0;i_<=nvars-1;i_++)
                  tmp[i_]=tmp[i_]-ct[j][i_];
               v=0.0;
               for(i_=0;i_<=nvars-1;i_++)
                  v+=tmp[i_]*tmp[i_];
               //--- check
               if(v<dclosest)
                 {
                  cclosest=j;
                  dclosest=v;
                 }
              }
            //--- check
            if(xyc[i]!=cclosest)
               waschanges=true;
            //--- change value
            xyc[i]=cclosest;
           }
         //--- Update centers
         for(j=0;j<=k-1;j++)
            csizes[j]=0;
         for(i=0;i<=k-1;i++)
           {
            for(j=0;j<=nvars-1;j++)
               ct[i].Set(j,0);
           }
         //--- change values
         for(i=0;i<=npoints-1;i++)
           {
            csizes[xyc[i]]=csizes[xyc[i]]+1;
            for(i_=0;i_<=nvars-1;i_++)
               ct[xyc[i]].Set(i_,ct[xyc[i]][i_]+xy[i][i_]);
           }
         zerosizeclusters=false;
         for(i=0;i<=k-1;i++)
           {
            cbusy[i]=csizes[i]!=0;
            zerosizeclusters=zerosizeclusters || csizes[i]==0;
           }
         //--- check
         if(zerosizeclusters)
           {
            //--- Some clusters have zero size - rare,but possible.
            //--- We'll choose new centers for such clusters using k-means++ rule
            //--- and restart algorithm
            if(!SelectCenterPP(xy,npoints,nvars,ct,cbusy,k,d2,p,tmp))
              {
               info=-3;
               return;
              }
            continue;
           }
         //--- copy
         for(j=0;j<=k-1;j++)
           {
            v=1.0/(double)csizes[j];
            for(i_=0;i_<=nvars-1;i_++)
               ct[j].Set(i_,v*ct[j][i_]);
           }
         //--- if nothing has changed during iteration
         if(!waschanges)
            break;
        }
      //--- 3. Calculate E,compare with best centers found so far
      e=0;
      for(i=0;i<=npoints-1;i++)
        {
         for(i_=0;i_<=nvars-1;i_++)
            tmp[i_]=xy[i][i_];
         for(i_=0;i_<=nvars-1;i_++)
            tmp[i_]=tmp[i_]-ct[xyc[i]][i_];
         //--- calculation
         v=0.0;
         for(i_=0;i_<=nvars-1;i_++)
            v+=tmp[i_]*tmp[i_];
         e=e+v;
        }
      //--- check
      if(e<ebest)
        {
         //--- store partition.
         ebest=e;
         //--- function call
         CBlas::CopyMatrix(ct,0,k-1,0,nvars-1,ctbest,0,k-1,0,nvars-1);
         //--- copy
         for(i=0;i<=npoints-1;i++)
            xycbest[i]=xyc[i];
        }
     }
//--- Copy and transpose
   c.Resize(nvars,k);
//--- function call
   CBlas::CopyAndTranspose(ctbest,0,k-1,0,nvars-1,c,0,nvars-1,0,k-1);
//--- copy
   for(i=0;i<=npoints-1;i++)
      xyc[i]=xycbest[i];
  }
//+------------------------------------------------------------------+
//| Select center for a new cluster using k-means++ rule             |
//+------------------------------------------------------------------+
static bool CKMeans::SelectCenterPP(CMatrixDouble &xy,const int npoints,
                                    const int nvars,CMatrixDouble &centers,
                                    bool &cbusycenters[],const int ccnt,
                                    double &d2[],double &p[],double &tmp[])
  {
//--- create variables
   bool   result;
   int    i=0;
   int    j=0;
   int    cc=0;
   double v=0;
   double s=0;
   int    i_=0;
//--- create array
   double busycenters[];
//--- copy
   ArrayCopy(busycenters,cbusycenters);
//--- initialization
   result=true;
//--- calculation
   for(cc=0;cc<=ccnt-1;cc++)
     {
      //--- check
      if(!busycenters[cc])
        {
         //--- fill D2
         for(i=0;i<=npoints-1;i++)
           {
            d2[i]=CMath::m_maxrealnumber;
            for(j=0;j<=ccnt-1;j++)
              {
               //--- check
               if(busycenters[j])
                 {
                  for(i_=0;i_<=nvars-1;i_++)
                     tmp[i_]=xy[i][i_];
                  for(i_=0;i_<=nvars-1;i_++)
                     tmp[i_]=tmp[i_]-centers[j][i_];
                  //--- calculation
                  v=0.0;
                  for(i_=0;i_<=nvars-1;i_++)
                     v+=tmp[i_]*tmp[i_];
                  //--- check
                  if(v<d2[i])
                     d2[i]=v;
                 }
              }
           }
         //--- calculate P (non-cumulative)
         s=0;
         for(i=0;i<=npoints-1;i++)
            s=s+d2[i];
         //--- check
         if(s==0.0)
            return(false);
         //--- change value
         s=1/s;
         for(i_=0;i_<=npoints-1;i_++)
            p[i_]=s*d2[i_];
         //--- choose one of points with probability P
         //--- random number within (0,1) is generated and
         //--- inverse empirical CDF is used to randomly choose a point.
         s=0;
         v=CMath::RandomReal();
         //--- calculation
         for(i=0;i<=npoints-1;i++)
           {
            s=s+p[i];
            //--- check
            if(v<=s || i==npoints-1)
              {
               for(i_=0;i_<=nvars-1;i_++)
                  centers[cc].Set(i_,xy[i][i_]);
               busycenters[cc]=true;
               //--- break the cycle
               break;
              }
           }
        }
     }
//--- return result
   return(result);
  }
//+------------------------------------------------------------------+
//| Multiclass Fisher LDA                                            |
//+------------------------------------------------------------------+
class CLDA
  {
public:
   //--- constructor, destructor
                     CLDA(void);
                    ~CLDA(void);
   //--- methods
   static void       FisherLDA(CMatrixDouble &xy,const int npoints,const int nvars,const int nclasses,int &info,double &w[]);
   static void       FisherLDAN(CMatrixDouble &xy,const int npoints,const int nvars,const int nclasses,int &info,CMatrixDouble &w);
  };
//+------------------------------------------------------------------+
//| Constructor without parameters                                   |
//+------------------------------------------------------------------+
CLDA::CLDA(void)
  {

  }
//+------------------------------------------------------------------+
//| Destructor                                                       |
//+------------------------------------------------------------------+
CLDA::~CLDA(void)
  {

  }
//+------------------------------------------------------------------+
//| Multiclass Fisher LDA                                            |
//| Subroutine finds coefficients of linear combination which        |
//| optimally separates training set on classes.                     |
//| INPUT PARAMETERS:                                                |
//|     XY          -   training set, array[0..NPoints-1,0..NVars].  |
//|                     First NVars columns store values of          |
//|                     independent variables, next column stores    |
//|                     number of class (from 0 to NClasses-1) which |
//|                     dataset element belongs to. Fractional values|
//|                     are rounded to nearest integer.              |
//|     NPoints     -   training set size, NPoints>=0                |
//|     NVars       -   number of independent variables, NVars>=1    |
//|     NClasses    -   number of classes, NClasses>=2               |
//| OUTPUT PARAMETERS:                                               |
//|     Info        -   return code:                                 |
//|                     * -4, if internal EVD subroutine hasn't      |
//|                           converged                              |
//|                     * -2, if there is a point with class number  |
//|                           outside of [0..NClasses-1].            |
//|                     * -1, if incorrect parameters was passed     |
//|                           (NPoints<0, NVars<1, NClasses<2)       |
//|                     *  1, if task has been solved                |
//|                     *  2, if there was a multicollinearity in    |
//|                           training set, but task has been solved.|
//|     W           -   linear combination coefficients,             |
//|                     array[0..NVars-1]                            |
//+------------------------------------------------------------------+
static void CLDA::FisherLDA(CMatrixDouble &xy,const int npoints,
                            const int nvars,const int nclasses,
                            int &info,double &w[])
  {
//--- create a variable
   int i_=0;
//--- create matrix
   CMatrixDouble w2;
//--- initialization
   info=0;
//--- function call
   FisherLDAN(xy,npoints,nvars,nclasses,info,w2);
//--- check
   if(info>0)
     {
      //--- allocation
      ArrayResizeAL(w,nvars);
      //--- copy
      for(i_=0;i_<=nvars-1;i_++)
         w[i_]=w2[i_][0];
     }
  }
//+------------------------------------------------------------------+
//| N-dimensional multiclass Fisher LDA                              |
//| Subroutine finds coefficients of linear combinations which       |
//| optimally separates                                              |
//| training set on classes. It returns N-dimensional basis whose    |
//| vector are sorted                                                |
//| by quality of training set separation (in descending order).     |
//| INPUT PARAMETERS:                                                |
//|     XY          -   training set, array[0..NPoints-1,0..NVars].  |
//|                     First NVars columns store values of          |
//|                     independent variables, next column stores    |
//|                     number of class (from 0 to NClasses-1) which |
//|                     dataset element belongs to. Fractional values|
//|                     are rounded to nearest integer.              |
//|     NPoints     -   training set size, NPoints>=0                |
//|     NVars       -   number of independent variables, NVars>=1    |
//|     NClasses    -   number of classes, NClasses>=2               |
//| OUTPUT PARAMETERS:                                               |
//|     Info        -   return code:                                 |
//|                     * -4, if internal EVD subroutine hasn't      |
//|                           converged                              |
//|                     * -2, if there is a point with class number  |
//|                           outside of [0..NClasses-1].            |
//|                     * -1, if incorrect parameters was passed     |
//|                           (NPoints<0, NVars<1, NClasses<2)       |
//|                     *  1, if task has been solved                |
//|                     *  2, if there was a multicollinearity in    |
//|                           training set, but task has been solved.|
//|     W           -   basis, array[0..NVars-1,0..NVars-1]          |
//|                     columns of matrix stores basis vectors,      |
//|                     sorted by quality of training set separation | 
//|                     (in descending order)                        |
//+------------------------------------------------------------------+
static void CLDA::FisherLDAN(CMatrixDouble &xy,const int npoints,const int nvars,
                             const int nclasses,int &info,CMatrixDouble &w)
  {
//--- create variables
   int    i=0;
   int    j=0;
   int    k=0;
   int    m=0;
   double v=0;
   int    i_=0;
//--- creating arrays
   int    c[];
   double mu[];
   int    nc[];
   double tf[];
   double d[];
   double d2[];
   double work[];
//--- create matrix
   CMatrixDouble muc;
   CMatrixDouble sw;
   CMatrixDouble st;
   CMatrixDouble z;
   CMatrixDouble z2;
   CMatrixDouble tm;
   CMatrixDouble sbroot;
   CMatrixDouble a;
   CMatrixDouble xyproj;
   CMatrixDouble wproj;
//--- initialization
   info=0;
//--- Test data
   if((npoints<0 || nvars<1) || nclasses<2)
     {
      info=-1;
      return;
     }
   for(i=0;i<=npoints-1;i++)
     {
      //--- check
      if((int)MathRound(xy[i][nvars])<0 || (int)MathRound(xy[i][nvars])>=nclasses)
        {
         info=-2;
         return;
        }
     }
//--- change value
   info=1;
//--- Special case: NPoints<=1
//--- Degenerate task.
   if(npoints<=1)
     {
      info=2;
      //--- allocation
      w.Resize(nvars,nvars);
      //--- initialization
      for(i=0;i<=nvars-1;i++)
        {
         for(j=0;j<=nvars-1;j++)
           {
            //--- check
            if(i==j)
               w[i].Set(j,1);
            else
               w[i].Set(j,0);
           }
        }
      //--- exit the function
      return;
     }
//--- Prepare temporaries
   ArrayResizeAL(tf,nvars);
   ArrayResizeAL(work,MathMax(nvars,npoints)+1);
//--- Convert class labels from reals to integers (just for convenience)
   ArrayResizeAL(c,npoints);
   for(i=0;i<=npoints-1;i++)
      c[i]=(int)MathRound(xy[i][nvars]);
//--- Calculate class sizes and means
   ArrayResizeAL(mu,nvars);
   muc.Resize(nclasses,nvars);
   ArrayResizeAL(nc,nclasses);
   for(j=0;j<=nvars-1;j++)
      mu[j]=0;
   for(i=0;i<=nclasses-1;i++)
     {
      nc[i]=0;
      for(j=0;j<=nvars-1;j++)
         muc[i].Set(j,0);
     }
//--- calculation
   for(i=0;i<=npoints-1;i++)
     {
      for(i_=0;i_<=nvars-1;i_++)
         mu[i_]=mu[i_]+xy[i][i_];
      for(i_=0;i_<=nvars-1;i_++)
         muc[c[i]].Set(i_,muc[c[i]][i_]+xy[i][i_]);
      nc[c[i]]=nc[c[i]]+1;
     }
   for(i=0;i<=nclasses-1;i++)
     {
      v=1.0/(double)nc[i];
      for(i_=0;i_<=nvars-1;i_++)
         muc[i].Set(i_,v*muc[i][i_]);
     }
//--- change values
   v=1.0/(double)npoints;
   for(i_=0;i_<=nvars-1;i_++)
      mu[i_]=v*mu[i_];
//--- Create ST matrix
   st.Resize(nvars,nvars);
   for(i=0;i<=nvars-1;i++)
     {
      for(j=0;j<=nvars-1;j++)
         st[i].Set(j,0);
     }
//--- calculation
   for(k=0;k<=npoints-1;k++)
     {
      for(i_=0;i_<=nvars-1;i_++)
         tf[i_]=xy[k][i_];
      for(i_=0;i_<=nvars-1;i_++)
         tf[i_]=tf[i_]-mu[i_];
      for(i=0;i<=nvars-1;i++)
        {
         v=tf[i];
         for(i_=0;i_<=nvars-1;i_++)
            st[i].Set(i_,st[i][i_]+v*tf[i_]);
        }
     }
//--- Create SW matrix
   sw.Resize(nvars,nvars);
   for(i=0;i<=nvars-1;i++)
     {
      for(j=0;j<=nvars-1;j++)
         sw[i].Set(j,0);
     }
//--- calculation
   for(k=0;k<=npoints-1;k++)
     {
      for(i_=0;i_<=nvars-1;i_++)
         tf[i_]=xy[k][i_];
      for(i_=0;i_<=nvars-1;i_++)
         tf[i_]=tf[i_]-muc[c[k]][i_];
      for(i=0;i<=nvars-1;i++)
        {
         v=tf[i];
         for(i_=0;i_<=nvars-1;i_++)
            sw[i].Set(i_,sw[i][i_]+v*tf[i_]);
        }
     }
//--- Maximize ratio J=(w'*ST*w)/(w'*SW*w).
//--- First,make transition from w to v such that w'*ST*w becomes v'*v:
//---    v=root(ST)*w=R*w
//---    R=root(D)*Z'
//---    w=(root(ST)^-1)*v=RI*v
//---    RI=Z*inv(root(D))
//---    J=(v'*v)/(v'*(RI'*SW*RI)*v)
//---    ST=Z*D*Z'
//---    so we have
//---    J=(v'*v) / (v'*(inv(root(D))*Z'*SW*Z*inv(root(D)))*v)=
//=(v'*v) / (v'*A*v)
   if(!CEigenVDetect::SMatrixEVD(st,nvars,1,true,d,z))
     {
      info=-4;
      return;
     }
//--- allocation
   w.Resize(nvars,nvars);
//--- check
   if(d[nvars-1]<=0.0 || d[0]<=1000*CMath::m_machineepsilon*d[nvars-1])
     {
      //--- Special case: D[NVars-1]<=0
      //--- Degenerate task (all variables takes the same value).
      if(d[nvars-1]<=0.0)
        {
         info=2;
         for(i=0;i<=nvars-1;i++)
           {
            for(j=0;j<=nvars-1;j++)
              {
               //--- check
               if(i==j)
                  w[i].Set(j,1);
               else
                  w[i].Set(j,0);
              }
           }
         //--- exit the function
         return;
        }
      //--- Special case: degenerate ST matrix,multicollinearity found.
      //--- Since we know ST eigenvalues/vectors we can translate task to
      //--- non-degenerate form.
      //--- Let WG is orthogonal basis of the non zero variance subspace
      //--- of the ST and let WZ is orthogonal basis of the zero variance
      //--- subspace.
      //--- Projection on WG allows us to use LDA on reduced M-dimensional
      //--- subspace,N-M vectors of WZ allows us to update reduced LDA
      //--- factors to full N-dimensional subspace.
      m=0;
      for(k=0;k<=nvars-1;k++)
        {
         //--- check
         if(d[k]<=1000*CMath::m_machineepsilon*d[nvars-1])
            m=k+1;
        }
      //--- check
      if(!CAp::Assert(m!=0,__FUNCTION__+": internal error #1"))
         return;
      //--- allocation
      xyproj.Resize(npoints,nvars-m+1);
      //--- function call
      CBlas::MatrixMatrixMultiply(xy,0,npoints-1,0,nvars-1,false,z,0,nvars-1,m,nvars-1,false,1.0,xyproj,0,npoints-1,0,nvars-m-1,0.0,work);
      for(i=0;i<=npoints-1;i++)
         xyproj[i].Set(nvars-m,xy[i][nvars]);
      //--- function call
      FisherLDAN(xyproj,npoints,nvars-m,nclasses,info,wproj);
      //--- check
      if(info<0)
         return;
      //--- function call
      CBlas::MatrixMatrixMultiply(z,0,nvars-1,m,nvars-1,false,wproj,0,nvars-m-1,0,nvars-m-1,false,1.0,w,0,nvars-1,0,nvars-m-1,0.0,work);
      //--- change values
      for(k=nvars-m;k<=nvars-1;k++)
        {
         for(i_=0;i_<=nvars-1;i_++)
            w[i_].Set(k,z[i_][k-nvars+m]);
        }
      info=2;
     }
   else
     {
      //--- General case: no multicollinearity
      tm.Resize(nvars,nvars);
      a.Resize(nvars,nvars);
      //--- function call
      CBlas::MatrixMatrixMultiply(sw,0,nvars-1,0,nvars-1,false,z,0,nvars-1,0,nvars-1,false,1.0,tm,0,nvars-1,0,nvars-1,0.0,work);
      CBlas::MatrixMatrixMultiply(z,0,nvars-1,0,nvars-1,true,tm,0,nvars-1,0,nvars-1,false,1.0,a,0,nvars-1,0,nvars-1,0.0,work);
      //--- change values
      for(i=0;i<=nvars-1;i++)
        {
         for(j=0;j<=nvars-1;j++)
            a[i].Set(j,a[i][j]/MathSqrt(d[i]*d[j]));
        }
      //--- check
      if(!CEigenVDetect::SMatrixEVD(a,nvars,1,true,d2,z2))
        {
         info=-4;
         return;
        }
      //--- calculation
      for(k=0;k<=nvars-1;k++)
        {
         for(i=0;i<=nvars-1;i++)
            tf[i]=z2[i][k]/MathSqrt(d[i]);
         for(i=0;i<=nvars-1;i++)
           {
            v=0.0;
            for(i_=0;i_<=nvars-1;i_++)
               v+=z[i][i_]*tf[i_];
            w[i].Set(k,v);
           }
        }
     }
//--- Post-processing:
//--- * normalization
//--- * converting to non-negative form,if possible
   for(k=0;k<=nvars-1;k++)
     {
      //--- calculation
      v=0.0;
      for(i_=0;i_<=nvars-1;i_++)
         v+=w[i_][k]*w[i_][k];
      v=1/MathSqrt(v);
      for(i_=0;i_<=nvars-1;i_++)
         w[i_].Set(k,v*w[i_][k]);
      v=0;
      for(i=0;i<=nvars-1;i++)
         v=v+w[i][k];
      //--- check
      if(v<0.0)
        {
         for(i_=0;i_<=nvars-1;i_++)
            w[i_].Set(k,-1*w[i_][k]);
        }
     }
  }
//+------------------------------------------------------------------+
//| Auxiliary class for CLinReg                                      |
//+------------------------------------------------------------------+
class CLinearModel
  {
public:
   double            m_w[];
   //--- constructor, destructor
                     CLinearModel(void);
                    ~CLinearModel(void);
   //--- copy
   void              Copy(CLinearModel &obj);
  };
//+------------------------------------------------------------------+
//| Constructor without parameters                                   |
//+------------------------------------------------------------------+
CLinearModel::CLinearModel(void)
  {

  }
//+------------------------------------------------------------------+
//| Destructor                                                       |
//+------------------------------------------------------------------+
CLinearModel::~CLinearModel(void)
  {

  }
//+------------------------------------------------------------------+
//| Copy                                                             |
//+------------------------------------------------------------------+
void CLinearModel::Copy(CLinearModel &obj)
  {
//--- copy array
   ArrayCopy(m_w,obj.m_w);
  }
//+------------------------------------------------------------------+
//| This class is a shell for class CLinearModel                     |
//+------------------------------------------------------------------+
class CLinearModelShell
  {
private:
   CLinearModel      m_innerobj;
public:
   //--- constructors, destructor
                     CLinearModelShell(void);
                     CLinearModelShell(CLinearModel &obj);
                    ~CLinearModelShell(void);
   //--- method
   CLinearModel     *GetInnerObj(void);
  };
//+------------------------------------------------------------------+
//| Constructor without parameters                                   |
//+------------------------------------------------------------------+
CLinearModelShell::CLinearModelShell(void)
  {

  }
//+------------------------------------------------------------------+
//| Copy constructor                                                 |
//+------------------------------------------------------------------+
CLinearModelShell::CLinearModelShell(CLinearModel &obj)
  {
//--- copy
   m_innerobj.Copy(obj);
  }
//+------------------------------------------------------------------+
//| Destructor                                                       |
//+------------------------------------------------------------------+
CLinearModelShell::~CLinearModelShell(void)
  {

  }
//+------------------------------------------------------------------+
//| Return object of class                                           |
//+------------------------------------------------------------------+
CLinearModel *CLinearModelShell::GetInnerObj(void)
  {
//--- return result
   return(GetPointer(m_innerobj));
  }
//+------------------------------------------------------------------+
//| LRReport structure contains additional information about linear  |
//| model:                                                           |
//| * C             -   covariation matrix, array[0..NVars,0..NVars].|
//|                     C[i,j] = Cov(A[i],A[j])                      |
//| * RMSError      -   root mean square error on a training set     |
//| * AvgError      -   average error on a training set              |
//| * AvgRelError   -   average relative error on a training set     |
//|                     (excluding observations with zero function   |
//|                     value).                                      |
//| * CVRMSError    -   leave-one-out cross-validation estimate of   |
//|                     generalization error. Calculated using fast  |
//|                     algorithm with O(NVars*NPoints) complexity.  |
//| * CVAvgError    -   cross-validation estimate of average error   |
//| * CVAvgRelError -   cross-validation estimate of average relative| 
//|                     error                                        |
//| All other fields of the structure are intended for internal use  |
//| and should not be used outside ALGLIB.                           |
//+------------------------------------------------------------------+
class CLRReport
  {
public:
   //--- variables
   double            m_rmserror;
   double            m_avgerror;
   double            m_avgrelerror;
   double            m_cvrmserror;
   double            m_cvavgerror;
   double            m_cvavgrelerror;
   int               m_ncvdefects;
   //--- array
   int               m_cvdefects[];
   //--- matrix
   CMatrixDouble     m_c;
   //--- constructor, destructor
                     CLRReport(void);
                    ~CLRReport(void);
   //--- copy
   void              Copy(CLRReport &obj);
  };
//+------------------------------------------------------------------+
//| Constructor without parameters                                   |
//+------------------------------------------------------------------+
CLRReport::CLRReport(void)
  {

  }
//+------------------------------------------------------------------+
//| Destructor                                                       |
//+------------------------------------------------------------------+
CLRReport::~CLRReport(void)
  {

  }
//+------------------------------------------------------------------+
//| Copy                                                             |
//+------------------------------------------------------------------+
void CLRReport::Copy(CLRReport &obj)
  {
//--- copy variables
   m_rmserror=obj.m_rmserror;
   m_avgerror=obj.m_avgerror;
   m_avgrelerror=obj.m_avgrelerror;
   m_cvrmserror=obj.m_cvrmserror;
   m_cvavgerror=obj.m_cvavgerror;
   m_cvavgrelerror=obj.m_cvavgrelerror;
   m_ncvdefects=obj.m_ncvdefects;
//--- copy array
   ArrayCopy(m_cvdefects,obj.m_cvdefects);
//--- copy matrix
   m_c=obj.m_c;
  }
//+------------------------------------------------------------------+
//| LRReport structure contains additional information about linear  | 
//| model:                                                           |
//| * C             -   covariation matrix, array[0..NVars,0..NVars].|
//|                     C[i,j]=Cov(A[i],A[j])                        |
//| * RMSError      -   root mean square error on a training set     |
//| * AvgError      -   average error on a training set              |
//| * AvgRelError   -   average relative error on a training set     |
//|                     (excluding observations with zero function   |
//|                     value).                                      |
//| * CVRMSError    -   leave-one-out cross-validation estimate of   |
//|                     generalization error. Calculated using fast  |
//|                     algorithm with O(NVars*NPoints) complexity.  |
//| * CVAvgError    -   cross-validation estimate of average error   |
//| * CVAvgRelError -   cross-validation estimate of average relative| 
//|                     error                                        |
//| All other fields of the structure are intended for internal use  | 
//| and should not be used outside ALGLIB.                           |
//+------------------------------------------------------------------+
class CLRReportShell
  {
private:
   CLRReport         m_innerobj;
public:
   //--- constructors, destructor
                     CLRReportShell(void);
                     CLRReportShell(CLRReport &obj);
                    ~CLRReportShell(void);
   //--- methods
   double            GetRMSError(void);
   void              SetRMSError(const double d);
   double            GetAvgError(void);
   void              SetAvgError(const double d);
   double            GetAvgRelError(void);
   void              SetAvgRelError(const double d);
   double            GetCVRMSError(void);
   void              SetCVRMSError(const double d);
   double            GetCVAvgError(void);
   void              SetCVAvgError(const double d);
   double            GetCVAvgRelError(void);
   void              SetCVAvgRelError(const double d);
   int               GetNCVDEfects(void);
   void              SetNCVDEfects(const int i);
   CLRReport        *GetInnerObj(void);
  };
//+------------------------------------------------------------------+
//| Constructor without parameters                                   |
//+------------------------------------------------------------------+
CLRReportShell::CLRReportShell(void)
  {

  }
//+------------------------------------------------------------------+
//| Copy constructor                                                 |
//+------------------------------------------------------------------+
CLRReportShell::CLRReportShell(CLRReport &obj)
  {
//--- copy
   m_innerobj.Copy(obj);
  }
//+------------------------------------------------------------------+
//| Destructor                                                       |
//+------------------------------------------------------------------+
CLRReportShell::~CLRReportShell(void)
  {

  }
//+------------------------------------------------------------------+
//| Returns the value of the variable rmserror                       |
//+------------------------------------------------------------------+
double CLRReportShell::GetRMSError(void)
  {
//--- return result
   return(m_innerobj.m_rmserror);
  }
//+------------------------------------------------------------------+
//| Changing the value of the variable rmserror                      |
//+------------------------------------------------------------------+
void CLRReportShell::SetRMSError(const double d)
  {
//--- change value
   m_innerobj.m_rmserror=d;
  }
//+------------------------------------------------------------------+
//| Returns the value of the variable avgerror                       |
//+------------------------------------------------------------------+
double CLRReportShell::GetAvgError(void)
  {
//--- return result
   return(m_innerobj.m_avgerror);
  }
//+------------------------------------------------------------------+
//| Changing the value of the variable avgerror                      |
//+------------------------------------------------------------------+
void CLRReportShell::SetAvgError(const double d)
  {
//--- change value
   m_innerobj.m_avgerror=d;
  }
//+------------------------------------------------------------------+
//| Returns the value of the variable avgrelerror                    |
//+------------------------------------------------------------------+
double CLRReportShell::GetAvgRelError(void)
  {
//--- return result
   return(m_innerobj.m_avgrelerror);
  }
//+------------------------------------------------------------------+
//| Changing the value of the variable avgrelerror                   |
//+------------------------------------------------------------------+
void CLRReportShell::SetAvgRelError(const double d)
  {
//--- change value
   m_innerobj.m_avgrelerror=d;
  }
//+------------------------------------------------------------------+
//| Returns the value of the variable cvrmserror                     |
//+------------------------------------------------------------------+
double CLRReportShell::GetCVRMSError(void)
  {
//--- return result
   return(m_innerobj.m_cvrmserror);
  }
//+------------------------------------------------------------------+
//| Changing the value of the variable cvrmserror                    |
//+------------------------------------------------------------------+
void CLRReportShell::SetCVRMSError(const double d)
  {
//--- change value
   m_innerobj.m_cvrmserror=d;
  }
//+------------------------------------------------------------------+
//| Returns the value of the variable cvavgerror                     |
//+------------------------------------------------------------------+
double CLRReportShell::GetCVAvgError(void)
  {
//--- return result
   return(m_innerobj.m_cvavgerror);
  }
//+------------------------------------------------------------------+
//| Changing the value of the variable cvavgerror                    |
//+------------------------------------------------------------------+
void CLRReportShell::SetCVAvgError(const double d)
  {
//--- change value
   m_innerobj.m_cvavgerror=d;
  }
//+------------------------------------------------------------------+
//| Returns the value of the variable cvavgrelerror                  |
//+------------------------------------------------------------------+
double CLRReportShell::GetCVAvgRelError(void)
  {
//--- return result
   return(m_innerobj.m_cvavgrelerror);
  }
//+------------------------------------------------------------------+
//| Changing the value of the variable cvavgrelerror                 |
//+------------------------------------------------------------------+
void CLRReportShell::SetCVAvgRelError(const double d)
  {
//--- change value
   m_innerobj.m_cvavgrelerror=d;
  }
//+------------------------------------------------------------------+
//| Returns the value of the variable ncvdefects                     |
//+------------------------------------------------------------------+
int CLRReportShell::GetNCVDEfects(void)
  {
//--- return result
   return(m_innerobj.m_ncvdefects);
  }
//+------------------------------------------------------------------+
//| Changing the value of the variable ncvdefects                    |
//+------------------------------------------------------------------+
void CLRReportShell::SetNCVDEfects(const int i)
  {
//--- change value
   m_innerobj.m_ncvdefects=i;
  }
//+------------------------------------------------------------------+
//| Return object of class                                           |
//+------------------------------------------------------------------+
CLRReport *CLRReportShell::GetInnerObj(void)
  {
//--- return result
   return(GetPointer(m_innerobj));
  }
//+------------------------------------------------------------------+
//| Linear regression class                                          |
//+------------------------------------------------------------------+
class CLinReg
  {
private:
   //--- private method
   static void       LRInternal(CMatrixDouble &xy,double &s[],const int npoints,const int nvars,int &info,CLinearModel &lm,CLRReport &ar);
public:
   //--- constant
   static const int  m_lrvnum;
   //--- constructor, destructor
                     CLinReg(void);
                    ~CLinReg(void);
   //--- public methods
   static void       LRBuild(CMatrixDouble &xy,const int npoints,const int nvars,int &info,CLinearModel &lm,CLRReport &ar);
   static void       LRBuildS(CMatrixDouble &xy,double &s[],const int npoints,const int nvars,int &info,CLinearModel &lm,CLRReport &ar);
   static void       LRBuildZS(CMatrixDouble &xy,double &s[],const int npoints,const int nvars,int &info,CLinearModel &lm,CLRReport &ar);
   static void       LRBuildZ(CMatrixDouble &xy,const int npoints,const int nvars,int &info,CLinearModel &lm,CLRReport &ar);
   static void       LRUnpack(CLinearModel &lm,double &v[],int &nvars);
   static void       LRPack(double &v[],const int nvars,CLinearModel &lm);
   static double     LRProcess(CLinearModel &lm,double &x[]);
   static double     LRRMSError(CLinearModel &lm,CMatrixDouble &xy,const int npoints);
   static double     LRAvgError(CLinearModel &lm,CMatrixDouble &xy,const int npoints);
   static double     LRAvgRelError(CLinearModel &lm,CMatrixDouble &xy,const int npoints);
   static void       LRCopy(CLinearModel &lm1,CLinearModel &lm2);
   static void       LRLines(CMatrixDouble &xy,double &s[],const int n,int &info,double &a,double &b,double &vara,double &varb,double &covab,double &corrab,double &p);
   static void       LRLine(CMatrixDouble &xy,const int n,int &info,double &a,double &b);
  };
//+------------------------------------------------------------------+
//| Initialize constant                                              |
//+------------------------------------------------------------------+
const int CLinReg::m_lrvnum=5;
//+------------------------------------------------------------------+
//| Constructor without parameters                                   |
//+------------------------------------------------------------------+
CLinReg::CLinReg(void)
  {

  }
//+------------------------------------------------------------------+
//| Destructor                                                       |
//+------------------------------------------------------------------+
CLinReg::~CLinReg(void)
  {

  }
//+------------------------------------------------------------------+
//| Linear regression                                                |
//| Subroutine builds model:                                         |
//|     Y = A(0)*X[0] + ... + A(N-1)*X[N-1] + A(N)                   |
//| and model found in ALGLIB format, covariation matrix, training   | 
//| set errors (rms, average, average relative) and leave-one-out    |
//| cross-validation estimate of the generalization error. CV        |
//| estimate calculated using fast algorithm with O(NPoints*NVars)   |
//| complexity.                                                      |
//| When  covariation  matrix  is  calculated  standard deviations of| 
//| function values are assumed to be equal to RMS error on the      |
//| training set.                                                    |
//| INPUT PARAMETERS:                                                |
//|     XY          -   training set, array [0..NPoints-1,0..NVars]: |
//|                     * NVars columns - independent variables      |
//|                     * last column - dependent variable           |
//|     NPoints     -   training set size, NPoints>NVars+1           |
//|     NVars       -   number of independent variables              |
//| OUTPUT PARAMETERS:                                               |
//|     Info        -   return code:                                 |
//|                     * -255, in case of unknown internal error    |
//|                     * -4, if internal SVD subroutine haven't     |
//|                           converged                              |
//|                     * -1, if incorrect parameters was passed     |
//|                           (NPoints<NVars+2, NVars<1).            |
//|                     *  1, if subroutine successfully finished    |
//|     LM          -   linear model in the ALGLIB format. Use       |
//|                     subroutines of this unit to work with the    |
//|                     model.                                       |
//|     AR          -   additional results                           |
//+------------------------------------------------------------------+
static void CLinReg::LRBuild(CMatrixDouble &xy,const int npoints,const int nvars,
                             int &info,CLinearModel &lm,CLRReport &ar)
  {
//--- create variables
   int    i=0;
   double sigma2=0;
   int    i_=0;
//--- create array
   double s[];
//--- initialization
   info=0;
//--- check
   if(npoints<=nvars+1 || nvars<1)
     {
      info=-1;
      return;
     }
//--- allocation
   ArrayResizeAL(s,npoints);
   for(i=0;i<=npoints-1;i++)
      s[i]=1;
//--- function call
   LRBuildS(xy,s,npoints,nvars,info,lm,ar);
//--- check
   if(info<0)
      return;
//--- calculation
   sigma2=CMath::Sqr(ar.m_rmserror)*npoints/(npoints-nvars-1);
   for(i=0;i<=nvars;i++)
     {
      for(i_=0;i_<=nvars;i_++)
         ar.m_c[i].Set(i_,sigma2*ar.m_c[i][i_]);
     }
  }
//+------------------------------------------------------------------+
//| Linear regression                                                |
//| Variant of LRBuild which uses vector of standatd deviations      |
//| (errors in function values).                                     |
//| INPUT PARAMETERS:                                                |
//|     XY          -   training set, array [0..NPoints-1,0..NVars]: |
//|                     * NVars columns - independent variables      |
//|                     * last column - dependent variable           |
//|     S           -   standard deviations (errors in function      |
//|                     values) array[0..NPoints-1], S[i]>0.         |
//|     NPoints     -   training set size, NPoints>NVars+1           |
//|     NVars       -   number of independent variables              |
//| OUTPUT PARAMETERS:                                               |
//|     Info        -   return code:                                 |
//|                     * -255, in case of unknown internal error    |
//|                     * -4, if internal SVD subroutine haven't     |
//|                     converged                                    |
//|                     * -1, if incorrect parameters was passed     |
//|                     (NPoints<NVars+2, NVars<1).                  |
//|                     * -2, if S[I]<=0                             |
//|                     *  1, if subroutine successfully finished    |
//|     LM          -   linear model in the ALGLIB format. Use       |
//|                     subroutines of this unit to work with the    |
//|                     model.                                       |
//|     AR          -   additional results                           |
//+------------------------------------------------------------------+
static void CLinReg::LRBuildS(CMatrixDouble &xy,double &s[],const int npoints,
                              const int nvars,int &info,CLinearModel &lm,
                              CLRReport &ar)
  {
//--- create variables
   int    i=0;
   int    j=0;
   double v=0;
   int    offs=0;
   double mean=0;
   double variance=0;
   double skewness=0;
   double kurtosis=0;
   int    i_=0;
//--- creating arrays
   double x[];
   double means[];
   double sigmas[];
//--- create array
   CMatrixDouble xyi;
//--- initialization
   info=0;
//--- Test parameters
   if(npoints<=nvars+1 || nvars<1)
     {
      info=-1;
      return;
     }
//--- Copy data,add one more column (constant term)
   xyi.Resize(npoints,nvars+2);
   for(i=0;i<=npoints-1;i++)
     {
      for(i_=0;i_<=nvars-1;i_++)
         xyi[i].Set(i_,xy[i][i_]);
      xyi[i].Set(nvars,1);
      xyi[i].Set(nvars+1,xy[i][nvars]);
     }
//--- Standartization
   ArrayResizeAL(x,npoints);
   ArrayResizeAL(means,nvars);
   ArrayResizeAL(sigmas,nvars);
   for(j=0;j<=nvars-1;j++)
     {
      //--- copy
      for(i_=0;i_<=npoints-1;i_++)
         x[i_]=xy[i_][j];
      //--- function call
      CBaseStat::SampleMoments(x,npoints,mean,variance,skewness,kurtosis);
      //--- change values
      means[j]=mean;
      sigmas[j]=MathSqrt(variance);
      //--- check
      if(sigmas[j]==0.0)
         sigmas[j]=1;
      //--- calculation
      for(i=0;i<=npoints-1;i++)
         xyi[i].Set(j,(xyi[i][j]-means[j])/sigmas[j]);
     }
//--- Internal processing
   LRInternal(xyi,s,npoints,nvars+1,info,lm,ar);
//--- check
   if(info<0)
      return;
//--- Un-standartization
   offs=(int)MathRound(lm.m_w[3]);
   for(j=0;j<=nvars-1;j++)
     {
      //--- Constant term is updated (and its covariance too,
      //--- since it gets some variance from J-th component)
      lm.m_w[offs+nvars]=lm.m_w[offs+nvars]-lm.m_w[offs+j]*means[j]/sigmas[j];
      v=means[j]/sigmas[j];
      for(i_=0;i_<=nvars;i_++)
         ar.m_c[nvars].Set(i_,ar.m_c[nvars][i_]-v*ar.m_c[j][i_]);
      for(i_=0;i_<=nvars;i_++)
         ar.m_c[i_].Set(nvars,ar.m_c[i_][nvars]-v*ar.m_c[i_][j]);
      //--- J-th term is updated
      lm.m_w[offs+j]=lm.m_w[offs+j]/sigmas[j];
      v=1/sigmas[j];
      for(i_=0;i_<=nvars;i_++)
         ar.m_c[j].Set(i_,v*ar.m_c[j][i_]);
      for(i_=0;i_<=nvars;i_++)
         ar.m_c[i_].Set(j,v*ar.m_c[i_][j]);
     }
  }
//+------------------------------------------------------------------+
//| Like LRBuildS, but builds model                                  |
//|     Y=A(0)*X[0] + ... + A(N-1)*X[N-1]                            |
//| i.m_e. with zero constant term.                                  |
//+------------------------------------------------------------------+
static void CLinReg::LRBuildZS(CMatrixDouble &xy,double &s[],const int npoints,
                               const int nvars,int &info,CLinearModel &lm,
                               CLRReport &ar)
  {
//--- create variables
   int    i=0;
   int    j=0;
   double v=0;
   int    offs=0;
   double mean=0;
   double variance=0;
   double skewness=0;
   double kurtosis=0;
   int    i_=0;
//--- creating arrays
   double x[];
   double c[];
//--- create matrix
   CMatrixDouble xyi;
//--- initialization
   info=0;
//--- Test parameters
   if(npoints<=nvars+1 || nvars<1)
     {
      info=-1;
      return;
     }
//--- Copy data,add one more column (constant term)
   xyi.Resize(npoints,nvars+2);
   for(i=0;i<=npoints-1;i++)
     {
      for(i_=0;i_<=nvars-1;i_++)
         xyi[i].Set(i_,xy[i][i_]);
      xyi[i].Set(nvars,0);
      xyi[i].Set(nvars+1,xy[i][nvars]);
     }
//--- Standartization: unusual scaling
   ArrayResizeAL(x,npoints);
   ArrayResizeAL(c,nvars);
   for(j=0;j<=nvars-1;j++)
     {
      for(i_=0;i_<=npoints-1;i_++)
         x[i_]=xy[i_][j];
      //--- function call
      CBaseStat::SampleMoments(x,npoints,mean,variance,skewness,kurtosis);
      //--- check
      if(MathAbs(mean)>MathSqrt(variance))
        {
         //--- variation is relatively small,it is better to
         //--- bring mean value to 1
         c[j]=mean;
        }
      else
        {
         //--- variation is large,it is better to bring variance to 1
         if(variance==0.0)
            variance=1;
         c[j]=MathSqrt(variance);
        }
      for(i=0;i<=npoints-1;i++)
         xyi[i].Set(j,xyi[i][j]/c[j]);
     }
//--- Internal processing
   LRInternal(xyi,s,npoints,nvars+1,info,lm,ar);
//--- check
   if(info<0)
      return;
//--- Un-standartization
   offs=(int)MathRound(lm.m_w[3]);
   for(j=0;j<=nvars-1;j++)
     {
      //--- J-th term is updated
      lm.m_w[offs+j]=lm.m_w[offs+j]/c[j];
      v=1/c[j];
      for(i_=0;i_<=nvars;i_++)
         ar.m_c[j].Set(i_,v*ar.m_c[j][i_]);
      for(i_=0;i_<=nvars;i_++)
         ar.m_c[i_].Set(j,v*ar.m_c[i_][j]);
     }
  }
//+------------------------------------------------------------------+
//| Like LRBuild but builds model                                    |
//|     Y=A(0)*X[0] + ... + A(N-1)*X[N-1]                            |
//| i.m_e. with zero constant term.                                  |
//+------------------------------------------------------------------+
static void CLinReg::LRBuildZ(CMatrixDouble &xy,const int npoints,const int nvars,
                              int &info,CLinearModel &lm,CLRReport &ar)
  {
//--- create variables
   int i=0;
   double sigma2=0;
   int i_=0;
//--- create array
   double s[];
//--- initialization
   info=0;
//--- check
   if(npoints<=nvars+1 || nvars<1)
     {
      info=-1;
      return;
     }
//--- allocation
   ArrayResizeAL(s,npoints);
   for(i=0;i<=npoints-1;i++)
      s[i]=1;
//--- function call
   LRBuildZS(xy,s,npoints,nvars,info,lm,ar);
//--- check
   if(info<0)
      return;
//--- calculation
   sigma2=CMath::Sqr(ar.m_rmserror)*npoints/(npoints-nvars-1);
   for(i=0;i<=nvars;i++)
     {
      for(i_=0;i_<=nvars;i_++)
         ar.m_c[i].Set(i_,sigma2*ar.m_c[i][i_]);
     }
  }
//+------------------------------------------------------------------+
//| Unpacks coefficients of linear model.                            |
//| INPUT PARAMETERS:                                                |
//|     LM          -   linear model in ALGLIB format                |
//| OUTPUT PARAMETERS:                                               |
//|     V           -   coefficients,array[0..NVars]                 |
//|                     constant term (intercept) is stored in the   |
//|                     V[NVars].                                    |
//|     NVars       -   number of independent variables (one less    |
//|                     than number of coefficients)                 |
//+------------------------------------------------------------------+
static void CLinReg::LRUnpack(CLinearModel &lm,double &v[],int &nvars)
  {
//--- create variables
   int offs=0;
   int i_=0;
   int i1_=0;
//--- initialization
   nvars=0;
//--- check
   if(!CAp::Assert((int)MathRound(lm.m_w[1])==m_lrvnum,__FUNCTION__+": Incorrect LINREG version!"))
      return;
//--- change values
   nvars=(int)MathRound(lm.m_w[2]);
   offs=(int)MathRound(lm.m_w[3]);
//--- allocation
   ArrayResizeAL(v,nvars+1);
//--- calculation
   i1_=offs;
   for(i_=0;i_<=nvars;i_++)
      v[i_]=lm.m_w[i_+i1_];
  }
//+------------------------------------------------------------------+
//| "Packs" coefficients and creates linear model in ALGLIB format   |
//| (LRUnpack reversed).                                             |
//| INPUT PARAMETERS:                                                |
//|     V           -   coefficients, array[0..NVars]                |
//|     NVars       -   number of independent variables              |
//| OUTPUT PAREMETERS:                                               |
//|     LM          -   linear model.                                |
//+------------------------------------------------------------------+
static void CLinReg::LRPack(double &v[],const int nvars,CLinearModel &lm)
  {
//--- create variables
   int offs=0;
   int i_=0;
   int i1_=0;
//--- allocation
   ArrayResizeAL(lm.m_w,5+nvars);
//--- change values
   offs=4;
   lm.m_w[0]=4+nvars+1;
   lm.m_w[1]=m_lrvnum;
   lm.m_w[2]=nvars;
   lm.m_w[3]=offs;
//--- calculation
   i1_=-offs;
   for(i_=offs;i_<=offs+nvars;i_++)
      lm.m_w[i_]=v[i_+i1_];
  }
//+------------------------------------------------------------------+
//| Procesing                                                        |
//| INPUT PARAMETERS:                                                |
//|     LM      -   linear model                                     |
//|     X       -   input vector, array[0..NVars-1].                 |
//| Result:                                                          |
//|     value of linear model regression estimate                    |
//+------------------------------------------------------------------+
static double CLinReg::LRProcess(CLinearModel &lm,double &x[])
  {
//--- create variables
   double v=0;
   int    offs=0;
   int    nvars=0;
   int    i_=0;
   int    i1_=0;
//--- check
   if(!CAp::Assert((int)MathRound(lm.m_w[1])==m_lrvnum,__FUNCTION__+": Incorrect LINREG version!"))
      return(EMPTY_VALUE);
//--- change values
   nvars=(int)MathRound(lm.m_w[2]);
   offs=(int)MathRound(lm.m_w[3]);
   i1_=offs;
   v=0.0;
//--- calculation
   for(i_=0;i_<=nvars-1;i_++)
      v+=x[i_]*lm.m_w[i_+i1_];
//--- return result
   return(v+lm.m_w[offs+nvars]);
  }
//+------------------------------------------------------------------+
//| RMS error on the test set                                        |
//| INPUT PARAMETERS:                                                |
//|     LM      -   linear model                                     |
//|     XY      -   test set                                         |
//|     NPoints -   test set size                                    |
//| RESULT:                                                          |
//|     root mean square error.                                      |
//+------------------------------------------------------------------+
static double CLinReg::LRRMSError(CLinearModel &lm,CMatrixDouble &xy,
                                  const int npoints)
  {
//--- create variables
   double result=0;
   int    i=0;
   double v=0;
   int    offs=0;
   int    nvars=0;
   int    i_=0;
   int    i1_=0;
//--- check
   if(!CAp::Assert((int)MathRound(lm.m_w[1])==m_lrvnum,__FUNCTION__+": Incorrect LINREG version!"))
      return(EMPTY_VALUE);
//--- change values
   nvars=(int)MathRound(lm.m_w[2]);
   offs=(int)MathRound(lm.m_w[3]);
   result=0;
//--- calculation
   for(i=0;i<=npoints-1;i++)
     {
      i1_=offs;
      v=0.0;
      for(i_=0;i_<=nvars-1;i_++)
         v+=xy[i][i_]*lm.m_w[i_+i1_];
      v=v+lm.m_w[offs+nvars];
      result=result+CMath::Sqr(v-xy[i][nvars]);
     }
//--- return result
   return(MathSqrt(result/npoints));
  }
//+------------------------------------------------------------------+
//| Average error on the test set                                    |
//| INPUT PARAMETERS:                                                |
//|     LM      -   linear model                                     |
//|     XY      -   test set                                         |
//|     NPoints -   test set size                                    |
//| RESULT:                                                          |
//|     average error.                                               |
//+------------------------------------------------------------------+
static double CLinReg::LRAvgError(CLinearModel &lm,CMatrixDouble &xy,
                                  const int npoints)
  {
//--- create variables
   double result=0;
   int    i=0;
   double v=0;
   int    offs=0;
   int    nvars=0;
   int    i_=0;
   int    i1_=0;
//--- check
   if(!CAp::Assert((int)MathRound(lm.m_w[1])==m_lrvnum,__FUNCTION__+": Incorrect LINREG version!"))
      return(EMPTY_VALUE);
//--- initialization
   nvars=(int)MathRound(lm.m_w[2]);
   offs=(int)MathRound(lm.m_w[3]);
   result=0;
//--- calculation
   for(i=0;i<=npoints-1;i++)
     {
      i1_=offs;
      v=0.0;
      for(i_=0;i_<=nvars-1;i_++)
         v+=xy[i][i_]*lm.m_w[i_+i1_];
      v=v+lm.m_w[offs+nvars];
      result=result+MathAbs(v-xy[i][nvars]);
     }
//--- return result
   return(result/npoints);
  }
//+------------------------------------------------------------------+
//| RMS error on the test set                                        |
//| INPUT PARAMETERS:                                                |
//|     LM      -   linear model                                     |
//|     XY      -   test set                                         |
//|     NPoints -   test set size                                    |
//| RESULT:                                                          |
//|     average relative error.                                      |
//+------------------------------------------------------------------+
static double CLinReg::LRAvgRelError(CLinearModel &lm,CMatrixDouble &xy,
                                     const int npoints)
  {
//--- create variables
   double result=0;
   int    i=0;
   int    k=0;
   double v=0;
   int    offs=0;
   int    nvars=0;
   int    i_=0;
   int    i1_=0;
//--- check
   if(!CAp::Assert((int)MathRound(lm.m_w[1])==m_lrvnum,__FUNCTION__+": Incorrect LINREG version!"))
      return(EMPTY_VALUE);
//--- initialization
   nvars=(int)MathRound(lm.m_w[2]);
   offs=(int)MathRound(lm.m_w[3]);
   result=0;
   k=0;
//--- calculation
   for(i=0;i<=npoints-1;i++)
     {
      //--- check
      if(xy[i][nvars]!=0.0)
        {
         i1_=offs;
         v=0.0;
         for(i_=0;i_<=nvars-1;i_++)
            v+=xy[i][i_]*lm.m_w[i_+i1_];
         v=v+lm.m_w[offs+nvars];
         //--- get result
         result=result+MathAbs((v-xy[i][nvars])/xy[i][nvars]);
         k=k+1;
        }
     }
//--- check
   if(k!=0)
      result=result/k;
//--- return result
   return(result);
  }
//+------------------------------------------------------------------+
//| Copying of LinearModel strucure                                  |
//| INPUT PARAMETERS:                                                |
//|     LM1 -   original                                             |
//| OUTPUT PARAMETERS:                                               |
//|     LM2 -   copy                                                 |
//+------------------------------------------------------------------+
static void CLinReg::LRCopy(CLinearModel &lm1,CLinearModel &lm2)
  {
//--- create variables
   int k=0;
   int i_=0;
//--- initialization
   k=(int)MathRound(lm1.m_w[0]);
//--- allocation
   ArrayResizeAL(lm2.m_w,k);
//--- copy
   for(i_=0;i_<=k-1;i_++)
      lm2.m_w[i_]=lm1.m_w[i_];
  }
//+------------------------------------------------------------------+
//| Class method                                                     |
//+------------------------------------------------------------------+
static void CLinReg::LRLines(CMatrixDouble &xy,double &s[],const int n,
                             int &info,double &a,double &b,double &vara,
                             double &varb,double &covab,double &corrab,
                             double &p)
  {
//--- create variables
   int    i=0;
   double ss=0;
   double sx=0;
   double sxx=0;
   double sy=0;
   double stt=0;
   double e1=0;
   double e2=0;
   double t=0;
   double chi2=0;
//--- initialization
   info=0;
   a=0;
   b=0;
   vara=0;
   varb=0;
   covab=0;
   corrab=0;
   p=0;
//--- check
   if(n<2)
     {
      info=-1;
      return;
     }
   for(i=0;i<=n-1;i++)
     {
      //--- check
      if((double)(s[i])<=0.0)
        {
         info=-2;
         return;
        }
     }
//--- change value
   info=1;
//--- Calculate S,SX,SY,SXX
   ss=0;
   sx=0;
   sy=0;
   sxx=0;
//--- calculation
   for(i=0;i<=n-1;i++)
     {
      t=CMath::Sqr(s[i]);
      ss=ss+1/t;
      sx=sx+xy[i][0]/t;
      sy=sy+xy[i][1]/t;
      sxx=sxx+CMath::Sqr(xy[i][0])/t;
     }
//--- Test for condition number
   t=MathSqrt(4*CMath::Sqr(sx)+CMath::Sqr(ss-sxx));
   e1=0.5*(ss+sxx+t);
   e2=0.5*(ss+sxx-t);
//--- check
   if(MathMin(e1,e2)<=1000*CMath::m_machineepsilon*MathMax(e1,e2))
     {
      info=-3;
      return;
     }
//--- Calculate A,B
   a=0;
   b=0;
   stt=0;
//--- calculation
   for(i=0;i<=n-1;i++)
     {
      t=(xy[i][0]-sx/ss)/s[i];
      b=b+t*xy[i][1]/s[i];
      stt=stt+CMath::Sqr(t);
     }
   b=b/stt;
   a=(sy-sx*b)/ss;
//--- Calculate goodness-of-fit
   if(n>2)
     {
      chi2=0;
      for(i=0;i<=n-1;i++)
         chi2=chi2+CMath::Sqr((xy[i][1]-a-b*xy[i][0])/s[i]);
      //--- function call
      p=CIncGammaF::IncompleteGammaC((double)(n-2)/(double)2,chi2/2);
     }
   else
      p=1;
//--- Calculate other parameters
   vara=(1+CMath::Sqr(sx)/(ss*stt))/ss;
   varb=1/stt;
   covab=-(sx/(ss*stt));
   corrab=covab/MathSqrt(vara*varb);
  }
//+------------------------------------------------------------------+
//| Class method                                                     |
//+------------------------------------------------------------------+
static void CLinReg::LRLine(CMatrixDouble &xy,const int n,int &info,
                            double &a,double &b)
  {
//--- create variables
   int    i=0;
   double vara=0;
   double varb=0;
   double covab=0;
   double corrab=0;
   double p=0;
//--- create array
   double s[];
//--- initialization
   info=0;
   a=0;
   b=0;
//--- check
   if(n<2)
     {
      info=-1;
      return;
     }
//--- allocation
   ArrayResizeAL(s,n);
   for(i=0;i<=n-1;i++)
      s[i]=1;
//--- function call
   LRLines(xy,s,n,info,a,b,vara,varb,covab,corrab,p);
  }
//+------------------------------------------------------------------+
//| Internal linear regression subroutine                            |
//+------------------------------------------------------------------+
static void CLinReg::LRInternal(CMatrixDouble &xy,double &s[],const int npoints,
                                const int nvars,int &info,CLinearModel &lm,
                                CLRReport &ar)
  {
//--- create variables
   int    i=0;
   int    j=0;
   int    k=0;
   int    ncv=0;
   int    na=0;
   int    nacv=0;
   double r=0;
   double p=0;
   double epstol=0;
   int    offs=0;
   int    i_=0;
   int    i1_=0;
//--- creating arrays
   double b[];
   double sv[];
   double t[];
   double svi[];
   double work[];
//--- create matrix
   CMatrixDouble a;
   CMatrixDouble u;
   CMatrixDouble vt;
   CMatrixDouble vm;
   CMatrixDouble xym;
//--- create objects of classes
   CLRReport    ar2;
   CLinearModel tlm;
//--- initialization
   info=0;
   epstol=1000;
//--- Check for errors in data
   if(npoints<nvars || nvars<1)
     {
      info=-1;
      return;
     }
   for(i=0;i<=npoints-1;i++)
     {
      //--- check
      if(s[i]<=0.0)
        {
         info=-2;
         return;
        }
     }
//--- change value
   info=1;
//--- Create design matrix
   a.Resize(npoints,nvars);
   ArrayResizeAL(b,npoints);
//--- calculation
   for(i=0;i<=npoints-1;i++)
     {
      r=1/s[i];
      for(i_=0;i_<=nvars-1;i_++)
         a[i].Set(i_,r*xy[i][i_]);
      b[i]=xy[i][nvars]/s[i];
     }
//--- Allocate W:
//--- W[0]     array size
//--- W[1]     version number,0
//--- W[2]     NVars (minus 1,to be compatible with external representation)
//--- W[3]     coefficients offset
   ArrayResizeAL(lm.m_w,4+nvars);
   offs=4;
   lm.m_w[0]=4+nvars;
   lm.m_w[1]=m_lrvnum;
   lm.m_w[2]=nvars-1;
   lm.m_w[3]=offs;
//--- Solve problem using SVD:
//--- 0. check for degeneracy (different types)
//--- 1. A = U*diag(sv)*V'
//--- 2. T = b'*U
//--- 3. w = SUM((T[i]/sv[i])*V[..,i])
//--- 4. cov(wi,wj) = SUM(Vji*Vjk/sv[i]^2,K=1..M)
//--- see $15.4 of "Numerical Recipes in C" for more information
   ArrayResizeAL(t,nvars);
   ArrayResizeAL(svi,nvars);
   ar.m_c.Resize(nvars,nvars);
   vm.Resize(nvars,nvars);
//--- check
   if(!CSingValueDecompose::RMatrixSVD(a,npoints,nvars,1,1,2,sv,u,vt))
     {
      info=-4;
      return;
     }
//--- check
   if(sv[0]<=0.0)
     {
      //--- Degenerate case: zero design matrix.
      for(i=offs;i<=offs+nvars-1;i++)
         lm.m_w[i]=0;
      //--- change values
      ar.m_rmserror=LRRMSError(lm,xy,npoints);
      ar.m_avgerror=LRAvgError(lm,xy,npoints);
      ar.m_avgrelerror=LRAvgRelError(lm,xy,npoints);
      ar.m_cvrmserror=ar.m_rmserror;
      ar.m_cvavgerror=ar.m_avgerror;
      ar.m_cvavgrelerror=ar.m_avgrelerror;
      ar.m_ncvdefects=0;
      //--- allocation
      ArrayResizeAL(ar.m_cvdefects,nvars);
      ar.m_c.Resize(nvars,nvars);
      for(i=0;i<=nvars-1;i++)
        {
         for(j=0;j<=nvars-1;j++)
            ar.m_c[i].Set(j,0);
        }
      //--- exit the function
      return;
     }
//--- check
   if(sv[nvars-1]<=epstol*CMath::m_machineepsilon*sv[0])
     {
      //--- Degenerate case,non-zero design matrix.
      //--- We can leave it and solve task in SVD least squares fashion.
      //--- Solution and covariance matrix will be obtained correctly,
      //--- but CV error estimates - will not. It is better to reduce
      //--- it to non-degenerate task and to obtain correct CV estimates.
      for(k=nvars;k>=1;k--)
        {
         //--- check
         if(sv[k-1]>epstol*CMath::m_machineepsilon*sv[0])
           {
            //--- Reduce
            xym.Resize(npoints,k+1);
            for(i=0;i<=npoints-1;i++)
              {
               for(j=0;j<=k-1;j++)
                 {
                  //--- calculation
                  r=0.0;
                  for(i_=0;i_<=nvars-1;i_++)
                     r+=xy[i][i_]*vt[j][i_];
                  xym[i].Set(j,r);
                 }
               xym[i].Set(k,xy[i][nvars]);
              }
            //--- Solve
            LRInternal(xym,s,npoints,k,info,tlm,ar2);
            //--- check
            if(info!=1)
               return;
            //--- Convert back to un-reduced format
            for(j=0;j<=nvars-1;j++)
               lm.m_w[offs+j]=0;
            for(j=0;j<=k-1;j++)
              {
               r=tlm.m_w[offs+j];
               i1_=-offs;
               for(i_=offs;i_<=offs+nvars-1;i_++)
                  lm.m_w[i_]=lm.m_w[i_]+r*vt[j][i_+i1_];
              }
            //--- change values
            ar.m_rmserror=ar2.m_rmserror;
            ar.m_avgerror=ar2.m_avgerror;
            ar.m_avgrelerror=ar2.m_avgrelerror;
            ar.m_cvrmserror=ar2.m_cvrmserror;
            ar.m_cvavgerror=ar2.m_cvavgerror;
            ar.m_cvavgrelerror=ar2.m_cvavgrelerror;
            ar.m_ncvdefects=ar2.m_ncvdefects;
            //--- allocation
            ArrayResizeAL(ar.m_cvdefects,nvars);
            for(j=0;j<=ar.m_ncvdefects-1;j++)
               ar.m_cvdefects[j]=ar2.m_cvdefects[j];
            //--- allocation
            ar.m_c.Resize(nvars,nvars);
            ArrayResizeAL(work,nvars+1);
            //--- function calls
            CBlas::MatrixMatrixMultiply(ar2.m_c,0,k-1,0,k-1,false,vt,0,k-1,0,nvars-1,false,1.0,vm,0,k-1,0,nvars-1,0.0,work);
            CBlas::MatrixMatrixMultiply(vt,0,k-1,0,nvars-1,true,vm,0,k-1,0,nvars-1,false,1.0,ar.m_c,0,nvars-1,0,nvars-1,0.0,work);
            //--- exit the function
            return;
           }
        }
      //--- change value
      info=-255;
      //--- exit the function
      return;
     }
//--- change values
   for(i=0;i<=nvars-1;i++)
     {
      //--- check
      if(sv[i]>epstol*CMath::m_machineepsilon*sv[0])
         svi[i]=1/sv[i];
      else
         svi[i]=0;
     }
//--- change values
   for(i=0;i<=nvars-1;i++)
      t[i]=0;
//--- change values
   for(i=0;i<=npoints-1;i++)
     {
      r=b[i];
      for(i_=0;i_<=nvars-1;i_++)
         t[i_]=t[i_]+r*u[i][i_];
     }
   for(i=0;i<=nvars-1;i++)
      lm.m_w[offs+i]=0;
//--- calculation
   for(i=0;i<=nvars-1;i++)
     {
      r=t[i]*svi[i];
      i1_=-offs;
      for(i_=offs;i_<=offs+nvars-1;i_++)
         lm.m_w[i_]=lm.m_w[i_]+r*vt[i][i_+i1_];
     }
//--- calculation
   for(j=0;j<=nvars-1;j++)
     {
      r=svi[j];
      for(i_=0;i_<=nvars-1;i_++)
         vm[i_].Set(j,r*vt[j][i_]);
     }
//--- calculation
   for(i=0;i<=nvars-1;i++)
     {
      for(j=i;j<=nvars-1;j++)
        {
         r=0.0;
         for(i_=0;i_<=nvars-1;i_++)
            r+=vm[i][i_]*vm[j][i_];
         ar.m_c[i].Set(j,r);
         ar.m_c[j].Set(i,r);
        }
     }
//--- Leave-1-out cross-validation error.
//--- NOTATIONS:
//--- A            design matrix
//--- A*x = b      original linear least squares task
//--- U*S*V'       SVD of A
//--- ai           i-th row of the A
//--- bi           i-th element of the b
//--- xf           solution of the original LLS task
//--- Cross-validation error of i-th element from a sample is
//--- calculated using following formula:
//---     ERRi = ai*xf - (ai*xf-bi*(ui*ui'))/(1-ui*ui')     (1)
//--- This formula can be derived from normal equations of the
//--- original task
//---     (A'*A)x = A'*b                                    (2)
//--- by applying modification (zeroing out i-th row of A) to (2):
//---     (A-ai)'*(A-ai) = (A-ai)'*b
//--- and using Sherman-Morrison formula for updating matrix inverse
//--- NOTE 1: b is not zeroed out since it is much simpler and
//--- does not influence final result.
//--- NOTE 2: some design matrices A have such ui that 1-ui*ui'=0.
//--- Formula (1) can't be applied for such cases and they are skipped
//--- from CV calculation (which distorts resulting CV estimate).
//--- But from the properties of U we can conclude that there can
//--- be no more than NVars such vectors. Usually
//--- NVars << NPoints, so in a normal case it only slightly
//--- influences result.
   ncv=0;
   na=0;
   nacv=0;
   ar.m_rmserror=0;
   ar.m_avgerror=0;
   ar.m_avgrelerror=0;
   ar.m_cvrmserror=0;
   ar.m_cvavgerror=0;
   ar.m_cvavgrelerror=0;
   ar.m_ncvdefects=0;
//--- allocation
   ArrayResizeAL(ar.m_cvdefects,nvars);
   for(i=0;i<=npoints-1;i++)
     {
      //--- Error on a training set
      i1_=offs;
      r=0.0;
      for(i_=0;i_<=nvars-1;i_++)
         r+=xy[i][i_]*lm.m_w[i_+i1_];
      //--- change values
      ar.m_rmserror=ar.m_rmserror+CMath::Sqr(r-xy[i][nvars]);
      ar.m_avgerror=ar.m_avgerror+MathAbs(r-xy[i][nvars]);
      //--- check
      if(xy[i][nvars]!=0.0)
        {
         ar.m_avgrelerror=ar.m_avgrelerror+MathAbs((r-xy[i][nvars])/xy[i][nvars]);
         na=na+1;
        }
      //--- Error using fast leave-one-out cross-validation
      p=0.0;
      for(i_=0;i_<=nvars-1;i_++)
         p+=u[i][i_]*u[i][i_];
      //--- check
      if(p>1-epstol*CMath::m_machineepsilon)
        {
         ar.m_cvdefects[ar.m_ncvdefects]=i;
         ar.m_ncvdefects=ar.m_ncvdefects+1;
         continue;
        }
      //--- change values
      r=s[i]*(r/s[i]-b[i]*p)/(1-p);
      ar.m_cvrmserror=ar.m_cvrmserror+CMath::Sqr(r-xy[i][nvars]);
      ar.m_cvavgerror=ar.m_cvavgerror+MathAbs(r-xy[i][nvars]);
      //--- check
      if(xy[i][nvars]!=0.0)
        {
         ar.m_cvavgrelerror=ar.m_cvavgrelerror+MathAbs((r-xy[i][nvars])/xy[i][nvars]);
         nacv=nacv+1;
        }
      ncv=ncv+1;
     }
//--- check
   if(ncv==0)
     {
      //--- Something strange: ALL ui are degenerate.
      //--- Unexpected...
      info=-255;
      //--- exit the function
      return;
     }
//--- change values
   ar.m_rmserror=MathSqrt(ar.m_rmserror/npoints);
   ar.m_avgerror=ar.m_avgerror/npoints;
//--- check
   if(na!=0)
      ar.m_avgrelerror=ar.m_avgrelerror/na;
   ar.m_cvrmserror=MathSqrt(ar.m_cvrmserror/ncv);
   ar.m_cvavgerror=ar.m_cvavgerror/ncv;
//--- check
   if(nacv!=0)
      ar.m_cvavgrelerror=ar.m_cvavgrelerror/nacv;
  }
//+------------------------------------------------------------------+
//| Auxiliary class for CMLPBase                                     |
//+------------------------------------------------------------------+
class CMultilayerPerceptron
  {
public:
   //--- variables
   int               m_hlnetworktype;
   int               m_hlnormtype;
   //--- arrays
   int               m_hllayersizes[];
   int               m_hlconnections[];
   int               m_hlneurons[];
   int               m_structinfo[];
   double            m_weights[];
   double            m_columnmeans[];
   double            m_columnsigmas[];
   double            m_neurons[];
   double            m_dfdnet[];
   double            m_derror[];
   double            m_x[];
   double            m_y[];
   double            m_nwbuf[];
   int               m_integerbuf[];
   //--- matrix
   CMatrixDouble     m_chunks;
   //--- constructor, destructor
                     CMultilayerPerceptron(void);
                    ~CMultilayerPerceptron(void);
   //--- copy
   void              Copy(CMultilayerPerceptron &obj);
  };
//+------------------------------------------------------------------+
//| Constructor without parameters                                   |
//+------------------------------------------------------------------+
CMultilayerPerceptron::CMultilayerPerceptron(void)
  {

  }
//+------------------------------------------------------------------+
//| Destructor                                                       |
//+------------------------------------------------------------------+
CMultilayerPerceptron::~CMultilayerPerceptron(void)
  {

  }
//+------------------------------------------------------------------+
//| Copy                                                             |
//+------------------------------------------------------------------+
void CMultilayerPerceptron::Copy(CMultilayerPerceptron &obj)
  {
//--- copy variables
   m_hlnetworktype=obj.m_hlnetworktype;
   m_hlnormtype=obj.m_hlnormtype;
//--- copy arrays
   ArrayCopy(m_hllayersizes,obj.m_hllayersizes);
   ArrayCopy(m_hlconnections,obj.m_hlconnections);
   ArrayCopy(m_hlneurons,obj.m_hlneurons);
   ArrayCopy(m_structinfo,obj.m_structinfo);
   ArrayCopy(m_weights,obj.m_weights);
   ArrayCopy(m_columnmeans,obj.m_columnmeans);
   ArrayCopy(m_columnsigmas,obj.m_columnsigmas);
   ArrayCopy(m_neurons,obj.m_neurons);
   ArrayCopy(m_dfdnet,obj.m_dfdnet);
   ArrayCopy(m_derror,obj.m_derror);
   ArrayCopy(m_x,obj.m_x);
   ArrayCopy(m_y,obj.m_y);
   ArrayCopy(m_nwbuf,obj.m_nwbuf);
   ArrayCopy(m_integerbuf,obj.m_integerbuf);
//--- copy matrix
   m_chunks=obj.m_chunks;
  }
//+------------------------------------------------------------------+
//| This class is a shell for class CMultilayerPerceptron            |
//+------------------------------------------------------------------+
class CMultilayerPerceptronShell
  {
private:
   CMultilayerPerceptron m_innerobj;
public:
   //--- constructors, destructor
                     CMultilayerPerceptronShell(void);
                     CMultilayerPerceptronShell(CMultilayerPerceptron &obj);
                    ~CMultilayerPerceptronShell(void);
   //--- method
   CMultilayerPerceptron *GetInnerObj(void);
  };
//+------------------------------------------------------------------+
//| Constructor without parameters                                   |
//+------------------------------------------------------------------+
CMultilayerPerceptronShell::CMultilayerPerceptronShell(void)
  {

  }
//+------------------------------------------------------------------+
//| Copy constructor                                                 |
//+------------------------------------------------------------------+
CMultilayerPerceptronShell::CMultilayerPerceptronShell(CMultilayerPerceptron &obj)
  {
//--- copy
   m_innerobj.Copy(obj);
  }
//+------------------------------------------------------------------+
//| Destructor                                                       |
//+------------------------------------------------------------------+
CMultilayerPerceptronShell::~CMultilayerPerceptronShell(void)
  {

  }
//+------------------------------------------------------------------+
//| Return object of class                                           |
//+------------------------------------------------------------------+
CMultilayerPerceptron *CMultilayerPerceptronShell::GetInnerObj(void)
  {
//--- return result
   return(GetPointer(m_innerobj));
  }
//+------------------------------------------------------------------+
//| Multilayer perceptron class                                      |
//+------------------------------------------------------------------+
class CMLPBase
  {
private:
   //--- private methods
   static void       AddInputLayer(const int ncount,int &lsizes[],int &ltypes[],int &lconnfirst[],int &lconnlast[],int &lastproc);
   static void       AddBiasedSummatorLayer(const int ncount,int &lsizes[],int &ltypes[],int &lconnfirst[],int &lconnlast[],int &lastproc);
   static void       AddActivationLayer(const int functype,int &lsizes[],int &ltypes[],int &lconnfirst[],int &lconnlast[],int &lastproc);
   static void       AddZeroLayer(int &lsizes[],int &ltypes[],int &lconnfirst[],int &lconnlast[],int &lastproc);
   static void       HLAddInputLayer(CMultilayerPerceptron &network,int &connidx,int &neuroidx,int &structinfoidx,int nin);
   static void       HLAddOutputLayer(CMultilayerPerceptron &network,int &connidx,int &neuroidx,int &structinfoidx,int &weightsidx,const int k,const int nprev,const int nout,const bool iscls,const bool islinearout);
   static void       HLAddHiddenLayer(CMultilayerPerceptron &network,int &connidx,int &neuroidx,int &structinfoidx,int &weightsidx,const int k,const int nprev,const int ncur);
   static void       FillHighLevelInformation(CMultilayerPerceptron &network,const int nin,const int nhid1,const int nhid2,const int nout,const bool iscls,const bool islinearout);
   static void       MLPCreate(const int nin,const int nout,int &lsizes[],int &ltypes[],int &lconnfirst[],int &lconnlast[],const int layerscount,const bool isclsnet,CMultilayerPerceptron &network);
   static void       MLPHessianBatchInternal(CMultilayerPerceptron &network,CMatrixDouble &xy,const int ssize,const bool naturalerr,double &e,double &grad[],CMatrixDouble &h);
   static void       MLPInternalCalculateGradient(CMultilayerPerceptron &network,double &neurons[],double &weights[],double &derror[],double &grad[],const bool naturalerrorfunc);
   static void       MLPChunkedGradient(CMultilayerPerceptron &network,CMatrixDouble &xy,const int cstart,const int csize,double &e,double &grad[],const bool naturalerrorfunc);
   static double     SafeCrossEntropy(const double t,const double z);
public:
   //--- variables
   static const int  m_mlpvnum;
   static const int  m_mlpfirstversion;
   static const int  m_nfieldwidth;
   static const int  m_hlconm_nfieldwidth;
   static const int  m_hlm_nfieldwidth;
   static const int  m_chunksize;
   //--- constructor, destructor
                     CMLPBase(void);
                    ~CMLPBase(void);
   //--- public methods
   static void       MLPCreate0(const int nin,const int nout,CMultilayerPerceptron &network);
   static void       MLPCreate1(const int nin,const int nhid,const int nout,CMultilayerPerceptron &network);
   static void       MLPCreate2(const int nin,const int nhid1,const int nhid2,const int nout,CMultilayerPerceptron &network);
   static void       MLPCreateB0(const int nin,const int nout,const double b,double d,CMultilayerPerceptron &network);
   static void       MLPCreateB1(const int nin,const int nhid,const int nout,const double b,double d,CMultilayerPerceptron &network);
   static void       MLPCreateB2(const int nin,const int nhid1,const int nhid2,const int nout,const double b,double d,CMultilayerPerceptron &network);
   static void       MLPCreateR0(const int nin,const int nout,const double a,const double b,CMultilayerPerceptron &network);
   static void       MLPCreateR1(const int nin,const int nhid,const int nout,const double a,const double b,CMultilayerPerceptron &network);
   static void       MLPCreateR2(const int nin,const int nhid1,const int nhid2,const int nout,const double a,const double b,CMultilayerPerceptron &network);
   static void       MLPCreateC0(const int nin,const int nout,CMultilayerPerceptron &network);
   static void       MLPCreateC1(const int nin,const int nhid,const int nout,CMultilayerPerceptron &network);
   static void       MLPCreateC2(const int nin,const int nhid1,const int nhid2,const int nout,CMultilayerPerceptron &network);
   static void       MLPCopy(CMultilayerPerceptron &network1,CMultilayerPerceptron &network2);
   static void       MLPSerializeOld(CMultilayerPerceptron &network,double &ra[],int &rlen);
   static void       MLPUnserializeOld(double &ra[],CMultilayerPerceptron &network);
   static void       MLPRandomize(CMultilayerPerceptron &network);
   static void       MLPRandomizeFull(CMultilayerPerceptron &network);
   static void       MLPInitPreprocessor(CMultilayerPerceptron &network,CMatrixDouble &xy,const int ssize);
   static void       MLPProperties(CMultilayerPerceptron &network,int &nin,int &nout,int &wcount);
   static bool       MLPIsSoftMax(CMultilayerPerceptron &network);
   static int        MLPGetLayersCount(CMultilayerPerceptron &network);
   static int        MLPGetLayerSize(CMultilayerPerceptron &network,const int k);
   static void       MLPGetInputScaling(CMultilayerPerceptron &network,const int i,double &mean,double &sigma);
   static void       MLPGetOutputScaling(CMultilayerPerceptron &network,const int i,double &mean,double &sigma);
   static void       MLPGetNeuronInfo(CMultilayerPerceptron &network,const int k,const int i,int &fkind,double &threshold);
   static double     MLPGetWeight(CMultilayerPerceptron &network,const int k0,const int i0,const int k1,const int i1);
   static void       MLPSetInputScaling(CMultilayerPerceptron &network,const int i,const double mean,double sigma);
   static void       MLPSetOutputScaling(CMultilayerPerceptron &network,const int i,const double mean,double sigma);
   static void       MLPSetNeuronInfo(CMultilayerPerceptron &network,const int k,const int i,const int fkind,const double threshold);
   static void       MLPSetWeight(CMultilayerPerceptron &network,const int k0,const int i0,const int k1,const int i1,const double w);
   static void       MLPActivationFunction(double net,const int k,double &f,double &df,double &d2f);
   static void       MLPProcess(CMultilayerPerceptron &network,double &x[],double &y[]);
   static void       MLPProcessI(CMultilayerPerceptron &network,double &x[],double &y[]);
   static double     MLPError(CMultilayerPerceptron &network,CMatrixDouble &xy,const int ssize);
   static double     MLPErrorN(CMultilayerPerceptron &network,CMatrixDouble &xy,const int ssize);
   static int        MLPClsError(CMultilayerPerceptron &network,CMatrixDouble &xy,const int ssize);
   static double     MLPRelClsError(CMultilayerPerceptron &network,CMatrixDouble &xy,const int npoints);
   static double     MLPAvgCE(CMultilayerPerceptron &network,CMatrixDouble &xy,const int npoints);
   static double     MLPRMSError(CMultilayerPerceptron &network,CMatrixDouble &xy,const int npoints);
   static double     MLPAvgError(CMultilayerPerceptron &network,CMatrixDouble &xy,const int npoints);
   static double     MLPAvgRelError(CMultilayerPerceptron &network,CMatrixDouble &xy,const int npoints);
   static void       MLPGrad(CMultilayerPerceptron &network,double &x[],double &desiredy[],double &e,double &grad[]);
   static void       MLPGradN(CMultilayerPerceptron &network,double &x[],double &desiredy[],double &e,double &grad[]);
   static void       MLPGradBatch(CMultilayerPerceptron &network,CMatrixDouble &xy,const int ssize,double &e,double &grad[]);
   static void       MLPGradNBatch(CMultilayerPerceptron &network,CMatrixDouble &xy,const int ssize,double &e,double &grad[]);
   static void       MLPHessianNBatch(CMultilayerPerceptron &network,CMatrixDouble &xy,const int ssize,double &e,double &grad[],CMatrixDouble &h);
   static void       MLPHessianBatch(CMultilayerPerceptron &network,CMatrixDouble &xy,const int ssize,double &e,double &grad[],CMatrixDouble &h);
   static void       MLPInternalProcessVector(int &structinfo[],double &weights[],double &columnmeans[],double &columnsigmas[],double &neurons[],double &dfdnet[],double &x[],double &y[]);
   static void       MLPAlloc(CSerializer &s,CMultilayerPerceptron &network);
   static void       MLPSerialize(CSerializer &s,CMultilayerPerceptron &network);
   static void       MLPUnserialize(CSerializer &s,CMultilayerPerceptron &network);
  };
//+------------------------------------------------------------------+
//| Initialize constants                                             |
//+------------------------------------------------------------------+
const int CMLPBase::m_mlpvnum=7;
const int CMLPBase::m_mlpfirstversion=0;
const int CMLPBase::m_nfieldwidth=4;
const int CMLPBase::m_hlconm_nfieldwidth=5;
const int CMLPBase::m_hlm_nfieldwidth=4;
const int CMLPBase::m_chunksize=32;
//+------------------------------------------------------------------+
//| Constructor without parameters                                   |
//+------------------------------------------------------------------+
CMLPBase::CMLPBase(void)
  {

  }
//+------------------------------------------------------------------+
//| Destructor                                                       |
//+------------------------------------------------------------------+
CMLPBase::~CMLPBase(void)
  {

  }
//+------------------------------------------------------------------+
//| Creates  neural  network  with  NIn  inputs,  NOut outputs,      |
//| without hidden layers, with linear output layer. Network weights | 
//| are filled with small random values.                             |
//+------------------------------------------------------------------+
static void CMLPBase::MLPCreate0(const int nin,const int nout,
                                 CMultilayerPerceptron &network)
  {
//--- create variables
   int layerscount=0;
   int lastproc=0;
//--- creating arrays
   int lsizes[];
   int ltypes[];
   int lconnfirst[];
   int lconnlast[];
//--- initialization
   layerscount=4;
//--- Allocate arrays
   ArrayResizeAL(lsizes,layerscount);
   ArrayResizeAL(ltypes,layerscount);
   ArrayResizeAL(lconnfirst,layerscount);
   ArrayResizeAL(lconnlast,layerscount);
//--- Layers
   AddInputLayer(nin,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddBiasedSummatorLayer(nout,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddActivationLayer(-5,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- Create
   MLPCreate(nin,nout,lsizes,ltypes,lconnfirst,lconnlast,layerscount,false,network);
//--- function call
   FillHighLevelInformation(network,nin,0,0,nout,false,true);
  }
//+------------------------------------------------------------------+
//| Same as MLPCreate0, but with one hidden layer (NHid neurons) with|
//| non-linear activation function. Output layer is linear.          |
//+------------------------------------------------------------------+
static void CMLPBase::MLPCreate1(const int nin,const int nhid,const int nout,
                                 CMultilayerPerceptron &network)
  {
//--- create variables
   int layerscount=0;
   int lastproc=0;
//--- creating arrays
   int lsizes[];
   int ltypes[];
   int lconnfirst[];
   int lconnlast[];
//--- create variables
   layerscount=7;
//--- Allocate arrays
   ArrayResizeAL(lsizes,layerscount);
   ArrayResizeAL(ltypes,layerscount);
   ArrayResizeAL(lconnfirst,layerscount);
   ArrayResizeAL(lconnlast,layerscount);
//--- Layers
   AddInputLayer(nin,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddBiasedSummatorLayer(nhid,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddActivationLayer(1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddBiasedSummatorLayer(nout,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddActivationLayer(-5,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- Create
   MLPCreate(nin,nout,lsizes,ltypes,lconnfirst,lconnlast,layerscount,false,network);
//--- function call
   FillHighLevelInformation(network,nin,nhid,0,nout,false,true);
  }
//+------------------------------------------------------------------+
//| Same as MLPCreate0,but with two hidden layers (NHid1 and NHid2   |
//| neurons) with non-linear activation function. Output layer is    |
//| linear.                                                          |
//|  $ALL                                                            |
//+------------------------------------------------------------------+
static void CMLPBase::MLPCreate2(const int nin,const int nhid1,const int nhid2,
                                 const int nout,CMultilayerPerceptron &network)
  {
//--- create variables
   int layerscount=0;
   int lastproc=0;
//--- creating arrays
   int lsizes[];
   int ltypes[];
   int lconnfirst[];
   int lconnlast[];
//--- initialization
   layerscount=10;
//--- Allocate arrays
   ArrayResizeAL(lsizes,layerscount);
   ArrayResizeAL(ltypes,layerscount);
   ArrayResizeAL(lconnfirst,layerscount);
   ArrayResizeAL(lconnlast,layerscount);
//--- Layers
   AddInputLayer(nin,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddBiasedSummatorLayer(nhid1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddActivationLayer(1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddBiasedSummatorLayer(nhid2,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddActivationLayer(1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddBiasedSummatorLayer(nout,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddActivationLayer(-5,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- Create
   MLPCreate(nin,nout,lsizes,ltypes,lconnfirst,lconnlast,layerscount,false,network);
//--- function call
   FillHighLevelInformation(network,nin,nhid1,nhid2,nout,false,true);
  }
//+------------------------------------------------------------------+
//| Creates neural network with NIn inputs, NOut outputs, without    |
//| hidden layers with non-linear output layer. Network weights are  | 
//| filled with small random values.                                 |
//| Activation function of the output layer takes values:            |
//|     (B, +INF), if D>=0                                           |
//| or                                                               |
//|     (-INF, B), if D<0.                                           |
//+------------------------------------------------------------------+
static void CMLPBase::MLPCreateB0(const int nin,const int nout,const double b,
                                  double d,CMultilayerPerceptron &network)
  {
//--- create variables
   int layerscount=0;
   int lastproc=0;
   int i=0;
//--- creating arrays
   int lsizes[];
   int ltypes[];
   int lconnfirst[];
   int lconnlast[];
//--- initialization
   layerscount=4;
//--- check
   if(d>=0.0)
      d=1;
   else
      d=-1;
//--- Allocate arrays
   ArrayResizeAL(lsizes,layerscount);
   ArrayResizeAL(ltypes,layerscount);
   ArrayResizeAL(lconnfirst,layerscount);
   ArrayResizeAL(lconnlast,layerscount);
//--- Layers
   AddInputLayer(nin,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddBiasedSummatorLayer(nout,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddActivationLayer(3,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- Create
   MLPCreate(nin,nout,lsizes,ltypes,lconnfirst,lconnlast,layerscount,false,network);
//--- function call
   FillHighLevelInformation(network,nin,0,0,nout,false,false);
//--- Turn on ouputs shift/scaling.
   for(i=nin;i<=nin+nout-1;i++)
     {
      network.m_columnmeans[i]=b;
      network.m_columnsigmas[i]=d;
     }
  }
//+------------------------------------------------------------------+
//| Same as MLPCreateB0 but with non-linear hidden layer.            |
//+------------------------------------------------------------------+
static void CMLPBase::MLPCreateB1(const int nin,const int nhid,const int nout,
                                  const double b,double d,CMultilayerPerceptron &network)
  {
//--- create variables
   int layerscount=0;
   int lastproc=0;
   int i=0;
//--- creating arrays
   int lsizes[];
   int ltypes[];
   int lconnfirst[];
   int lconnlast[];
   layerscount=7;
//--- check
   if(d>=0.0)
      d=1;
   else
      d=-1;
//--- Allocate arrays
   ArrayResizeAL(lsizes,layerscount);
   ArrayResizeAL(ltypes,layerscount);
   ArrayResizeAL(lconnfirst,layerscount);
   ArrayResizeAL(lconnlast,layerscount);
//--- Layers
   AddInputLayer(nin,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddBiasedSummatorLayer(nhid,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddActivationLayer(1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddBiasedSummatorLayer(nout,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddActivationLayer(3,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- Create
   MLPCreate(nin,nout,lsizes,ltypes,lconnfirst,lconnlast,layerscount,false,network);
//--- function call
   FillHighLevelInformation(network,nin,nhid,0,nout,false,false);
//--- Turn on ouputs shift/scaling.
   for(i=nin;i<=nin+nout-1;i++)
     {
      network.m_columnmeans[i]=b;
      network.m_columnsigmas[i]=d;
     }
  }
//+------------------------------------------------------------------+
//| Same as MLPCreateB0 but with two non-linear hidden layers.       |
//+------------------------------------------------------------------+
static void CMLPBase::MLPCreateB2(const int nin,const int nhid1,const int nhid2,
                                  const int nout,const double b,double d,
                                  CMultilayerPerceptron &network)
  {
//--- create variables
   int layerscount=0;
   int lastproc=0;
   int i=0;
//--- creating arrays
   int lsizes[];
   int ltypes[];
   int lconnfirst[];
   int lconnlast[];
//--- initialization
   layerscount=10;
//--- check
   if(d>=0.0)
      d=1;
   else
      d=-1;
//--- Allocate arrays
   ArrayResizeAL(lsizes,layerscount);
   ArrayResizeAL(ltypes,layerscount);
   ArrayResizeAL(lconnfirst,layerscount);
   ArrayResizeAL(lconnlast,layerscount);
//--- Layers
   AddInputLayer(nin,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddBiasedSummatorLayer(nhid1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddActivationLayer(1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddBiasedSummatorLayer(nhid2,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddActivationLayer(1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddBiasedSummatorLayer(nout,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddActivationLayer(3,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- Create
   MLPCreate(nin,nout,lsizes,ltypes,lconnfirst,lconnlast,layerscount,false,network);
//--- function call
   FillHighLevelInformation(network,nin,nhid1,nhid2,nout,false,false);
//--- Turn on ouputs shift/scaling.
   for(i=nin;i<=nin+nout-1;i++)
     {
      network.m_columnmeans[i]=b;
      network.m_columnsigmas[i]=d;
     }
  }
//+------------------------------------------------------------------+
//| Creates  neural  network  with  NIn  inputs,  NOut outputs,      |
//| without hidden layers with non-linear output layer. Network      |
//| weights are filled with small random values. Activation function | 
//| of the output layer takes values [A,B].                          |
//+------------------------------------------------------------------+
static void CMLPBase::MLPCreateR0(const int nin,const int nout,const double a,
                                  const double b,CMultilayerPerceptron &network)
  {
//--- create variables
   int layerscount=0;
   int lastproc=0;
   int i=0;
//--- creating arrays
   int lsizes[];
   int ltypes[];
   int lconnfirst[];
   int lconnlast[];
//--- initialization
   layerscount=1+3;
//--- Allocate arrays
   ArrayResizeAL(lsizes,layerscount);
   ArrayResizeAL(ltypes,layerscount);
   ArrayResizeAL(lconnfirst,layerscount);
   ArrayResizeAL(lconnlast,layerscount);
//--- Layers
   AddInputLayer(nin,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddBiasedSummatorLayer(nout,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddActivationLayer(1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- Create
   MLPCreate(nin,nout,lsizes,ltypes,lconnfirst,lconnlast,layerscount,false,network);
//--- function call
   FillHighLevelInformation(network,nin,0,0,nout,false,false);
//--- Turn on outputs shift/scaling.
   for(i=nin;i<=nin+nout-1;i++)
     {
      network.m_columnmeans[i]=0.5*(a+b);
      network.m_columnsigmas[i]=0.5*(a-b);
     }
  }
//+------------------------------------------------------------------+
//| Same as MLPCreateR0,but with non-linear hidden layer.            |
//+------------------------------------------------------------------+
static void CMLPBase::MLPCreateR1(const int nin,const int nhid,const int nout,
                                  const double a,const double b,
                                  CMultilayerPerceptron &network)
  {
//--- create variables
   int layerscount=0;
   int lastproc=0;
   int i=0;
//--- creating arrays
   int lsizes[];
   int ltypes[];
   int lconnfirst[];
   int lconnlast[];
//--- initialization
   layerscount=7;
//--- Allocate arrays
   ArrayResizeAL(lsizes,layerscount);
   ArrayResizeAL(ltypes,layerscount);
   ArrayResizeAL(lconnfirst,layerscount);
   ArrayResizeAL(lconnlast,layerscount);
//--- Layers
   AddInputLayer(nin,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddBiasedSummatorLayer(nhid,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddActivationLayer(1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddBiasedSummatorLayer(nout,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddActivationLayer(1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- Create
   MLPCreate(nin,nout,lsizes,ltypes,lconnfirst,lconnlast,layerscount,false,network);
//--- function call
   FillHighLevelInformation(network,nin,nhid,0,nout,false,false);
//--- Turn on outputs shift/scaling.
   for(i=nin;i<=nin+nout-1;i++)
     {
      network.m_columnmeans[i]=0.5*(a+b);
      network.m_columnsigmas[i]=0.5*(a-b);
     }
  }
//+------------------------------------------------------------------+
//| Same as MLPCreateR0,but with two non-linear hidden layers.       |
//+------------------------------------------------------------------+
static void CMLPBase::MLPCreateR2(const int nin,const int nhid1,const int nhid2,
                                  const int nout,const double a,const double b,
                                  CMultilayerPerceptron &network)
  {
//--- create variables
   int layerscount=0;
   int lastproc=0;
   int i=0;
//--- creating arrays
   int lsizes[];
   int ltypes[];
   int lconnfirst[];
   int lconnlast[];
//--- initialization
   layerscount=10;
//--- Allocate arrays
   ArrayResizeAL(lsizes,layerscount);
   ArrayResizeAL(ltypes,layerscount);
   ArrayResizeAL(lconnfirst,layerscount);
   ArrayResizeAL(lconnlast,layerscount);
//--- Layers
   AddInputLayer(nin,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddBiasedSummatorLayer(nhid1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddActivationLayer(1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddBiasedSummatorLayer(nhid2,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddActivationLayer(1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddBiasedSummatorLayer(nout,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddActivationLayer(1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- Create
   MLPCreate(nin,nout,lsizes,ltypes,lconnfirst,lconnlast,layerscount,false,network);
//--- function call
   FillHighLevelInformation(network,nin,nhid1,nhid2,nout,false,false);
//--- Turn on outputs shift/scaling.
   for(i=nin;i<=nin+nout-1;i++)
     {
      network.m_columnmeans[i]=0.5*(a+b);
      network.m_columnsigmas[i]=0.5*(a-b);
     }
  }
//+------------------------------------------------------------------+
//| Creates classifier network with NIn inputs and NOut possible     |
//| classes.                                                         |
//| Network contains no hidden layers and linear output layer with   |
//| SOFTMAX-normalization (so outputs sums up to 1.0 and converge to |
//| posterior probabilities).                                        |
//+------------------------------------------------------------------+
static void CMLPBase::MLPCreateC0(const int nin,const int nout,
                                  CMultilayerPerceptron &network)
  {
//--- create variables
   int layerscount=0;
   int lastproc=0;
//--- creating arrays
   int lsizes[];
   int ltypes[];
   int lconnfirst[];
   int lconnlast[];
//--- check
   if(!CAp::Assert(nout>=2,__FUNCTION__+": NOut<2!"))
      return;
//--- initialization
   layerscount=4;
//--- Allocate arrays
   ArrayResizeAL(lsizes,layerscount);
   ArrayResizeAL(ltypes,layerscount);
   ArrayResizeAL(lconnfirst,layerscount);
   ArrayResizeAL(lconnlast,layerscount);
//--- Layers
   AddInputLayer(nin,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddBiasedSummatorLayer(nout-1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddZeroLayer(lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- Create
   MLPCreate(nin,nout,lsizes,ltypes,lconnfirst,lconnlast,layerscount,true,network);
//--- function call
   FillHighLevelInformation(network,nin,0,0,nout,true,true);
  }
//+------------------------------------------------------------------+
//| Same as MLPCreateC0,but with one non-linear hidden layer.        |
//+------------------------------------------------------------------+
static void CMLPBase::MLPCreateC1(const int nin,const int nhid,const int nout,
                                  CMultilayerPerceptron &network)
  {
//--- create variables
   int layerscount=0;
   int lastproc=0;
//--- creating arrays
   int lsizes[];
   int ltypes[];
   int lconnfirst[];
   int lconnlast[];
//--- check
   if(!CAp::Assert(nout>=2,__FUNCTION__+": NOut<2!"))
      return;
//--- initialization
   layerscount=7;
//--- Allocate arrays
   ArrayResizeAL(lsizes,layerscount);
   ArrayResizeAL(ltypes,layerscount);
   ArrayResizeAL(lconnfirst,layerscount);
   ArrayResizeAL(lconnlast,layerscount);
//--- Layers
   AddInputLayer(nin,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddBiasedSummatorLayer(nhid,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddActivationLayer(1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddBiasedSummatorLayer(nout-1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddZeroLayer(lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- Create
   MLPCreate(nin,nout,lsizes,ltypes,lconnfirst,lconnlast,layerscount,true,network);
//--- function call
   FillHighLevelInformation(network,nin,nhid,0,nout,true,true);
  }
//+------------------------------------------------------------------+
//| Same as MLPCreateC0, but with two non-linear hidden layers.      |
//+------------------------------------------------------------------+
static void CMLPBase::MLPCreateC2(const int nin,const int nhid1,const int nhid2,
                                  const int nout,CMultilayerPerceptron &network)
  {
//--- create variables
   int layerscount=0;
   int lastproc=0;
//--- creating arrays
   int lsizes[];
   int ltypes[];
   int lconnfirst[];
   int lconnlast[];
//--- check
   if(!CAp::Assert(nout>=2,__FUNCTION__+": NOut<2!"))
      return;
//--- initialization
   layerscount=10;
//--- Allocate arrays
   ArrayResizeAL(lsizes,layerscount);
   ArrayResizeAL(ltypes,layerscount);
   ArrayResizeAL(lconnfirst,layerscount);
   ArrayResizeAL(lconnlast,layerscount);
//--- Layers
   AddInputLayer(nin,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddBiasedSummatorLayer(nhid1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddActivationLayer(1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddBiasedSummatorLayer(nhid2,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddActivationLayer(1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddBiasedSummatorLayer(nout-1,lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- function call
   AddZeroLayer(lsizes,ltypes,lconnfirst,lconnlast,lastproc);
//--- Create
   MLPCreate(nin,nout,lsizes,ltypes,lconnfirst,lconnlast,layerscount,true,network);
//--- function call
   FillHighLevelInformation(network,nin,nhid1,nhid2,nout,true,true);
  }
//+------------------------------------------------------------------+
//| Copying of neural network                                        |
//| INPUT PARAMETERS:                                                |
//|     Network1 -   original                                        |
//| OUTPUT PARAMETERS:                                               |
//|     Network2 -   copy                                            |
//+------------------------------------------------------------------+
static void CMLPBase::MLPCopy(CMultilayerPerceptron &network1,
                              CMultilayerPerceptron &network2)
  {
//--- copy
   network2.m_hlnetworktype=network1.m_hlnetworktype;
   network2.m_hlnormtype=network1.m_hlnormtype;
//--- function calls
   CApServ::CopyIntegerArray(network1.m_hllayersizes,network2.m_hllayersizes);
   CApServ::CopyIntegerArray(network1.m_hlconnections,network2.m_hlconnections);
   CApServ::CopyIntegerArray(network1.m_hlneurons,network2.m_hlneurons);
   CApServ::CopyIntegerArray(network1.m_structinfo,network2.m_structinfo);
   CApServ::CopyRealArray(network1.m_weights,network2.m_weights);
   CApServ::CopyRealArray(network1.m_columnmeans,network2.m_columnmeans);
   CApServ::CopyRealArray(network1.m_columnsigmas,network2.m_columnsigmas);
   CApServ::CopyRealArray(network1.m_neurons,network2.m_neurons);
   CApServ::CopyRealArray(network1.m_dfdnet,network2.m_dfdnet);
   CApServ::CopyRealArray(network1.m_derror,network2.m_derror);
   CApServ::CopyRealArray(network1.m_x,network2.m_x);
   CApServ::CopyRealArray(network1.m_y,network2.m_y);
   CApServ::CopyRealMatrix(network1.m_chunks,network2.m_chunks);
   CApServ::CopyRealArray(network1.m_nwbuf,network2.m_nwbuf);
   CApServ::CopyIntegerArray(network1.m_integerbuf,network2.m_integerbuf);
  }
//+------------------------------------------------------------------+
//| Serialization of MultiLayerPerceptron strucure                   |
//| INPUT PARAMETERS:                                                |
//|     Network -   original                                         |
//| OUTPUT PARAMETERS:                                               |
//|     RA      -   array of real numbers which stores network,      |
//|                 array[0..RLen-1]                                 |
//|     RLen    -   RA lenght                                        |
//+------------------------------------------------------------------+
static void CMLPBase::MLPSerializeOld(CMultilayerPerceptron &network,
                                      double &ra[],int &rlen)
  {
//--- create variables
   int i=0;
   int ssize=0;
   int ntotal=0;
   int nin=0;
   int nout=0;
   int wcount=0;
   int sigmalen=0;
   int offs=0;
   int i_=0;
   int i1_=0;
//--- initialization
   rlen=0;
//--- Unload info
   ssize=network.m_structinfo[0];
   nin=network.m_structinfo[1];
   nout=network.m_structinfo[2];
   ntotal=network.m_structinfo[3];
   wcount=network.m_structinfo[4];
//--- check
   if(MLPIsSoftMax(network))
      sigmalen=nin;
   else
      sigmalen=nin+nout;
//---  RA format:
//---      LEN         DESRC.
//---      1           RLen
//---      1           version (MLPVNum)
//---      1           StructInfo size
//---      SSize       StructInfo
//---      WCount      Weights
//---      SigmaLen    ColumnMeans
//---      SigmaLen    ColumnSigmas
   rlen=3+ssize+wcount+2*sigmalen;
//--- allocation
   ArrayResizeAL(ra,rlen);
//--- change values
   ra[0]=rlen;
   ra[1]=m_mlpvnum;
   ra[2]=ssize;
//--- calculation
   offs=3;
   for(i=0;i<=ssize-1;i++)
      ra[offs+i]=network.m_structinfo[i];
//--- calculation
   offs=offs+ssize;
   i1_=-offs;
   for(i_=offs;i_<=offs+wcount-1;i_++)
      ra[i_]=network.m_weights[i_+i1_];
//--- calculation
   offs=offs+wcount;
   i1_=-offs;
   for(i_=offs;i_<=offs+sigmalen-1;i_++)
      ra[i_]=network.m_columnmeans[i_+i1_];
//--- calculation
   offs=offs+sigmalen;
   i1_=-offs;
   for(i_=offs;i_<=offs+sigmalen-1;i_++)
      ra[i_]=network.m_columnsigmas[i_+i1_];
   offs=offs+sigmalen;
  }
//+------------------------------------------------------------------+
//| Unserialization of MultiLayerPerceptron strucure                 |
//| INPUT PARAMETERS:                                                |
//|     RA      -   real array which stores network                  |
//| OUTPUT PARAMETERS:                                               |
//|     Network -   restored network                                 |
//+------------------------------------------------------------------+
static void CMLPBase::MLPUnserializeOld(double &ra[],CMultilayerPerceptron &network)
  {
//--- create variables
   int i=0;
   int ssize=0;
   int ntotal=0;
   int nin=0;
   int nout=0;
   int wcount=0;
   int sigmalen=0;
   int offs=0;
   int i_=0;
   int i1_=0;
//--- check
   if(!CAp::Assert((int)MathRound(ra[1])==m_mlpvnum,__FUNCTION__+": incorrect array!"))
      return;
//--- Unload StructInfo from IA
   offs=3;
   ssize=(int)MathRound(ra[2]);
//--- allocation
   ArrayResizeAL(network.m_structinfo,ssize);
   for(i=0;i<=ssize-1;i++)
      network.m_structinfo[i]=(int)MathRound(ra[offs+i]);
   offs=offs+ssize;
//--- Unload info from StructInfo
   ssize=network.m_structinfo[0];
   nin=network.m_structinfo[1];
   nout=network.m_structinfo[2];
   ntotal=network.m_structinfo[3];
   wcount=network.m_structinfo[4];
//--- check
   if(network.m_structinfo[6]==0)
      sigmalen=nin+nout;
   else
      sigmalen=nin;
//--- Allocate space for other fields
   ArrayResizeAL(network.m_weights,wcount);
   ArrayResizeAL(network.m_columnmeans,sigmalen);
   ArrayResizeAL(network.m_columnsigmas,sigmalen);
   ArrayResizeAL(network.m_neurons,ntotal);
   network.m_chunks.Resize(3*ntotal+1,m_chunksize);
   ArrayResizeAL(network.m_nwbuf,MathMax(wcount,2*nout));
   ArrayResizeAL(network.m_dfdnet,ntotal);
   ArrayResizeAL(network.m_x,nin);
   ArrayResizeAL(network.m_y,nout);
   ArrayResizeAL(network.m_derror,ntotal);
//--- Copy parameters from RA
   i1_=offs;
   for(i_=0;i_<=wcount-1;i_++)
      network.m_weights[i_]=ra[i_+i1_];
//--- calculation
   offs=offs+wcount;
   i1_=offs;
   for(i_=0;i_<=sigmalen-1;i_++)
      network.m_columnmeans[i_]=ra[i_+i1_];
//--- calculation
   offs=offs+sigmalen;
   i1_=offs;
   for(i_=0;i_<=sigmalen-1;i_++)
      network.m_columnsigmas[i_]=ra[i_+i1_];
   offs=offs+sigmalen;
  }
//+------------------------------------------------------------------+
//| Randomization of neural network weights                          |
//+------------------------------------------------------------------+
static void CMLPBase::MLPRandomize(CMultilayerPerceptron &network)
  {
//--- create variables
   int i=0;
   int nin=0;
   int nout=0;
   int wcount=0;
//--- function call
   MLPProperties(network,nin,nout,wcount);
//--- change values
   for(i=0;i<=wcount-1;i++)
      network.m_weights[i]=CMath::RandomReal()-0.5;
  }
//+------------------------------------------------------------------+
//| Randomization of neural network weights and standartisator       |
//+------------------------------------------------------------------+
static void CMLPBase::MLPRandomizeFull(CMultilayerPerceptron &network)
  {
//--- create variables
   int i=0;
   int nin=0;
   int nout=0;
   int wcount=0;
   int ntotal=0;
   int istart=0;
   int offs=0;
   int ntype=0;
//--- function call
   MLPProperties(network,nin,nout,wcount);
//--- initialization
   ntotal=network.m_structinfo[3];
   istart=network.m_structinfo[5];
//--- Process network
   for(i=0;i<=wcount-1;i++)
      network.m_weights[i]=CMath::RandomReal()-0.5;
   for(i=0;i<=nin-1;i++)
     {
      network.m_columnmeans[i]=2*CMath::RandomReal()-1;
      network.m_columnsigmas[i]=1.5*CMath::RandomReal()+0.5;
     }
//--- check
   if(!MLPIsSoftMax(network))
     {
      for(i=0;i<=nout-1;i++)
        {
         offs=istart+(ntotal-nout+i)*m_nfieldwidth;
         ntype=network.m_structinfo[offs+0];
         //--- check
         if(ntype==0)
           {
            //--- Shifts are changed only for linear outputs neurons
            network.m_columnmeans[nin+i]=2*CMath::RandomReal()-1;
           }
         //--- check
         if(ntype==0 || ntype==3)
           {
            //--- Scales are changed only for linear or bounded outputs neurons.
            //--- Note that scale randomization preserves sign.
            network.m_columnsigmas[nin+i]=MathSign(network.m_columnsigmas[nin+i])*(1.5*CMath::RandomReal()+0.5);
           }
        }
     }
  }
//+------------------------------------------------------------------+
//| Internal subroutine.                                             |
//+------------------------------------------------------------------+
static void CMLPBase::MLPInitPreprocessor(CMultilayerPerceptron &network,
                                          CMatrixDouble &xy,const int ssize)
  {
//--- create variables
   int    i=0;
   int    j=0;
   int    jmax=0;
   int    nin=0;
   int    nout=0;
   int    wcount=0;
   int    ntotal=0;
   int    istart=0;
   int    offs=0;
   int    ntype=0;
   double s=0;
//--- creating arrays
   double means[];
   double sigmas[];
//--- function call
   MLPProperties(network,nin,nout,wcount);
//--- initialization
   ntotal=network.m_structinfo[3];
   istart=network.m_structinfo[5];
//--- Means/Sigmas
   if(MLPIsSoftMax(network))
      jmax=nin-1;
   else
      jmax=nin+nout-1;
//--- allocation
   ArrayResizeAL(means,jmax+1);
   ArrayResizeAL(sigmas,jmax+1);
//--- calculation
   for(j=0;j<=jmax;j++)
     {
      //--- means
      means[j]=0;
      for(i=0;i<=ssize-1;i++)
         means[j]=means[j]+xy[i][j];
      means[j]=means[j]/ssize;
      //--- sigmas
      sigmas[j]=0;
      for(i=0;i<=ssize-1;i++)
         sigmas[j]=sigmas[j]+CMath::Sqr(xy[i][j]-means[j]);
      sigmas[j]=MathSqrt(sigmas[j]/ssize);
     }
//--- Inputs
   for(i=0;i<=nin-1;i++)
     {
      network.m_columnmeans[i]=means[i];
      network.m_columnsigmas[i]=sigmas[i];
      //--- check
      if(network.m_columnsigmas[i]==0.0)
         network.m_columnsigmas[i]=1;
     }
//--- Outputs
   if(!MLPIsSoftMax(network))
     {
      for(i=0;i<=nout-1;i++)
        {
         offs=istart+(ntotal-nout+i)*m_nfieldwidth;
         ntype=network.m_structinfo[offs+0];
         //--- Linear outputs
         if(ntype==0)
           {
            network.m_columnmeans[nin+i]=means[nin+i];
            network.m_columnsigmas[nin+i]=sigmas[nin+i];
            //--- check
            if(network.m_columnsigmas[nin+i]==0.0)
               network.m_columnsigmas[nin+i]=1;
           }
         //--- Bounded outputs (half-interval)
         if(ntype==3)
           {
            s=means[nin+i]-network.m_columnmeans[nin+i];
            //--- check
            if(s==0.0)
               s=MathSign(network.m_columnsigmas[nin+i]);
            //--- check
            if(s==0.0)
               s=1.0;
            //--- change value
            network.m_columnsigmas[nin+i]=MathSign(network.m_columnsigmas[nin+i])*MathAbs(s);
            //--- check
            if((double)(network.m_columnsigmas[nin+i])==0.0)
               network.m_columnsigmas[nin+i]=1;
           }
        }
     }
  }
//+------------------------------------------------------------------+
//| Returns information about initialized network: number of inputs, | 
//| outputs, weights.                                                |
//+------------------------------------------------------------------+
static void CMLPBase::MLPProperties(CMultilayerPerceptron &network,int &nin,
                                    int &nout,int &wcount)
  {
//--- change values
   nin=network.m_structinfo[1];
   nout=network.m_structinfo[2];
   wcount=network.m_structinfo[4];
  }
//+------------------------------------------------------------------+
//| Tells whether network is SOFTMAX-normalized (i.m_e. classifier)  |
//| or not.                                                          |
//+------------------------------------------------------------------+
static bool CMLPBase::MLPIsSoftMax(CMultilayerPerceptron &network)
  {
//--- check
   if(network.m_structinfo[6]==1)
      return(true);
//--- return result
   return(false);
  }
//+------------------------------------------------------------------+
//| This function returns total number of layers (including input,   |
//| hidden and output layers).                                       |
//+------------------------------------------------------------------+
static int CMLPBase::MLPGetLayersCount(CMultilayerPerceptron &network)
  {
//--- return result
   return(CAp::Len(network.m_hllayersizes));
  }
//+------------------------------------------------------------------+
//| This function returns size of K-th layer.                        |
//| K=0 corresponds to input layer, K=CNT-1 corresponds to output    |
//| layer.                                                           |
//| Size of the output layer is always equal to the number of        |
//| outputs, although when we have softmax-normalized network, last  | 
//| neuron doesn't have any connections - it is just zero.           |
//+------------------------------------------------------------------+
static int CMLPBase::MLPGetLayerSize(CMultilayerPerceptron &network,
                                     const int k)
  {
//--- check
   if(!CAp::Assert(k>=0 && k<CAp::Len(network.m_hllayersizes),__FUNCTION__+": incorrect layer index"))
      return(-1);
//--- return result
   return(network.m_hllayersizes[k]);
  }
//+------------------------------------------------------------------+
//| This function returns offset/scaling coefficients for I-th input | 
//| of the network.                                                  |
//| INPUT PARAMETERS:                                                |
//|     Network     -   network                                      |
//|     I           -   input index                                  |
//| OUTPUT PARAMETERS:                                               |
//|     Mean        -   mean term                                    |
//|     Sigma       -   sigma term,guaranteed to be nonzero.         |
//| I-th input is passed through linear transformation               |
//|     IN[i]=(IN[i]-Mean)/Sigma                                     |
//| before feeding to the network                                    |
//+------------------------------------------------------------------+
static void CMLPBase::MLPGetInputScaling(CMultilayerPerceptron &network,
                                         const int i,double &mean,
                                         double &sigma)
  {
//--- initialization
   mean=0;
   sigma=0;
//--- check
   if(!CAp::Assert(i>=0 && i<network.m_hllayersizes[0],__FUNCTION__+": incorrect (nonexistent) I"))
      return;
//--- change values
   mean=network.m_columnmeans[i];
   sigma=network.m_columnsigmas[i];
//--- check
   if(sigma==0.0)
      sigma=1;
  }
//+------------------------------------------------------------------+
//| This function returns offset/scaling coefficients for I-th output|
//| of the network.                                                  |
//| INPUT PARAMETERS:                                                |
//|     Network     -   network                                      |
//|     I           -   input index                                  |
//| OUTPUT PARAMETERS:                                               |
//|     Mean        -   mean term                                    |
//|     Sigma       -   sigma term, guaranteed to be nonzero.        |
//| I-th output is passed through linear transformation              |
//|     OUT[i] = OUT[i]*Sigma+Mean                                   |
//| before returning it to user. In case we have SOFTMAX-normalized  |
//| network, we return (Mean,Sigma)=(0.0,1.0).                       |
//+------------------------------------------------------------------+
static void CMLPBase::MLPGetOutputScaling(CMultilayerPerceptron &network,
                                          const int i,double &mean,
                                          double &sigma)
  {
//--- initialization
   mean=0;
   sigma=0;
//--- check
   if(!CAp::Assert(i>=0 && i<network.m_hllayersizes[CAp::Len(network.m_hllayersizes)-1],__FUNCTION__+": incorrect (nonexistent) I"))
      return;
//--- check
   if(network.m_structinfo[6]==1)
     {
      //--- change values
      mean=0;
      sigma=1;
     }
   else
     {
      //--- change values
      mean=network.m_columnmeans[network.m_hllayersizes[0]+i];
      sigma=network.m_columnsigmas[network.m_hllayersizes[0]+i];
     }
  }
//+------------------------------------------------------------------+
//| This function returns information about Ith neuron of Kth layer  |
//| INPUT PARAMETERS:                                                |
//|     Network     -   network                                      |
//|     K           -   layer index                                  |
//|     I           -   neuron index (within layer)                  |
//| OUTPUT PARAMETERS:                                               |
//|     FKind       -   activation function type (used by            |
//|                     MLPActivationFunction()) this value is zero  |
//|                     for input or linear neurons                  |
//|     Threshold   -   also called offset, bias                     |
//|                     zero for input neurons                       |
//| NOTE: this function throws exception if layer or neuron with     |
//| given index do not exists.                                       |
//+------------------------------------------------------------------+
static void CMLPBase::MLPGetNeuronInfo(CMultilayerPerceptron &network,
                                       const int k,const int i,int &fkind,
                                       double &threshold)
  {
//--- create variables
   int ncnt=0;
   int istart=0;
   int highlevelidx=0;
   int activationoffset=0;
//--- initialization
   fkind=0;
   threshold=0;
   ncnt=CAp::Len(network.m_hlneurons)/m_hlm_nfieldwidth;
   istart=network.m_structinfo[5];
//--- search
   network.m_integerbuf[0]=k;
   network.m_integerbuf[1]=i;
//--- function call
   highlevelidx=CApServ::RecSearch(network.m_hlneurons,m_hlm_nfieldwidth,2,0,ncnt,network.m_integerbuf);
//--- check
   if(!CAp::Assert(highlevelidx>=0,__FUNCTION__+": incorrect (nonexistent) layer or neuron index"))
      return;
//--- 1. find offset of the activation function record in the
   if(network.m_hlneurons[highlevelidx*m_hlm_nfieldwidth+2]>=0)
     {
      activationoffset=istart+network.m_hlneurons[highlevelidx*m_hlm_nfieldwidth+2]*m_nfieldwidth;
      fkind=network.m_structinfo[activationoffset+0];
     }
   else
      fkind=0;
//--- check
   if(network.m_hlneurons[highlevelidx*m_hlm_nfieldwidth+3]>=0)
      threshold=network.m_weights[network.m_hlneurons[highlevelidx*m_hlm_nfieldwidth+3]];
   else
      threshold=0;
  }
//+------------------------------------------------------------------+
//| This function returns information about connection from I0-th    |
//| neuron of K0-th layer to I1-th neuron of K1-th layer.            |
//| INPUT PARAMETERS:                                                |
//|     Network     -   network                                      |
//|     K0          -   layer index                                  |
//|     I0          -   neuron index (within layer)                  |
//|     K1          -   layer index                                  |
//|     I1          -   neuron index (within layer)                  |
//| RESULT:                                                          |
//|     connection weight (zero for non-existent connections)        |
//| This function:                                                   |
//| 1. throws exception if layer or neuron with given index do not   |
//|    exists.                                                       |
//| 2. returns zero if neurons exist, but there is no connection     |
//|    between them                                                  |
//+------------------------------------------------------------------+
static double CMLPBase::MLPGetWeight(CMultilayerPerceptron &network,
                                     const int k0,const int i0,
                                     const int k1,const int i1)
  {
//--- create variables
   double result=0;
   int    ccnt=0;
   int    highlevelidx=0;
//--- initialization
   ccnt=CAp::Len(network.m_hlconnections)/m_hlconm_nfieldwidth;
//--- check params
   if(!CAp::Assert(k0>=0 && k0<CAp::Len(network.m_hllayersizes),__FUNCTION__+": incorrect (nonexistent) K0"))
      return(EMPTY_VALUE);
//--- check
   if(!CAp::Assert(i0>=0 && i0<network.m_hllayersizes[k0],__FUNCTION__+": incorrect (nonexistent) I0"))
      return(EMPTY_VALUE);
//--- check
   if(!CAp::Assert(k1>=0 && k1<CAp::Len(network.m_hllayersizes),__FUNCTION__+": incorrect (nonexistent) K1"))
      return(EMPTY_VALUE);
//--- check
   if(!CAp::Assert(i1>=0 && i1<network.m_hllayersizes[k1],__FUNCTION__+": incorrect (nonexistent) I1"))
      return(EMPTY_VALUE);
//--- search
   network.m_integerbuf[0]=k0;
   network.m_integerbuf[1]=i0;
   network.m_integerbuf[2]=k1;
   network.m_integerbuf[3]=i1;
//--- function call
   highlevelidx=CApServ::RecSearch(network.m_hlconnections,m_hlconm_nfieldwidth,4,0,ccnt,network.m_integerbuf);
//--- check
   if(highlevelidx>=0)
      result=network.m_weights[network.m_hlconnections[highlevelidx*m_hlconm_nfieldwidth+4]];
   else
      result=0;
//--- return result
   return(result);
  }
//+------------------------------------------------------------------+
//| This function sets offset/scaling coefficients for I-th input of | 
//| the network.                                                     |
//| INPUT PARAMETERS:                                                |
//|     Network     -   network                                      |
//|     I           -   input index                                  |
//|     Mean        -   mean term                                    |
//|     Sigma       -   sigma term (if zero,will be replaced by 1.0) |
//| NTE: I-th input is passed through linear transformation          |
//|     IN[i]=(IN[i]-Mean)/Sigma                                     |
//| before feeding to the network. This function sets Mean and Sigma.|
//+------------------------------------------------------------------+
static void CMLPBase::MLPSetInputScaling(CMultilayerPerceptron &network,
                                         const int i,const double mean,
                                         double sigma)
  {
//--- check
   if(!CAp::Assert(i>=0 && i<network.m_hllayersizes[0],__FUNCTION__+": incorrect (nonexistent) I"))
      return;
//--- check
   if(!CAp::Assert(CMath::IsFinite(mean),__FUNCTION__+": infinite or NAN Mean"))
      return;
//--- check
   if(!CAp::Assert(CMath::IsFinite(sigma),__FUNCTION__+": infinite or NAN Sigma"))
      return;
//--- check
   if(sigma==0.0)
      sigma=1;
//--- change values
   network.m_columnmeans[i]=mean;
   network.m_columnsigmas[i]=sigma;
  }
//+------------------------------------------------------------------+
//| This function sets offset/scaling coefficients for I-th output of| 
//| the network.                                                     |
//| INPUT PARAMETERS:                                                |
//|     Network     -   network                                      |
//|     I           -   input index                                  |
//|     Mean        -   mean term                                    |
//|     Sigma       -   sigma term (if zero, will be replaced by 1.0)|
//| OUTPUT PARAMETERS:                                               |
//| NOTE: I-th output is passed through linear transformation        |
//|     OUT[i] = OUT[i]*Sigma+Mean                                   |
//| before returning it to user. This function sets Sigma/Mean. In   | 
//| case we have SOFTMAX-normalized network, you can not set (Sigma, |
//| Mean) to anything other than(0.0,1.0) - this function will throw |
//| exception.                                                       |
//+------------------------------------------------------------------+
static void CMLPBase::MLPSetOutputScaling(CMultilayerPerceptron &network,
                                          const int i,const double mean,
                                          double sigma)
  {
//--- check
   if(!CAp::Assert(i>=0 && i<network.m_hllayersizes[CAp::Len(network.m_hllayersizes)-1],__FUNCTION__+": incorrect (nonexistent) I"))
      return;
//--- check
   if(!CAp::Assert(CMath::IsFinite(mean),__FUNCTION__+": infinite or NAN Mean"))
      return;
//--- check
   if(!CAp::Assert(CMath::IsFinite(sigma),__FUNCTION__+": infinite or NAN Sigma"))
      return;
//--- check
   if(network.m_structinfo[6]==1)
     {
      //--- check
      if(!CAp::Assert(mean==0.0,__FUNCTION__+": you can not set non-zero Mean term for classifier network"))
         return;
      //--- check
      if(!CAp::Assert(sigma==1.0,__FUNCTION__+": you can not set non-unit Sigma term for classifier network"))
         return;
     }
   else
     {
      //--- check
      if(sigma==0.0)
         sigma=1;
      //--- change values
      network.m_columnmeans[network.m_hllayersizes[0]+i]=mean;
      network.m_columnsigmas[network.m_hllayersizes[0]+i]=sigma;
     }
  }
//+------------------------------------------------------------------+
//| This function modifies information about Ith neuron of Kth layer |
//| INPUT PARAMETERS:                                                |
//|     Network     -   network                                      |
//|     K           -   layer index                                  |
//|     I           -   neuron index (within layer)                  |
//|     FKind       -   activation function type (used by            |
//|                     MLPActivationFunction()) this value must be  |
//|                     zero for input neurons (you can not set      |
//|                     activation function for input neurons)       |
//|     Threshold   -   also called offset, bias                     |
//|                     this value must be zero for input neurons    |
//|                     (you can not set threshold for input neurons)|
//| NOTES:                                                           |
//| 1. this function throws exception if layer or neuron with given  |
//|    index do not exists.                                          |
//| 2. this function also throws exception when you try to set       |
//|    non-linear activation function for input neurons (any kind    |
//|    of network) or for output neurons of classifier network.      |
//| 3. this function throws exception when you try to set non-zero   |
//|    threshold for input neurons (any kind of network).            |
//+------------------------------------------------------------------+
static void CMLPBase::MLPSetNeuronInfo(CMultilayerPerceptron &network,
                                       const int k,const int i,
                                       const int fkind,const double threshold)
  {
//--- create variables
   int ncnt=0;
   int istart=0;
   int highlevelidx=0;
   int activationoffset=0;
//--- check
   if(!CAp::Assert(CMath::IsFinite(threshold),__FUNCTION__+": infinite or NAN Threshold"))
      return;
//--- convenience vars
   ncnt=CAp::Len(network.m_hlneurons)/m_hlm_nfieldwidth;
   istart=network.m_structinfo[5];
//--- search
   network.m_integerbuf[0]=k;
   network.m_integerbuf[1]=i;
//--- function call
   highlevelidx=CApServ::RecSearch(network.m_hlneurons,m_hlm_nfieldwidth,2,0,ncnt,network.m_integerbuf);
//--- check
   if(!CAp::Assert(highlevelidx>=0,__FUNCTION__+": incorrect (nonexistent) layer or neuron index"))
      return;
//--- activation function
   if(network.m_hlneurons[highlevelidx*m_hlm_nfieldwidth+2]>=0)
     {
      activationoffset=istart+network.m_hlneurons[highlevelidx*m_hlm_nfieldwidth+2]*m_nfieldwidth;
      network.m_structinfo[activationoffset+0]=fkind;
     }
   else
     {
      //--- check
      if(!CAp::Assert(fkind==0,__FUNCTION__+": you try to set activation function for neuron which can not have one"))
         return;
     }
//--- Threshold
   if(network.m_hlneurons[highlevelidx*m_hlm_nfieldwidth+3]>=0)
      network.m_weights[network.m_hlneurons[highlevelidx*m_hlm_nfieldwidth+3]]=threshold;
   else
     {
      //--- check
      if(!CAp::Assert(threshold==0.0,__FUNCTION__+": you try to set non-zero threshold for neuron which can not have one"))
         return;
     }
  }
//+------------------------------------------------------------------+
//| This function modifies information about connection from I0-th   |
//| neuron of K0-th layer to I1-th neuron of K1-th layer.            |
//| INPUT PARAMETERS:                                                |
//|     Network     -   network                                      |
//|     K0          -   layer index                                  |
//|     I0          -   neuron index (within layer)                  |
//|     K1          -   layer index                                  |
//|     I1          -   neuron index (within layer)                  |
//|     W           -   connection weight (must be zero for          |
//|                     non-existent connections)                    |
//| This function:                                                   |
//| 1. throws exception if layer or neuron with given index do not   |
//|    exists.                                                       |
//| 2. throws exception if you try to set non-zero weight for        |
//|    non-existent connection                                       |
//+------------------------------------------------------------------+
static void CMLPBase::MLPSetWeight(CMultilayerPerceptron &network,const int k0,
                                   const int i0,const int k1,
                                   const int i1,const double w)
  {
//--- create variables
   int ccnt=0;
   int highlevelidx=0;
//--- initialization
   ccnt=CAp::Len(network.m_hlconnections)/m_hlconm_nfieldwidth;
//--- check params
   if(!CAp::Assert(k0>=0 && k0<CAp::Len(network.m_hllayersizes),__FUNCTION__+": incorrect (nonexistent) K0"))
      return;
//--- check
   if(!CAp::Assert(i0>=0 && i0<network.m_hllayersizes[k0],__FUNCTION__+": incorrect (nonexistent) I0"))
      return;
//--- check
   if(!CAp::Assert(k1>=0 && k1<CAp::Len(network.m_hllayersizes),__FUNCTION__+": incorrect (nonexistent) K1"))
      return;
//--- check
   if(!CAp::Assert(i1>=0 && i1<network.m_hllayersizes[k1],__FUNCTION__+": incorrect (nonexistent) I1"))
      return;
//--- check
   if(!CAp::Assert(CMath::IsFinite(w),__FUNCTION__+": infinite or NAN weight"))
      return;
//--- search
   network.m_integerbuf[0]=k0;
   network.m_integerbuf[1]=i0;
   network.m_integerbuf[2]=k1;
   network.m_integerbuf[3]=i1;
//--- function call
   highlevelidx=CApServ::RecSearch(network.m_hlconnections,m_hlconm_nfieldwidth,4,0,ccnt,network.m_integerbuf);
//--- check
   if(highlevelidx>=0)
      network.m_weights[network.m_hlconnections[highlevelidx*m_hlconm_nfieldwidth+4]]=w;
   else
     {
      //--- check
      if(!CAp::Assert(w==0.0,__FUNCTION__+": you try to set non-zero weight for non-existent connection"))
         return;
     }
  }
//+------------------------------------------------------------------+
//| Neural network activation function                               |
//| INPUT PARAMETERS:                                                |
//|     NET         -   neuron input                                 |
//|     K           -   function index (zero for linear function)    |
//| OUTPUT PARAMETERS:                                               |
//|     F           -   function                                     |
//|     DF          -   its derivative                               |
//|     D2F         -   its second derivative                        |
//+------------------------------------------------------------------+
static void CMLPBase::MLPActivationFunction(double net,const int k,double &f,
                                            double &df,double &d2f)
  {
//--- create variables
   double net2=0;
   double arg=0;
   double root=0;
   double r=0;
//--- initialization
   f=0;
   df=0;
   d2f=0;
//--- check
   if(k==0 || k==-5)
     {
      f=net;
      df=1;
      d2f=0;
      //--- exit the function
      return;
     }
//--- check
   if(k==1)
     {
      //--- TanH activation function
      if(MathAbs(net)<100.0)
         f=MathTanh(net);
      else
         f=MathSign(net);
      //--- change values
      df=1-CMath::Sqr(f);
      d2f=-(2*f*df);
      //--- exit the function
      return;
     }
//--- check
   if(k==3)
     {
      //--- EX activation function
      if(net>=0.0)
        {
         //--- change values
         net2=net*net;
         arg=net2+1;
         root=MathSqrt(arg);
         f=net+root;
         r=net/root;
         df=1+r;
         d2f=(root-net*r)/arg;
        }
      else
        {
         //--- change values
         f=MathExp(net);
         df=f;
         d2f=f;
        }
      //--- exit the function
      return;
     }
//--- check
   if(k==2)
     {
      //--- calculation
      f=MathExp(-CMath::Sqr(net));
      df=-(2*net*f);
      d2f=-(2*(f+df*net));
      //--- exit the function
      return;
     }
//--- change values
   f=0;
   df=0;
   d2f=0;
  }
//+------------------------------------------------------------------+
//| Procesing                                                        |
//| INPUT PARAMETERS:                                                |
//|     Network -   neural network                                   |
//|     X       -   input vector,  array[0..NIn-1].                  |
//| OUTPUT PARAMETERS:                                               |
//|     Y       -   result. Regression estimate when solving         |
//|                 regression task, vector of posterior             |
//|                 probabilities for classification task.           |
//| See also MLPProcessI                                             |
//+------------------------------------------------------------------+
static void CMLPBase::MLPProcess(CMultilayerPerceptron &network,double &x[],
                                 double &y[])
  {
//--- check
   if(CAp::Len(y)<network.m_structinfo[2])
      ArrayResizeAL(y,network.m_structinfo[2]);
//--- function call
   MLPInternalProcessVector(network.m_structinfo,network.m_weights,network.m_columnmeans,network.m_columnsigmas,network.m_neurons,network.m_dfdnet,x,y);
  }
//+------------------------------------------------------------------+
//| 'interactive' variant of MLPProcess for languages like Python    | 
//| which support constructs like "Y = MLPProcess(NN,X)" and         |
//| interactive mode of the interpreter                              |
//| This function allocates new array on each call, so it is         |
//| significantly slower than its 'non-interactive' counterpart,     |
//| but it is more convenient when you call it from command line.    |
//+------------------------------------------------------------------+
static void CMLPBase::MLPProcessI(CMultilayerPerceptron &network,double &x[],
                                  double &y[])
  {
//--- function call
   MLPProcess(network,x,y);
  }
//+------------------------------------------------------------------+
//| Error function for neural network,internal subroutine.           |
//+------------------------------------------------------------------+
static double CMLPBase::MLPError(CMultilayerPerceptron &network,
                                 CMatrixDouble &xy,const int ssize)
  {
//--- create variables
   double result=0;
   int    i=0;
   int    k=0;
   int    nin=0;
   int    nout=0;
   int    wcount=0;
   double e=0;
   int    i_=0;
   int    i1_=0;
//--- function call
   MLPProperties(network,nin,nout,wcount);
//--- calculation
   for(i=0;i<=ssize-1;i++)
     {
      for(i_=0;i_<=nin-1;i_++)
         network.m_x[i_]=xy[i][i_];
      //--- function call
      MLPProcess(network,network.m_x,network.m_y);
      //--- check
      if(MLPIsSoftMax(network))
        {
         //--- class labels outputs
         k=(int)MathRound(xy[i][nin]);
         //--- check
         if(k>=0 && k<nout)
            network.m_y[k]=network.m_y[k]-1;
        }
      else
        {
         //--- real outputs
         i1_=nin;
         for(i_=0;i_<=nout-1;i_++)
            network.m_y[i_]=network.m_y[i_]-xy[i][i_+i1_];
        }
      //--- calculation
      e=0.0;
      for(i_=0;i_<=nout-1;i_++)
         e+=network.m_y[i_]*network.m_y[i_];
      result=result+e/2;
     }
//--- return result
   return(result);
  }
//+------------------------------------------------------------------+
//| Natural error function for neural network,internal subroutine.   |
//+------------------------------------------------------------------+
static double CMLPBase::MLPErrorN(CMultilayerPerceptron &network,
                                  CMatrixDouble &xy,const int ssize)
  {
//--- create variables
   double result=0;
   int    i=0;
   int    k=0;
   int    nin=0;
   int    nout=0;
   int    wcount=0;
   double e=0;
   int    i_=0;
   int    i1_=0;
//--- function call
   MLPProperties(network,nin,nout,wcount);
//--- calculation
   for(i=0;i<=ssize-1;i++)
     {
      //--- Process vector
      for(i_=0;i_<=nin-1;i_++)
         network.m_x[i_]=xy[i][i_];
      //--- function call
      MLPProcess(network,network.m_x,network.m_y);
      //--- Update error function
      if(network.m_structinfo[6]==0)
        {
         //--- Least squares error function
         i1_=nin;
         for(i_=0;i_<=nout-1;i_++)
            network.m_y[i_]=network.m_y[i_]-xy[i][i_+i1_];
         //--- calculation
         e=0.0;
         for(i_=0;i_<=nout-1;i_++)
            e+=network.m_y[i_]*network.m_y[i_];
         result=result+e/2;
        }
      else
        {
         //--- Cross-entropy error function
         k=(int)MathRound(xy[i][nin]);
         //--- check
         if(k>=0 && k<nout)
            result=result+SafeCrossEntropy(1,network.m_y[k]);
        }
     }
//--- return result
   return(result);
  }
//+------------------------------------------------------------------+
//| Classification error                                             |
//+------------------------------------------------------------------+
static int CMLPBase::MLPClsError(CMultilayerPerceptron &network,
                                 CMatrixDouble &xy,const int ssize)
  {
//--- create variables
   int result=0;
   int i=0;
   int j=0;
   int nin=0;
   int nout=0;
   int wcount=0;
   int nn=0;
   int ns=0;
   int nmax=0;
   int i_=0;
//--- creating arrays
   double workx[];
   double worky[];
//--- function call
   MLPProperties(network,nin,nout,wcount);
//--- allocation
   ArrayResizeAL(workx,nin);
   ArrayResizeAL(worky,nout);
//--- calculation
   for(i=0;i<=ssize-1;i++)
     {
      //--- Process
      for(i_=0;i_<=nin-1;i_++)
         workx[i_]=xy[i][i_];
      //--- function call
      MLPProcess(network,workx,worky);
      //--- Network version of the answer
      nmax=0;
      for(j=0;j<=nout-1;j++)
        {
         //--- check
         if(worky[j]>worky[nmax])
            nmax=j;
        }
      nn=nmax;
      //--- Right answer
      if(MLPIsSoftMax(network))
         ns=(int)MathRound(xy[i][nin]);
      else
        {
         nmax=0;
         for(j=0;j<=nout-1;j++)
           {
            //--- check
            if(xy[i][nin+j]>xy[i][nin+nmax])
               nmax=j;
           }
         ns=nmax;
        }
      //--- compare
      if(nn!=ns)
         result=result+1;
     }
//--- return result
   return(result);
  }
//+------------------------------------------------------------------+
//| Relative classification error on the test set                    |
//| INPUT PARAMETERS:                                                |
//|     Network -   network                                          |
//|     XY      -   test set                                         |
//|     NPoints -   test set size                                    |
//| RESULT:                                                          |
//|     percent of incorrectly classified cases. Works both for      |
//|     classifier networks and general purpose networks used as     |
//|     classifiers.                                                 |
//+------------------------------------------------------------------+
static double CMLPBase::MLPRelClsError(CMultilayerPerceptron &network,
                                       CMatrixDouble &xy,const int npoints)
  {
//--- return result
   return((double)MLPClsError(network,xy,npoints)/(double)npoints);
  }
//+------------------------------------------------------------------+
//| Average cross-entropy (in bits per element) on the test set      |
//| INPUT PARAMETERS:                                                |
//|     Network -   neural network                                   |
//|     XY      -   test set                                         |
//|     NPoints -   test set size                                    |
//| RESULT:                                                          |
//|     CrossEntropy/(NPoints*LN(2)).                                |
//|     Zero if network solves regression task.                      |
//+------------------------------------------------------------------+
static double CMLPBase::MLPAvgCE(CMultilayerPerceptron &network,CMatrixDouble &xy,
                                 const int npoints)
  {
//--- create variables
   double result=0;
   int    nin=0;
   int    nout=0;
   int    wcount=0;
//--- check
   if(MLPIsSoftMax(network))
     {
      //--- function call
      MLPProperties(network,nin,nout,wcount);
      //--- get result
      result=MLPErrorN(network,xy,npoints)/(npoints*MathLog(2));
     }
   else
      result=0;
//--- return result
   return(result);
  }
//+------------------------------------------------------------------+
//| RMS error on the test set                                        |
//| INPUT PARAMETERS:                                                |
//|     Network -   neural network                                   |
//|     XY      -   test set                                         |
//|     NPoints -   test set size                                    |
//| RESULT:                                                          |
//|     root mean square error.                                      |
//|     Its meaning for regression task is obvious. As for           |
//|     classification task,RMS error means error when estimating    |
//|     posterior probabilities.                                     |
//+------------------------------------------------------------------+
static double CMLPBase::MLPRMSError(CMultilayerPerceptron &network,
                                    CMatrixDouble &xy,const int npoints)
  {
//--- create variables
   int nin=0;
   int nout=0;
   int wcount=0;
//--- function call
   MLPProperties(network,nin,nout,wcount);
//--- return result
   return(MathSqrt(2*MLPError(network,xy,npoints)/(npoints*nout)));
  }
//+------------------------------------------------------------------+
//| Average error on the test set                                    |
//| INPUT PARAMETERS:                                                |
//|     Network -   neural network                                   |
//|     XY      -   test set                                         |
//|     NPoints -   test set size                                    |
//| RESULT:                                                          |
//|     Its meaning for regression task is obvious. As for           |
//|     classification task,it means average error when estimating   |
//|     posterior probabilities.                                     |
//+------------------------------------------------------------------+
static double CMLPBase::MLPAvgError(CMultilayerPerceptron &network,
                                    CMatrixDouble &xy,const int npoints)
  {
//--- create variables
   double result=0;
   int    i=0;
   int    j=0;
   int    k=0;
   int    nin=0;
   int    nout=0;
   int    wcount=0;
   int    i_=0;
//--- function call
   MLPProperties(network,nin,nout,wcount);
//--- calculation
   for(i=0;i<=npoints-1;i++)
     {
      for(i_=0;i_<=nin-1;i_++)
         network.m_x[i_]=xy[i][i_];
      //--- function call
      MLPProcess(network,network.m_x,network.m_y);
      //--- check
      if(MLPIsSoftMax(network))
        {
         //--- class labels
         k=(int)MathRound(xy[i][nin]);
         for(j=0;j<=nout-1;j++)
           {
            //--- check
            if(j==k)
               result=result+MathAbs(1-network.m_y[j]);
            else
               result=result+MathAbs(network.m_y[j]);
           }
        }
      else
        {
         //--- real outputs
         for(j=0;j<=nout-1;j++)
            result=result+MathAbs(xy[i][nin+j]-network.m_y[j]);
        }
     }
//--- return result
   return(result/(npoints*nout));
  }
//+------------------------------------------------------------------+
//| Average relative error on the test set                           |
//| INPUT PARAMETERS:                                                |
//|     Network -   neural network                                   |
//|     XY      -   test set                                         |
//|     NPoints -   test set size                                    |
//| RESULT:                                                          |
//|     Its meaning for regression task is obvious. As for           |
//|     classification task, it means average relative error when    |
//|     estimating posterior probability of belonging to the correct |
//|     class.                                                       |
//+------------------------------------------------------------------+
static double CMLPBase::MLPAvgRelError(CMultilayerPerceptron &network,
                                       CMatrixDouble &xy,const int npoints)
  {
//--- create variables
   double result=0;
   int    i=0;
   int    j=0;
   int    k=0;
   int    lk=0;
   int    nin=0;
   int    nout=0;
   int    wcount=0;
   int    i_=0;
//--- function call
   MLPProperties(network,nin,nout,wcount);
//--- initialization
   result=0;
   k=0;
//--- calculation
   for(i=0;i<=npoints-1;i++)
     {
      for(i_=0;i_<=nin-1;i_++)
         network.m_x[i_]=xy[i][i_];
      //--- function call
      MLPProcess(network,network.m_x,network.m_y);
      //--- check
      if(MLPIsSoftMax(network))
        {
         //--- class labels
         lk=(int)MathRound(xy[i][nin]);
         for(j=0;j<=nout-1;j++)
           {
            //--- check
            if(j==lk)
              {
               result=result+MathAbs(1-network.m_y[j]);
               k=k+1;
              }
           }
        }
      else
        {
         //--- real outputs
         for(j=0;j<=nout-1;j++)
           {
            //--- check
            if(xy[i][nin+j]!=0.0)
              {
               result=result+MathAbs(xy[i][nin+j]-network.m_y[j])/MathAbs(xy[i][nin+j]);
               k=k+1;
              }
           }
        }
     }
//--- check
   if(k!=0)
      result=result/k;
//--- return result
   return(result);
  }
//+------------------------------------------------------------------+
//| Gradient calculation                                             |
//| INPUT PARAMETERS:                                                |
//|     Network -   network initialized with one of the network      |
//|                 creation funcs                                   |
//|     X       -   input vector, length of array must be at least   |
//|                 NIn                                              |
//|     DesiredY-   desired outputs, length of array must be at least|
//|                 NOut                                             |
//|     Grad    -   possibly preallocated array. If size of array is |
//|                 smaller than WCount, it will be reallocated. It  |
//|                 is recommended to reuse previously allocated     |
//|                 array to reduce allocation overhead.             |
//| OUTPUT PARAMETERS:                                               |
//|     E       -   error function, SUM(sqr(y[i]-desiredy[i])/2,i)   |
//|     Grad    -   gradient of E with respect to weights of network,|
//|                 array[WCount]                                    |
//+------------------------------------------------------------------+
static void CMLPBase::MLPGrad(CMultilayerPerceptron &network,double &x[],
                              double &desiredy[],double &e,double &grad[])
  {
//--- create variables
   int i=0;
   int nout=0;
   int ntotal=0;
//--- Alloc
   if(CAp::Len(grad)<network.m_structinfo[4])
      ArrayResizeAL(grad,network.m_structinfo[4]);
//--- Prepare dError/dOut,internal structures
   MLPProcess(network,x,network.m_y);
//--- initialization
   nout=network.m_structinfo[2];
   ntotal=network.m_structinfo[3];
   e=0;
//--- change values
   for(i=0;i<=ntotal-1;i++)
      network.m_derror[i]=0;
   for(i=0;i<=nout-1;i++)
     {
      network.m_derror[ntotal-nout+i]=network.m_y[i]-desiredy[i];
      e=e+CMath::Sqr(network.m_y[i]-desiredy[i])/2;
     }
//--- gradient
   MLPInternalCalculateGradient(network,network.m_neurons,network.m_weights,network.m_derror,grad,false);
  }
//+------------------------------------------------------------------+
//| Gradient calculation (natural error function is used)            |
//| INPUT PARAMETERS:                                                |
//|     Network -   network initialized with one of the network      |
//|                 creation funcs                                   |
//|     X       -   input vector, length of array must be at least   | 
//|                 NIn                                              |
//|     DesiredY-   desired outputs, length of array must be at least| 
//|                 NOut                                             |
//|     Grad    -   possibly preallocated array. If size of array is |
//|                 smaller than WCount, it will be reallocated. It  |
//|                 is recommended to reuse previously allocated     |
//|                 array to reduce allocation overhead.             |
//| OUTPUT PARAMETERS:                                               |
//|     E       -   error function, sum-of-squares for regression    |
//|                 networks, cross-entropy for classification       |
//|                 networks.                                        |
//|     Grad    -   gradient of E with respect to weights of network,| 
//|                 array[WCount]                                    |
//+------------------------------------------------------------------+
static void CMLPBase::MLPGradN(CMultilayerPerceptron &network,double &x[],
                               double &desiredy[],double &e,double &grad[])
  {
//--- create variables
   double s=0;
   int    i=0;
   int    nout=0;
   int    ntotal=0;
//--- initialization
   e=0;
//--- Alloc
   if(CAp::Len(grad)<network.m_structinfo[4])
      ArrayResizeAL(grad,network.m_structinfo[4]);
//--- Prepare dError/dOut,internal structures
   MLPProcess(network,x,network.m_y);
//--- change values
   nout=network.m_structinfo[2];
   ntotal=network.m_structinfo[3];
   for(i=0;i<=ntotal-1;i++)
      network.m_derror[i]=0;
   e=0;
//--- check
   if(network.m_structinfo[6]==0)
     {
      //--- Regression network,least squares
      for(i=0;i<=nout-1;i++)
        {
         network.m_derror[ntotal-nout+i]=network.m_y[i]-desiredy[i];
         e=e+CMath::Sqr(network.m_y[i]-desiredy[i])/2;
        }
     }
   else
     {
      //--- Classification network,cross-entropy
      s=0;
      for(i=0;i<=nout-1;i++)
         s=s+desiredy[i];
      for(i=0;i<=nout-1;i++)
        {
         network.m_derror[ntotal-nout+i]=s*network.m_y[i]-desiredy[i];
         e=e+SafeCrossEntropy(desiredy[i],network.m_y[i]);
        }
     }
//--- gradient
   MLPInternalCalculateGradient(network,network.m_neurons,network.m_weights,network.m_derror,grad,true);
  }
//+------------------------------------------------------------------+
//| Batch gradient calculation for a set of inputs/outputs           |
//| INPUT PARAMETERS:                                                |
//|     Network -   network initialized with one of the network      |
//|                 creation funcs                                   |
//|     XY      -   set of inputs/outputs; one sample = one row;     |
//|                 first NIn columns contain inputs,                |
//|                 next NOut columns - desired outputs.             |
//|     SSize   -   number of elements in XY                         |
//|     Grad    -   possibly preallocated array. If size of array is | 
//|                 smaller than WCount, it will be reallocated. It  | 
//|                 is recommended to reuse previously allocated     |
//|                 array to reduce allocation overhead.             |
//| OUTPUT PARAMETERS:                                               |
//|     E       -   error function, SUM(sqr(y[i]-desiredy[i])/2,i)   |
//|     Grad    -   gradient of E with respect to weights of network,| 
//|                 array[WCount]                                    |
//+------------------------------------------------------------------+
static void CMLPBase::MLPGradBatch(CMultilayerPerceptron &network,
                                   CMatrixDouble &xy,const int ssize,
                                   double &e,double &grad[])
  {
//--- create variables
   int i=0;
   int nin=0;
   int nout=0;
   int wcount=0;
//--- function call
   MLPProperties(network,nin,nout,wcount);
//--- initialization
   for(i=0;i<=wcount-1;i++)
      grad[i]=0;
   e=0;
   i=0;
//--- calculation
   while(i<=ssize-1)
     {
      MLPChunkedGradient(network,xy,i,MathMin(ssize,i+m_chunksize)-i,e,grad,false);
      i=i+m_chunksize;
     }
  }
//+------------------------------------------------------------------+
//| Batch gradient calculation for a set of inputs/outputs           |
//| (natural error function is used)                                 |
//| INPUT PARAMETERS:                                                |
//|     Network -   network initialized with one of the network      |
//|                 creation funcs                                   |
//|     XY      -   set of inputs/outputs; one sample=one row;       |
//|                 first NIn columns contain inputs,                |
//|                 next NOut columns - desired outputs.             |
//|     SSize   -   number of elements in XY                         |
//|     Grad    -   possibly preallocated array. If size of array is |
//|                 smaller than WCount, it will be reallocated. It  |
//|                 is recommended to reuse previously allocated     |
//|                 array to reduce allocation overhead.             |
//| OUTPUT PARAMETERS:                                               |
//|     E       -   error function, sum-of-squares for regression    |
//|                 networks, cross-entropy for classification       |
//|                 networks.                                        |
//|     Grad    -   gradient of E with respect to weights of network,|
//|                 array[WCount]                                    |
//+------------------------------------------------------------------+
static void CMLPBase::MLPGradNBatch(CMultilayerPerceptron &network,
                                    CMatrixDouble &xy,const int ssize,
                                    double &e,double &grad[])
  {
//--- create variables
   int i=0;
   int nin=0;
   int nout=0;
   int wcount=0;
//--- function call
   MLPProperties(network,nin,nout,wcount);
//--- initialization
   for(i=0;i<=wcount-1;i++)
      grad[i]=0;
   e=0;
   i=0;
//--- calculation
   while(i<=ssize-1)
     {
      MLPChunkedGradient(network,xy,i,MathMin(ssize,i+m_chunksize)-i,e,grad,true);
      i=i+m_chunksize;
     }
  }
//+------------------------------------------------------------------+
//| Batch Hessian calculation (natural error function) using         |
//| R-algorithm. Internal subroutine.                                |
//|      Hessian calculation based on R-algorithm described in       |
//|      "Fast Exact Multiplication by the Hessian",                 |
//|      B. A. Pearlmutter,                                          |
//|      Neural Computation, 1994.                                   |
//+------------------------------------------------------------------+
static void CMLPBase::MLPHessianNBatch(CMultilayerPerceptron &network,
                                       CMatrixDouble &xy,const int ssize,
                                       double &e,double &grad[],
                                       CMatrixDouble &h)
  {
//--- initialization
   e=0;
//--- function call
   MLPHessianBatchInternal(network,xy,ssize,true,e,grad,h);
  }
//+------------------------------------------------------------------+
//| Batch Hessian calculation using R-algorithm.                     |
//| Internal subroutine.                                             |
//|      Hessian calculation based on R-algorithm described in       |
//|      "Fast Exact Multiplication by the Hessian",                 |
//|      B. A. Pearlmutter,                                          |
//|      Neural Computation, 1994.                                   |
//+------------------------------------------------------------------+
static void CMLPBase::MLPHessianBatch(CMultilayerPerceptron &network,
                                      CMatrixDouble &xy,const int ssize,
                                      double &e,double &grad[],
                                      CMatrixDouble &h)
  {
//--- initialization
   e=0;
//--- function call
   MLPHessianBatchInternal(network,xy,ssize,false,e,grad,h);
  }
//+------------------------------------------------------------------+
//| Internal subroutine, shouldn't be called by user.                |
//+------------------------------------------------------------------+
static void CMLPBase::MLPInternalProcessVector(int &structinfo[],double &weights[],
                                               double &columnmeans[],
                                               double &columnsigmas[],
                                               double &neurons[],
                                               double &dfdnet[],
                                               double &x[],double &y[])
  {
//--- create variables
   int    i=0;
   int    n1=0;
   int    n2=0;
   int    w1=0;
   int    w2=0;
   int    ntotal=0;
   int    nin=0;
   int    nout=0;
   int    istart=0;
   int    offs=0;
   double net=0;
   double f=0;
   double df=0;
   double d2f=0;
   double mx=0;
   bool   perr;
   int    i_=0;
   int    i1_=0;
//--- Read network geometry
   nin=structinfo[1];
   nout=structinfo[2];
   ntotal=structinfo[3];
   istart=structinfo[5];
//--- Inputs standartisation and putting in the network
   for(i=0;i<=nin-1;i++)
     {
      //--- check
      if(columnsigmas[i]!=0.0)
         neurons[i]=(x[i]-columnmeans[i])/columnsigmas[i];
      else
         neurons[i]=x[i]-columnmeans[i];
     }
//--- Process network
   for(i=0;i<=ntotal-1;i++)
     {
      offs=istart+i*m_nfieldwidth;
      //--- check
      if(structinfo[offs+0]>0 || structinfo[offs+0]==-5)
        {
         //--- Activation function
         MLPActivationFunction(neurons[structinfo[offs+2]],structinfo[offs+0],f,df,d2f);
         //--- change values
         neurons[i]=f;
         dfdnet[i]=df;
         continue;
        }
      //--- check
      if(structinfo[offs+0]==0)
        {
         //--- Adaptive summator
         n1=structinfo[offs+2];
         n2=n1+structinfo[offs+1]-1;
         w1=structinfo[offs+3];
         w2=w1+structinfo[offs+1]-1;
         i1_=(n1)-(w1);
         net=0.0;
         //--- calculation
         for(i_=w1;i_<=w2;i_++)
            net+=weights[i_]*neurons[i_+i1_];
         neurons[i]=net;
         dfdnet[i]=1.0;
         continue;
        }
      //--- check
      if(structinfo[offs+0]<0)
        {
         perr=true;
         //--- check
         if(structinfo[offs+0]==-2)
           {
            //--- input neuron,left unchanged
            perr=false;
           }
         //--- check
         if(structinfo[offs+0]==-3)
           {
            //--- "-1" neuron
            neurons[i]=-1;
            perr=false;
           }
         //--- check
         if(structinfo[offs+0]==-4)
           {
            //--- "0" neuron
            neurons[i]=0;
            perr=false;
           }
         //--- check
         if(!CAp::Assert(!perr,__FUNCTION__+": internal error - unknown neuron type!"))
            return;
         continue;
        }
     }
//--- Extract result
   i1_=ntotal-nout;
   for(i_=0;i_<=nout-1;i_++)
      y[i_]=neurons[i_+i1_];
//--- Softmax post-processing or standardisation if needed
   if(!CAp::Assert(structinfo[6]==0 || structinfo[6]==1,__FUNCTION__+": unknown normalization type!"))
      return;
//--- check
   if(structinfo[6]==1)
     {
      //--- Softmax
      mx=y[0];
      for(i=1;i<=nout-1;i++)
         mx=MathMax(mx,y[i]);
      //--- calculation
      net=0;
      for(i=0;i<=nout-1;i++)
        {
         y[i]=MathExp(y[i]-mx);
         net=net+y[i];
        }
      for(i=0;i<=nout-1;i++)
         y[i]=y[i]/net;
     }
   else
     {
      //--- Standardisation
      for(i=0;i<=nout-1;i++)
         y[i]=y[i]*columnsigmas[nin+i]+columnmeans[nin+i];
     }
  }
//+------------------------------------------------------------------+
//| Serializer: allocation                                           |
//+------------------------------------------------------------------+
static void CMLPBase::MLPAlloc(CSerializer &s,CMultilayerPerceptron &network)
  {
//--- create variables
   int    i=0;
   int    j=0;
   int    k=0;
   int    fkind=0;
   double threshold=0;
   double v0=0;
   double v1=0;
   int    nin=0;
   int    nout=0;
//--- initialization
   nin=network.m_hllayersizes[0];
   nout=network.m_hllayersizes[CAp::Len(network.m_hllayersizes)-1];
//--- preparation to serialize
   s.Alloc_Entry();
   s.Alloc_Entry();
   s.Alloc_Entry();
//--- function call
   CApServ::AllocIntegerArray(s,network.m_hllayersizes,-1);
   for(i=1;i<=CAp::Len(network.m_hllayersizes)-1;i++)
     {
      for(j=0;j<=network.m_hllayersizes[i]-1;j++)
        {
         //--- function call
         MLPGetNeuronInfo(network,i,j,fkind,threshold);
         //--- preparation to serialize
         s.Alloc_Entry();
         s.Alloc_Entry();
         for(k=0;k<=network.m_hllayersizes[i-1]-1;k++)
            s.Alloc_Entry();
        }
     }
   for(j=0;j<=nin-1;j++)
     {
      //--- function call
      MLPGetInputScaling(network,j,v0,v1);
      //--- preparation to serialize
      s.Alloc_Entry();
      s.Alloc_Entry();
     }
   for(j=0;j<=nout-1;j++)
     {
      //--- function call
      MLPGetOutputScaling(network,j,v0,v1);
      //--- preparation to serialize
      s.Alloc_Entry();
      s.Alloc_Entry();
     }
  }
//+------------------------------------------------------------------+
//| Serializer: serialization                                        |
//+------------------------------------------------------------------+
static void CMLPBase::MLPSerialize(CSerializer &s,CMultilayerPerceptron &network)
  {
//--- create variables
   int    i=0;
   int    j=0;
   int    k=0;
   int    fkind=0;
   double threshold=0;
   double v0=0;
   double v1=0;
   int    nin=0;
   int    nout=0;
//--- change values
   nin=network.m_hllayersizes[0];
   nout=network.m_hllayersizes[CAp::Len(network.m_hllayersizes)-1];
//--- serializetion
   s.Serialize_Int(CSCodes::GetMLPSerializationCode());
   s.Serialize_Int(m_mlpfirstversion);
   s.Serialize_Bool(MLPIsSoftMax(network));
//--- function call
   CApServ::SerializeIntegerArray(s,network.m_hllayersizes,-1);
   for(i=1;i<=CAp::Len(network.m_hllayersizes)-1;i++)
     {
      for(j=0;j<=network.m_hllayersizes[i]-1;j++)
        {
         //--- function call
         MLPGetNeuronInfo(network,i,j,fkind,threshold);
         //--- serializetion
         s.Serialize_Int(fkind);
         s.Serialize_Double(threshold);
         for(k=0;k<=network.m_hllayersizes[i-1]-1;k++)
            s.Serialize_Double(MLPGetWeight(network,i-1,k,i,j));
        }
     }
   for(j=0;j<=nin-1;j++)
     {
      //--- function call
      MLPGetInputScaling(network,j,v0,v1);
      //--- serializetion
      s.Serialize_Double(v0);
      s.Serialize_Double(v1);
     }
   for(j=0;j<=nout-1;j++)
     {
      //--- function call
      MLPGetOutputScaling(network,j,v0,v1);
      //--- serializetion
      s.Serialize_Double(v0);
      s.Serialize_Double(v1);
     }
  }
//+------------------------------------------------------------------+
//| Serializer: unserialization                                      |
//+------------------------------------------------------------------+
static void CMLPBase::MLPUnserialize(CSerializer &s,CMultilayerPerceptron &network)
  {
//--- create variables
   int    i0=0;
   int    i1=0;
   int    i=0;
   int    j=0;
   int    k=0;
   int    fkind=0;
   double threshold=0;
   double v0=0;
   double v1=0;
   int    nin=0;
   int    nout=0;
   bool   issoftmax;
//--- create array
   int    layersizes[];
//--- check correctness of header
   i0=s.Unserialize_Int();
//--- check
   if(!CAp::Assert(i0==CSCodes::GetMLPSerializationCode(),__FUNCTION__+": stream header corrupted"))
      return;
//--- unserializetion
   i1=s.Unserialize_Int();
//--- check
   if(!CAp::Assert(i1==m_mlpfirstversion,__FUNCTION__+": stream header corrupted"))
      return;
//--- Create network
   issoftmax=s.Unserialize_Bool();
//--- function call
   CApServ::UnserializeIntegerArray(s,layersizes);
//--- check
   if(!CAp::Assert((CAp::Len(layersizes)==2 || CAp::Len(layersizes)==3) || CAp::Len(layersizes)==4,__FUNCTION__+": too many hidden layers!"))
      return;
//--- change values
   nin=layersizes[0];
   nout=layersizes[CAp::Len(layersizes)-1];
//--- check
   if(CAp::Len(layersizes)==2)
     {
      //--- check
      if(issoftmax)
         MLPCreateC0(layersizes[0],layersizes[1],network);
      else
         MLPCreate0(layersizes[0],layersizes[1],network);
     }
//--- check
   if(CAp::Len(layersizes)==3)
     {
      //--- check
      if(issoftmax)
         MLPCreateC1(layersizes[0],layersizes[1],layersizes[2],network);
      else
         MLPCreate1(layersizes[0],layersizes[1],layersizes[2],network);
     }
//--- check
   if(CAp::Len(layersizes)==4)
     {
      //--- check
      if(issoftmax)
         MLPCreateC2(layersizes[0],layersizes[1],layersizes[2],layersizes[3],network);
      else
         MLPCreate2(layersizes[0],layersizes[1],layersizes[2],layersizes[3],network);
     }
//--- Load neurons and weights
   for(i=1;i<=CAp::Len(layersizes)-1;i++)
     {
      for(j=0;j<=layersizes[i]-1;j++)
        {
         //--- unserializetion
         fkind=s.Unserialize_Int();
         threshold=s.Unserialize_Double();
         //--- function call
         MLPSetNeuronInfo(network,i,j,fkind,threshold);
         //--- unserializetion
         for(k=0;k<=layersizes[i-1]-1;k++)
           {
            v0=s.Unserialize_Double();
            //--- function call
            MLPSetWeight(network,i-1,k,i,j,v0);
           }
        }
     }

//
//--- Load standartizator
//
   for(j=0;j<=nin-1;j++)
     {
      //--- unserializetion
      v0=s.Unserialize_Double();
      v1=s.Unserialize_Double();
      //--- function call
      MLPSetInputScaling(network,j,v0,v1);
     }
   for(j=0;j<=nout-1;j++)
     {
      //--- unserializetion
      v0=s.Unserialize_Double();
      v1=s.Unserialize_Double();
      //--- function call
      MLPSetOutputScaling(network,j,v0,v1);
     }
  }
//+------------------------------------------------------------------+
//| Internal subroutine: adding new input layer to network           |
//+------------------------------------------------------------------+
static void CMLPBase::AddInputLayer(const int ncount,int &lsizes[],
                                    int &ltypes[],int &lconnfirst[],
                                    int &lconnlast[],int &lastproc)
  {
//--- change values
   lsizes[0]=ncount;
   ltypes[0]=-2;
   lconnfirst[0]=0;
   lconnlast[0]=0;
   lastproc=0;
  }
//+------------------------------------------------------------------+
//| Internal subroutine: adding new summator layer to network        |
//+------------------------------------------------------------------+
static void CMLPBase::AddBiasedSummatorLayer(const int ncount,int &lsizes[],
                                             int &ltypes[],int &lconnfirst[],
                                             int &lconnlast[],int &lastproc)
  {
//--- change values
   lsizes[lastproc+1]=1;
   ltypes[lastproc+1]=-3;
   lconnfirst[lastproc+1]=0;
   lconnlast[lastproc+1]=0;
   lsizes[lastproc+2]=ncount;
   ltypes[lastproc+2]=0;
   lconnfirst[lastproc+2]=lastproc;
   lconnlast[lastproc+2]=lastproc+1;
   lastproc=lastproc+2;
  }
//+------------------------------------------------------------------+
//| Internal subroutine: adding new summator layer to network        |
//+------------------------------------------------------------------+
static void CMLPBase::AddActivationLayer(const int functype,int &lsizes[],
                                         int &ltypes[],int &lconnfirst[],
                                         int &lconnlast[],int &lastproc)
  {
//--- check
   if(!CAp::Assert(functype>0 || functype==-5,__FUNCTION__+": incorrect function type"))
      return;
//--- change values
   lsizes[lastproc+1]=lsizes[lastproc];
   ltypes[lastproc+1]=functype;
   lconnfirst[lastproc+1]=lastproc;
   lconnlast[lastproc+1]=lastproc;
   lastproc=lastproc+1;
  }
//+------------------------------------------------------------------+
//| Internal subroutine: adding new zero layer to network            |
//+------------------------------------------------------------------+
static void CMLPBase::AddZeroLayer(int &lsizes[],int &ltypes[],
                                   int &lconnfirst[],int &lconnlast[],
                                   int &lastproc)
  {
//--- change values
   lsizes[lastproc+1]=1;
   ltypes[lastproc+1]=-4;
   lconnfirst[lastproc+1]=0;
   lconnlast[lastproc+1]=0;
   lastproc=lastproc+1;
  }
//+------------------------------------------------------------------+
//| This routine adds input layer to the high-level description of   |
//| the network.                                                     |
//| It modifies Network.HLConnections and Network.HLNeurons and      |
//| assumes that these arrays have enough place to store data.       |
//| It accepts following parameters:                                 |
//|     Network     -   network                                      |
//|     ConnIdx     -   index of the first free entry in the         |
//|                     HLConnections                                |
//|     NeuroIdx    -   index of the first free entry in the         |
//|                     HLNeurons                                    |
//|     StructInfoIdx-  index of the first entry in the low level    |
//|                     description of the current layer (in the     |
//|                     StructInfo array)                            |
//|     NIn         -   number of inputs                             |
//| It modified Network and indices.                                 |
//+------------------------------------------------------------------+
static void CMLPBase::HLAddInputLayer(CMultilayerPerceptron &network,
                                      int &connidx,int &neuroidx,
                                      int &structinfoidx,int nin)
  {
//--- create variables
   int i=0;
   int offs=0;
//--- initialization
   offs=m_hlm_nfieldwidth*neuroidx;
//--- change values
   for(i=0;i<=nin-1;i++)
     {
      network.m_hlneurons[offs+0]=0;
      network.m_hlneurons[offs+1]=i;
      network.m_hlneurons[offs+2]=-1;
      network.m_hlneurons[offs+3]=-1;
      offs=offs+m_hlm_nfieldwidth;
     }
//--- change values
   neuroidx=neuroidx+nin;
   structinfoidx=structinfoidx+nin;
  }
//+------------------------------------------------------------------+
//| This routine adds output layer to the high-level description of  |
//| the network.                                                     |
//| It modifies Network.HLConnections and Network. HLNeurons and     |
//| assumes that these arrays have enough place to store data. It    |
//| accepts following parameters:                                    |
//|     Network     -   network                                      |
//|     ConnIdx     -   index of the first free entry in the         |
//|                     HLConnections                                |
//|     NeuroIdx    -   index of the first free entry in the         |
//|                     HLNeurons                                    |
//|     StructInfoIdx-  index of the first entry in the low level    |
//|                     description of the current layer (in the     |
//|                     StructInfo array)                            |
//|     WeightsIdx  -   index of the first entry in the Weights      |
//|                     array which corresponds to the current layer |
//|     K           -   current layer index                          |
//|     NPrev       -   number of neurons in the previous layer      |
//|     NOut        -   number of outputs                            |
//|     IsCls       -   is it classifier network?                    |
//|     IsLinear    -   is it network with linear output?            |
//| It modified Network and ConnIdx/NeuroIdx/StructInfoIdx/WeightsIdx|
//+------------------------------------------------------------------+
static void CMLPBase::HLAddOutputLayer(CMultilayerPerceptron &network,
                                       int &connidx,int &neuroidx,
                                       int &structinfoidx,int &weightsidx,
                                       const int k,const int nprev,
                                       const int nout,const bool iscls,
                                       const bool islinearout)
  {
//--- create variables
   int i=0;
   int j=0;
   int neurooffs=0;
   int connoffs=0;
//--- check
   if(!CAp::Assert((iscls && islinearout) || !iscls,__FUNCTION__+": internal error"))
      return;
//--- initialization
   neurooffs=m_hlm_nfieldwidth*neuroidx;
   connoffs=m_hlconm_nfieldwidth*connidx;
//--- check
   if(!iscls)
     {
      //--- Regression network
      for(i=0;i<=nout-1;i++)
        {
         //--- change values
         network.m_hlneurons[neurooffs+0]=k;
         network.m_hlneurons[neurooffs+1]=i;
         network.m_hlneurons[neurooffs+2]=structinfoidx+1+nout+i;
         network.m_hlneurons[neurooffs+3]=weightsidx+nprev+(nprev+1)*i;
         neurooffs=neurooffs+m_hlm_nfieldwidth;
        }
      for(i=0;i<=nprev-1;i++)
        {
         for(j=0;j<=nout-1;j++)
           {
            //--- change values
            network.m_hlconnections[connoffs+0]=k-1;
            network.m_hlconnections[connoffs+1]=i;
            network.m_hlconnections[connoffs+2]=k;
            network.m_hlconnections[connoffs+3]=j;
            network.m_hlconnections[connoffs+4]=weightsidx+i+j*(nprev+1);
            connoffs=connoffs+m_hlconm_nfieldwidth;
           }
        }
      //--- change values
      connidx=connidx+nprev*nout;
      neuroidx=neuroidx+nout;
      structinfoidx=structinfoidx+2*nout+1;
      weightsidx=weightsidx+nout*(nprev+1);
     }
   else
     {
      //--- Classification network
      for(i=0;i<=nout-2;i++)
        {
         //--- change values
         network.m_hlneurons[neurooffs+0]=k;
         network.m_hlneurons[neurooffs+1]=i;
         network.m_hlneurons[neurooffs+2]=-1;
         network.m_hlneurons[neurooffs+3]=weightsidx+nprev+(nprev+1)*i;
         neurooffs=neurooffs+m_hlm_nfieldwidth;
        }
      //--- change values
      network.m_hlneurons[neurooffs+0]=k;
      network.m_hlneurons[neurooffs+1]=i;
      network.m_hlneurons[neurooffs+2]=-1;
      network.m_hlneurons[neurooffs+3]=-1;
      for(i=0;i<=nprev-1;i++)
        {
         for(j=0;j<=nout-2;j++)
           {
            //--- change values
            network.m_hlconnections[connoffs+0]=k-1;
            network.m_hlconnections[connoffs+1]=i;
            network.m_hlconnections[connoffs+2]=k;
            network.m_hlconnections[connoffs+3]=j;
            network.m_hlconnections[connoffs+4]=weightsidx+i+j*(nprev+1);
            connoffs=connoffs+m_hlconm_nfieldwidth;
           }
        }
      //--- change values
      connidx=connidx+nprev*(nout-1);
      neuroidx=neuroidx+nout;
      structinfoidx=structinfoidx+nout+2;
      weightsidx=weightsidx+(nout-1)*(nprev+1);
     }
  }
//+------------------------------------------------------------------+
//| This routine adds hidden layer to the high-level description of  |
//| the network.                                                     |
//| It modifies Network.HLConnections and Network.HLNeurons and      |
//| assumes that these arrays have enough place to store data. It    | 
//| accepts following parameters:                                    |
//|     Network     -   network                                      |
//|     ConnIdx     -   index of the first free entry in the         |
//|                     HLConnections                                |
//|     NeuroIdx    -   index of the first free entry in the         |
//|                     HLNeurons                                    |
//|     StructInfoIdx-  index of the first entry in the low level    |
//|                     description of the current layer (in the     |
//|                     StructInfo array)                            |
//|     WeightsIdx  -   index of the first entry in the Weights      |
//|                     array which corresponds to the current layer |
//|     K           -   current layer index                          |
//|     NPrev       -   number of neurons in the previous layer      |
//|     NCur        -   number of neurons in the current layer       |
//| It modified Network and ConnIdx/NeuroIdx/StructInfoIdx/WeightsIdx|
//+------------------------------------------------------------------+
static void CMLPBase::HLAddHiddenLayer(CMultilayerPerceptron &network,
                                       int &connidx,int &neuroidx,
                                       int &structinfoidx,int &weightsidx,
                                       const int k,const int nprev,
                                       const int ncur)
  {
//--- create variables
   int i=0;
   int j=0;
   int neurooffs=0;
   int connoffs=0;
//--- change values
   neurooffs=m_hlm_nfieldwidth*neuroidx;
   connoffs=m_hlconm_nfieldwidth*connidx;
   for(i=0;i<=ncur-1;i++)
     {
      //--- change values
      network.m_hlneurons[neurooffs+0]=k;
      network.m_hlneurons[neurooffs+1]=i;
      network.m_hlneurons[neurooffs+2]=structinfoidx+1+ncur+i;
      network.m_hlneurons[neurooffs+3]=weightsidx+nprev+(nprev+1)*i;
      neurooffs=neurooffs+m_hlm_nfieldwidth;
     }
   for(i=0;i<=nprev-1;i++)
     {
      for(j=0;j<=ncur-1;j++)
        {
         //--- change values
         network.m_hlconnections[connoffs+0]=k-1;
         network.m_hlconnections[connoffs+1]=i;
         network.m_hlconnections[connoffs+2]=k;
         network.m_hlconnections[connoffs+3]=j;
         network.m_hlconnections[connoffs+4]=weightsidx+i+j*(nprev+1);
         connoffs=connoffs+m_hlconm_nfieldwidth;
        }
     }
//--- change values
   connidx=connidx+nprev*ncur;
   neuroidx=neuroidx+ncur;
   structinfoidx=structinfoidx+2*ncur+1;
   weightsidx=weightsidx+ncur*(nprev+1);
  }
//+------------------------------------------------------------------+
//| This function fills high level information about network created | 
//| using internal MLPCreate() function.                             |
//| This function does NOT examine StructInfo for low level          | 
//| information, it just expects that network has following          |
//| structure:                                                       |
//|     input neuron            \                                    |
//|     ...                      | input layer                       |
//|     input neuron            /                                    |
//|     "-1" neuron             \                                    |
//|     biased summator          |                                   |
//|     ...                      |                                   |
//|     biased summator          | hidden layer(s), if there are     |
//|     activation function      | exists any                        |
//|     ...                      |                                   |
//|     activation function     /                                    |
//|     "-1" neuron            \                                     |
//|     biased summator         | output layer:                      |
//|     ...                     | * we have NOut summators/activators|
//|     biased summator         |   for regression networks          |
//|     activation function     | * we have only NOut-1 summators and| 
//|     ...                     |   no activators for classifiers    |
//|     activation function     | * we have "0" neuron only when we  |
//|     "0" neuron              /   have classifier                  |
//+------------------------------------------------------------------+
static void CMLPBase::FillHighLevelInformation(CMultilayerPerceptron &network,
                                               const int nin,const int nhid1,
                                               const int nhid2,const int nout,
                                               const bool iscls,const bool islinearout)
  {
//--- create variables
   int idxweights=0;
   int idxstruct=0;
   int idxneuro=0;
   int idxconn=0;
//--- check
   if(!CAp::Assert((iscls && islinearout) || !iscls,__FUNCTION__+": internal error"))
      return;
//--- Preparations common to all types of networks
   idxweights=0;
   idxneuro=0;
   idxstruct=0;
   idxconn=0;
   network.m_hlnetworktype=0;
//--- network without hidden layers
   if(nhid1==0)
     {
      //--- allocation
      ArrayResizeAL(network.m_hllayersizes,2);
      //--- change values
      network.m_hllayersizes[0]=nin;
      network.m_hllayersizes[1]=nout;
      //--- check
      if(!iscls)
        {
         //--- allocation
         ArrayResizeAL(network.m_hlconnections,m_hlconm_nfieldwidth*nin*nout);
         ArrayResizeAL(network.m_hlneurons,m_hlm_nfieldwidth*(nin+nout));
         network.m_hlnormtype=0;
        }
      else
        {
         //--- allocation
         ArrayResizeAL(network.m_hlconnections,m_hlconm_nfieldwidth*nin*(nout-1));
         ArrayResizeAL(network.m_hlneurons,m_hlm_nfieldwidth*(nin+nout));
         network.m_hlnormtype=1;
        }
      //--- function call
      HLAddInputLayer(network,idxconn,idxneuro,idxstruct,nin);
      //--- function call
      HLAddOutputLayer(network,idxconn,idxneuro,idxstruct,idxweights,1,nin,nout,iscls,islinearout);
      //--- exit the function
      return;
     }
//--- network with one hidden layers
   if(nhid2==0)
     {
      //--- allocation
      ArrayResizeAL(network.m_hllayersizes,3);
      //--- change values
      network.m_hllayersizes[0]=nin;
      network.m_hllayersizes[1]=nhid1;
      network.m_hllayersizes[2]=nout;
      //--- check
      if(!iscls)
        {
         //--- allocation
         ArrayResizeAL(network.m_hlconnections,m_hlconm_nfieldwidth*(nin*nhid1+nhid1*nout));
         ArrayResizeAL(network.m_hlneurons,m_hlm_nfieldwidth*(nin+nhid1+nout));
         network.m_hlnormtype=0;
        }
      else
        {
         //--- allocation
         ArrayResizeAL(network.m_hlconnections,m_hlconm_nfieldwidth*(nin*nhid1+nhid1*(nout-1)));
         ArrayResizeAL(network.m_hlneurons,m_hlm_nfieldwidth*(nin+nhid1+nout));
         network.m_hlnormtype=1;
        }
      //--- function call
      HLAddInputLayer(network,idxconn,idxneuro,idxstruct,nin);
      //--- function call
      HLAddHiddenLayer(network,idxconn,idxneuro,idxstruct,idxweights,1,nin,nhid1);
      //--- function call
      HLAddOutputLayer(network,idxconn,idxneuro,idxstruct,idxweights,2,nhid1,nout,iscls,islinearout);
      //--- exit the function
      return;
     }
//--- Two hidden layers
   ArrayResizeAL(network.m_hllayersizes,4);
//--- change values
   network.m_hllayersizes[0]=nin;
   network.m_hllayersizes[1]=nhid1;
   network.m_hllayersizes[2]=nhid2;
   network.m_hllayersizes[3]=nout;
//--- check
   if(!iscls)
     {
      //--- allocation
      ArrayResizeAL(network.m_hlconnections,m_hlconm_nfieldwidth*(nin*nhid1+nhid1*nhid2+nhid2*nout));
      ArrayResizeAL(network.m_hlneurons,m_hlm_nfieldwidth*(nin+nhid1+nhid2+nout));
      network.m_hlnormtype=0;
     }
   else
     {
      //--- allocation
      ArrayResizeAL(network.m_hlconnections,m_hlconm_nfieldwidth*(nin*nhid1+nhid1*nhid2+nhid2*(nout-1)));
      ArrayResizeAL(network.m_hlneurons,m_hlm_nfieldwidth*(nin+nhid1+nhid2+nout));
      network.m_hlnormtype=1;
     }
//--- function call
   HLAddInputLayer(network,idxconn,idxneuro,idxstruct,nin);
//--- function call
   HLAddHiddenLayer(network,idxconn,idxneuro,idxstruct,idxweights,1,nin,nhid1);
//--- function call
   HLAddHiddenLayer(network,idxconn,idxneuro,idxstruct,idxweights,2,nhid1,nhid2);
//--- function call
   HLAddOutputLayer(network,idxconn,idxneuro,idxstruct,idxweights,3,nhid2,nout,iscls,islinearout);
  }
//+------------------------------------------------------------------+
//| Internal subroutine.                                             |
//+------------------------------------------------------------------+
static void CMLPBase::MLPCreate(const int nin,const int nout,int &lsizes[],
                                int &ltypes[],int &lconnfirst[],int &lconnlast[],
                                const int layerscount,const bool isclsnet,
                                CMultilayerPerceptron &network)
  {
//--- create variables
   int i=0;
   int j=0;
   int ssize=0;
   int ntotal=0;
   int wcount=0;
   int offs=0;
   int nprocessed=0;
   int wallocated=0;
//--- creating arrays
   int localtemp[];
   int lnfirst[];
   int lnsyn[];
//--- Check
   if(!CAp::Assert(layerscount>0,__FUNCTION__+": wrong parameters!"))
      return;
//--- check
   if(!CAp::Assert(ltypes[0]==-2,__FUNCTION__+": wrong LTypes[0] (must be -2)!"))
      return;
   for(i=0;i<=layerscount-1;i++)
     {
      //--- check
      if(!CAp::Assert(lsizes[i]>0,__FUNCTION__+": wrong LSizes!"))
         return;
      //--- check
      if(!CAp::Assert(lconnfirst[i]>=0 &&(lconnfirst[i]<i || i==0),__FUNCTION__+": wrong LConnFirst!"))
         return;
      //--- check
      if(!CAp::Assert(lconnlast[i]>=lconnfirst[i]&&(lconnlast[i]<i || i==0),__FUNCTION__+": wrong LConnLast!"))
         return;
     }
//--- Build network geometry
   ArrayResizeAL(lnfirst,layerscount);
   ArrayResizeAL(lnsyn,layerscount);
//--- initialization
   ntotal=0;
   wcount=0;
//--- calculation
   for(i=0;i<=layerscount-1;i++)
     {
      //--- Analyze connections.
      //--- This code must throw an assertion in case of unknown LTypes[I]
      lnsyn[i]=-1;
      //--- check
      if(ltypes[i]>=0 || ltypes[i]==-5)
        {
         lnsyn[i]=0;
         for(j=lconnfirst[i];j<=lconnlast[i];j++)
            lnsyn[i]=lnsyn[i]+lsizes[j];
        }
      else
        {
         //--- check
         if((ltypes[i]==-2 || ltypes[i]==-3) || ltypes[i]==-4)
            lnsyn[i]=0;
        }
      //--- check
      if(!CAp::Assert(lnsyn[i]>=0,__FUNCTION__+": internal error #0!"))
         return;
      //--- Other info
      lnfirst[i]=ntotal;
      ntotal=ntotal+lsizes[i];
      //--- check
      if(ltypes[i]==0)
         wcount=wcount+lnsyn[i]*lsizes[i];
     }
   ssize=7+ntotal*m_nfieldwidth;
//--- Allocate
   ArrayResizeAL(network.m_structinfo,ssize);
   ArrayResizeAL(network.m_weights,wcount);
//--- check
   if(isclsnet)
     {
      //--- allocation
      ArrayResizeAL(network.m_columnmeans,nin);
      ArrayResizeAL(network.m_columnsigmas,nin);
     }
   else
     {
      //--- allocation
      ArrayResizeAL(network.m_columnmeans,nin+nout);
      ArrayResizeAL(network.m_columnsigmas,nin+nout);
     }
//--- allocation
   ArrayResizeAL(network.m_neurons,ntotal);
   network.m_chunks.Resize(3*ntotal+1,m_chunksize);
   ArrayResizeAL(network.m_nwbuf,MathMax(wcount,2*nout));
   ArrayResizeAL(network.m_integerbuf,4);
   ArrayResizeAL(network.m_dfdnet,ntotal);
   ArrayResizeAL(network.m_x,nin);
   ArrayResizeAL(network.m_y,nout);
   ArrayResizeAL(network.m_derror,ntotal);
//--- Fill structure: global info
   network.m_structinfo[0]=ssize;
   network.m_structinfo[1]=nin;
   network.m_structinfo[2]=nout;
   network.m_structinfo[3]=ntotal;
   network.m_structinfo[4]=wcount;
   network.m_structinfo[5]=7;
//--- check
   if(isclsnet)
      network.m_structinfo[6]=1;
   else
      network.m_structinfo[6]=0;
//--- Fill structure: neuron connections
   nprocessed=0;
   wallocated=0;
//--- calculation
   for(i=0;i<=layerscount-1;i++)
     {
      for(j=0;j<=lsizes[i]-1;j++)
        {
         offs=network.m_structinfo[5]+nprocessed*m_nfieldwidth;
         network.m_structinfo[offs+0]=ltypes[i];
         //--- check
         if(ltypes[i]==0)
           {
            //--- Adaptive summator:
            //--- * connections with weights to previous neurons
            network.m_structinfo[offs+1]=lnsyn[i];
            network.m_structinfo[offs+2]=lnfirst[lconnfirst[i]];
            network.m_structinfo[offs+3]=wallocated;
            wallocated=wallocated+lnsyn[i];
            nprocessed=nprocessed+1;
           }
         //--- check
         if(ltypes[i]>0 || ltypes[i]==-5)
           {
            //--- Activation layer:
            //--- * each neuron connected to one (only one) of previous neurons.
            //--- * no weights
            network.m_structinfo[offs+1]=1;
            network.m_structinfo[offs+2]=lnfirst[lconnfirst[i]]+j;
            network.m_structinfo[offs+3]=-1;
            nprocessed=nprocessed+1;
           }
         //--- check
         if((ltypes[i]==-2 || ltypes[i]==-3) || ltypes[i]==-4)
            nprocessed=nprocessed+1;
        }
     }
//--- check
   if(!CAp::Assert(wallocated==wcount,__FUNCTION__+": internal error #1!"))
      return;
//--- check
   if(!CAp::Assert(nprocessed==ntotal,__FUNCTION__+": internal error #2!"))
      return;
//--- Fill weights by small random values
//--- Initialize means and sigmas
   for(i=0;i<=wcount-1;i++)
      network.m_weights[i]=CMath::RandomReal()-0.5;
   for(i=0;i<=nin-1;i++)
     {
      network.m_columnmeans[i]=0;
      network.m_columnsigmas[i]=1;
     }
//--- check
   if(!isclsnet)
     {
      for(i=0;i<=nout-1;i++)
        {
         network.m_columnmeans[nin+i]=0;
         network.m_columnsigmas[nin+i]=1;
        }
     }
  }
//+------------------------------------------------------------------+
//| Internal subroutine for Hessian calculation.                     |
//| WARNING!!! Unspeakable math far beyong human capabilities :)     |
//+------------------------------------------------------------------+
static void CMLPBase::MLPHessianBatchInternal(CMultilayerPerceptron &network,
                                              CMatrixDouble &xy,const int ssize,
                                              const bool naturalerr,double &e,
                                              double &grad[],CMatrixDouble &h)
  {
//--- create variables
   int    nin=0;
   int    nout=0;
   int    wcount=0;
   int    ntotal=0;
   int    istart=0;
   int    i=0;
   int    j=0;
   int    k=0;
   int    kl=0;
   int    offs=0;
   int    n1=0;
   int    n2=0;
   int    w1=0;
   int    w2=0;
   double s=0;
   double t=0;
   double v=0;
   double et=0;
   bool   bflag;
   double f=0;
   double df=0;
   double d2f=0;
   double deidyj=0;
   double mx=0;
   double q=0;
   double z=0;
   double s2=0;
   double expi=0;
   double expj=0;
   int    i_=0;
   int    i1_=0;
//--- creating arrays
   double x[];
   double desiredy[];
   double gt[];
   double zeros[];
//--- create matrix
   CMatrixDouble rx;
   CMatrixDouble ry;
   CMatrixDouble rdx;
   CMatrixDouble rdy;
//--- initialization
   e=0;
//--- function call
   MLPProperties(network,nin,nout,wcount);
//--- initialization
   ntotal=network.m_structinfo[3];
   istart=network.m_structinfo[5];
//--- Prepare
   ArrayResizeAL(x,nin);
   ArrayResizeAL(desiredy,nout);
   ArrayResizeAL(zeros,wcount);
   ArrayResizeAL(gt,wcount);
   rx.Resize(ntotal+nout,wcount);
   ry.Resize(ntotal+nout,wcount);
   rdx.Resize(ntotal+nout,wcount);
   rdy.Resize(ntotal+nout,wcount);
//--- initialization
   e=0;
   for(i=0;i<=wcount-1;i++)
      zeros[i]=0;
   for(i_=0;i_<=wcount-1;i_++)
      grad[i_]=zeros[i_];
   for(i=0;i<=wcount-1;i++)
     {
      for(i_=0;i_<=wcount-1;i_++)
         h[i].Set(i_,zeros[i_]);
     }
//--- Process
   for(k=0;k<=ssize-1;k++)
     {
      //--- Process vector with MLPGradN.
      //--- Now Neurons,DFDNET and DError contains results of the last run.
      for(i_=0;i_<=nin-1;i_++)
         x[i_]=xy[k][i_];
      //--- check
      if(MLPIsSoftMax(network))
        {
         //--- class labels outputs
         kl=(int)MathRound(xy[k][nin]);
         for(i=0;i<=nout-1;i++)
           {
            //--- check
            if(i==kl)
               desiredy[i]=1;
            else
               desiredy[i]=0;
           }
        }
      else
        {
         //--- real outputs
         i1_=nin;
         for(i_=0;i_<=nout-1;i_++)
            desiredy[i_]=xy[k][i_+i1_];
        }
      //--- check
      if(naturalerr)
         MLPGradN(network,x,desiredy,et,gt);
      else
         MLPGrad(network,x,desiredy,et,gt);
      //--- grad,error
      e=e+et;
      for(i_=0;i_<=wcount-1;i_++)
         grad[i_]=grad[i_]+gt[i_];
      //--- Hessian.
      //--- Forward pass of the R-algorithm
      for(i=0;i<=ntotal-1;i++)
        {
         offs=istart+i*m_nfieldwidth;
         for(i_=0;i_<=wcount-1;i_++)
            rx[i].Set(i_,zeros[i_]);
         for(i_=0;i_<=wcount-1;i_++)
            ry[i].Set(i_,zeros[i_]);
         //--- check
         if(network.m_structinfo[offs+0]>0 || network.m_structinfo[offs+0]==-5)
           {
            //--- Activation function
            n1=network.m_structinfo[offs+2];
            for(i_=0;i_<=wcount-1;i_++)
               rx[i].Set(i_,ry[n1][i_]);
            //--- calculation
            v=network.m_dfdnet[i];
            for(i_=0;i_<=wcount-1;i_++)
               ry[i].Set(i_,v*rx[i][i_]);
            continue;
           }
         //--- check
         if(network.m_structinfo[offs+0]==0)
           {
            //--- Adaptive summator
            n1=network.m_structinfo[offs+2];
            n2=n1+network.m_structinfo[offs+1]-1;
            w1=network.m_structinfo[offs+3];
            w2=w1+network.m_structinfo[offs+1]-1;
            //--- calculation
            for(j=n1;j<=n2;j++)
              {
               v=network.m_weights[w1+j-n1];
               for(i_=0;i_<=wcount-1;i_++)
                  rx[i].Set(i_,rx[i][i_]+v*ry[j][i_]);
               rx[i].Set(w1+j-n1,rx[i][w1+j-n1]+network.m_neurons[j]);
              }
            for(i_=0;i_<=wcount-1;i_++)
               ry[i].Set(i_,rx[i][i_]);
            continue;
           }
         //--- check
         if(network.m_structinfo[offs+0]<0)
           {
            bflag=true;
            //--- check
            if(network.m_structinfo[offs+0]==-2)
              {
               //--- input neuron,left unchanged
               bflag=false;
              }
            //--- check
            if(network.m_structinfo[offs+0]==-3)
              {
               //--- "-1" neuron,left unchanged
               bflag=false;
              }
            //--- check
            if(network.m_structinfo[offs+0]==-4)
              {
               //--- "0" neuron,left unchanged
               bflag=false;
              }
            //--- check
            if(!CAp::Assert(!bflag,__FUNCTION__+": internal error - unknown neuron type!"))
               return;
            continue;
           }
        }
      //--- Hessian. Backward pass of the R-algorithm.
      //--- Stage 1. Initialize RDY
      for(i=0;i<=ntotal+nout-1;i++)
        {
         for(i_=0;i_<=wcount-1;i_++)
            rdy[i].Set(i_,zeros[i_]);
        }
      //--- check
      if(network.m_structinfo[6]==0)
        {
         //--- Standardisation.
         //--- In context of the Hessian calculation standardisation
         //--- is considered as additional layer with weightless
         //--- activation function:
         //--- F(NET) :=Sigma*NET
         //--- So we add one more layer to forward pass,and
         //--- make forward/backward pass through this layer.
         for(i=0;i<=nout-1;i++)
           {
            n1=ntotal-nout+i;
            n2=ntotal+i;
            //--- Forward pass from N1 to N2
            for(i_=0;i_<=wcount-1;i_++)
               rx[n2].Set(i_,ry[n1][i_]);
            v=network.m_columnsigmas[nin+i];
            for(i_=0;i_<=wcount-1;i_++)
               ry[n2].Set(i_,v*rx[n2][i_]);
            //--- Initialization of RDY
            for(i_=0;i_<=wcount-1;i_++)
               rdy[n2].Set(i_,ry[n2][i_]);
            //--- Backward pass from N2 to N1:
            //--- 1. Calculate R(dE/dX).
            //--- 2. No R(dE/dWij) is needed since weight of activation neuron
            //---    is fixed to 1. So we can update R(dE/dY) for
            //---    the connected neuron (note that Vij=0,Wij=1)
            df=network.m_columnsigmas[nin+i];
            for(i_=0;i_<=wcount-1;i_++)
               rdx[n2].Set(i_,df*rdy[n2][i_]);
            for(i_=0;i_<=wcount-1;i_++)
               rdy[n1].Set(i_,rdy[n1][i_]+rdx[n2][i_]);
           }
        }
      else
        {
         //--- Softmax.
         //--- Initialize RDY using generalized expression for ei'(yi)
         //--- (see expression (9) from p. 5 of "Fast Exact Multiplication by the Hessian").
         //--- When we are working with softmax network,generalized
         //--- expression for ei'(yi) is used because softmax
         //--- normalization leads to ei,which depends on all y's
         if(naturalerr)
           {
            //--- softmax + cross-entropy.
            //--- We have:
            //--- S=sum(exp(yk)),
            //--- ei=sum(trn)*exp(yi)/S-trn_i
            //--- j=i:   d(ei)/d(yj)=T*exp(yi)*(S-exp(yi))/S^2
            //--- j<>i:  d(ei)/d(yj)=-T*exp(yi)*exp(yj)/S^2
            t=0;
            for(i=0;i<=nout-1;i++)
               t=t+desiredy[i];
            mx=network.m_neurons[ntotal-nout];
            //--- calculation
            for(i=0;i<=nout-1;i++)
               mx=MathMax(mx,network.m_neurons[ntotal-nout+i]);
            s=0;
            for(i=0;i<=nout-1;i++)
              {
               network.m_nwbuf[i]=MathExp(network.m_neurons[ntotal-nout+i]-mx);
               s=s+network.m_nwbuf[i];
              }
            //--- calculation
            for(i=0;i<=nout-1;i++)
              {
               for(j=0;j<=nout-1;j++)
                 {
                  //--- check
                  if(j==i)
                    {
                     deidyj=t*network.m_nwbuf[i]*(s-network.m_nwbuf[i])/CMath::Sqr(s);
                     for(i_=0;i_<=wcount-1;i_++)
                        rdy[ntotal-nout+i].Set(i_,rdy[ntotal-nout+i][i_]+deidyj*ry[ntotal-nout+i][i_]);
                    }
                  else
                    {
                     deidyj=-(t*network.m_nwbuf[i]*network.m_nwbuf[j]/CMath::Sqr(s));
                     for(i_=0;i_<=wcount-1;i_++)
                        rdy[ntotal-nout+i].Set(i_,rdy[ntotal-nout+i][i_]+deidyj*ry[ntotal-nout+j][i_]);
                    }
                 }
              }
           }
         else
           {
            //--- For a softmax + squared error we have expression
            //--- far beyond human imagination so we dont even try
            //--- to comment on it. Just enjoy the code...
            //--- P.S. That's why "natural error" is called "natural" -
            //--- compact beatiful expressions,fast code....
            mx=network.m_neurons[ntotal-nout];
            for(i=0;i<=nout-1;i++)
               mx=MathMax(mx,network.m_neurons[ntotal-nout+i]);
            //--- calculation
            s=0;
            s2=0;
            for(i=0;i<=nout-1;i++)
              {
               network.m_nwbuf[i]=MathExp(network.m_neurons[ntotal-nout+i]-mx);
               s=s+network.m_nwbuf[i];
               s2=s2+CMath::Sqr(network.m_nwbuf[i]);
              }
            //--- calculation
            q=0;
            for(i=0;i<=nout-1;i++)
               q=q+(network.m_y[i]-desiredy[i])*network.m_nwbuf[i];
            for(i=0;i<=nout-1;i++)
              {
               //--- change values
               z=-q+(network.m_y[i]-desiredy[i])*s;
               expi=network.m_nwbuf[i];
               for(j=0;j<=nout-1;j++)
                 {
                  expj=network.m_nwbuf[j];
                  //--- check
                  if(j==i)
                     deidyj=expi/CMath::Sqr(s)*((z+expi)*(s-2*expi)/s+expi*s2/CMath::Sqr(s));
                  else
                     deidyj=expi*expj/CMath::Sqr(s)*(s2/CMath::Sqr(s)-2*z/s-(expi+expj)/s+(network.m_y[i]-desiredy[i])-(network.m_y[j]-desiredy[j]));
                  for(i_=0;i_<=wcount-1;i_++)
                     rdy[ntotal-nout+i].Set(i_,rdy[ntotal-nout+i][i_]+deidyj*ry[ntotal-nout+j][i_]);
                 }
              }
           }
        }
      //--- Hessian. Backward pass of the R-algorithm
      //--- Stage 2. Process.
      for(i=ntotal-1;i>=0;i--)
        {
         //--- Possible variants:
         //--- 1. Activation function
         //--- 2. Adaptive summator
         //--- 3. Special neuron
         offs=istart+i*m_nfieldwidth;
         //--- check
         if(network.m_structinfo[offs+0]>0 || network.m_structinfo[offs+0]==-5)
           {
            n1=network.m_structinfo[offs+2];
            //--- First,calculate R(dE/dX).
            MLPActivationFunction(network.m_neurons[n1],network.m_structinfo[offs+0],f,df,d2f);
            v=d2f*network.m_derror[i];
            for(i_=0;i_<=wcount-1;i_++)
               rdx[i].Set(i_,df*rdy[i][i_]);
            for(i_=0;i_<=wcount-1;i_++)
               rdx[i].Set(i_,rdx[i][i_]+v*rx[i][i_]);
            //--- No R(dE/dWij) is needed since weight of activation neuron
            //--- is fixed to 1.
            //--- So we can update R(dE/dY) for the connected neuron.
            //--- (note that Vij=0,Wij=1)
            for(i_=0;i_<=wcount-1;i_++)
               rdy[n1].Set(i_,rdy[n1][i_]+rdx[i][i_]);
            continue;
           }
         //--- check
         if(network.m_structinfo[offs+0]==0)
           {
            //--- Adaptive summator
            n1=network.m_structinfo[offs+2];
            n2=n1+network.m_structinfo[offs+1]-1;
            w1=network.m_structinfo[offs+3];
            w2=w1+network.m_structinfo[offs+1]-1;
            //--- First,calculate R(dE/dX).
            for(i_=0;i_<=wcount-1;i_++)
               rdx[i].Set(i_,rdy[i][i_]);
            //--- Then,calculate R(dE/dWij)
            for(j=w1;j<=w2;j++)
              {
               v=network.m_neurons[n1+j-w1];
               for(i_=0;i_<=wcount-1;i_++)
                  h[j].Set(i_,h[j][i_]+v*rdx[i][i_]);
               //--- calculation
               v=network.m_derror[i];
               for(i_=0;i_<=wcount-1;i_++)
                  h[j].Set(i_,h[j][i_]+v*ry[n1+j-w1][i_]);
              }
            //--- And finally,update R(dE/dY) for connected neurons.
            for(j=w1;j<=w2;j++)
              {
               v=network.m_weights[j];
               for(i_=0;i_<=wcount-1;i_++)
                  rdy[n1+j-w1].Set(i_,rdy[n1+j-w1][i_]+v*rdx[i][i_]);
               rdy[n1+j-w1].Set(j,rdy[n1+j-w1][j]+network.m_derror[i]);
              }
            continue;
           }
         //--- check
         if(network.m_structinfo[offs+0]<0)
           {
            bflag=false;
            //--- check
            if((network.m_structinfo[offs+0]==-2 || network.m_structinfo[offs+0]==-3) || network.m_structinfo[offs+0]==-4)
              {
               //--- Special neuron type,no back-propagation required
               bflag=true;
              }
            //--- check
            if(!CAp::Assert(bflag,__FUNCTION__+": unknown neuron type!"))
               return;
            continue;
           }
        }
     }
  }
//+------------------------------------------------------------------+
//| Internal subroutine                                              |
//| Network must be processed by MLPProcess on X                     |
//+------------------------------------------------------------------+
static void CMLPBase::MLPInternalCalculateGradient(CMultilayerPerceptron &network,
                                                   double &neurons[],
                                                   double &weights[],
                                                   double &derror[],
                                                   double &grad[],
                                                   const bool naturalerrorfunc)
  {
//--- create variables
   int    i=0;
   int    n1=0;
   int    n2=0;
   int    w1=0;
   int    w2=0;
   int    ntotal=0;
   int    istart=0;
   int    nin=0;
   int    nout=0;
   int    offs=0;
   double dedf=0;
   double dfdnet=0;
   double v=0;
   double fown=0;
   double deown=0;
   double net=0;
   double mx=0;
   bool   bflag;
   int    i_=0;
   int    i1_=0;
//--- Read network geometry
   nin=network.m_structinfo[1];
   nout=network.m_structinfo[2];
   ntotal=network.m_structinfo[3];
   istart=network.m_structinfo[5];
//--- Pre-processing of dError/dOut:
//--- from dError/dOut(normalized) to dError/dOut(non-normalized)
   if(!CAp::Assert(network.m_structinfo[6]==0 || network.m_structinfo[6]==1,__FUNCTION__+": unknown normalization type!"))
      return;
//--- check
   if(network.m_structinfo[6]==1)
     {
      //--- Softmax
      if(!naturalerrorfunc)
        {
         mx=network.m_neurons[ntotal-nout];
         for(i=0;i<=nout-1;i++)
            mx=MathMax(mx,network.m_neurons[ntotal-nout+i]);
         net=0;
         for(i=0;i<=nout-1;i++)
           {
            network.m_nwbuf[i]=MathExp(network.m_neurons[ntotal-nout+i]-mx);
            net=net+network.m_nwbuf[i];
           }
         //--- calculation
         i1_=-(ntotal-nout);
         v=0.0;
         for(i_=ntotal-nout;i_<=ntotal-1;i_++)
            v+=network.m_derror[i_]*network.m_nwbuf[i_+i1_];
         for(i=0;i<=nout-1;i++)
           {
            fown=network.m_nwbuf[i];
            deown=network.m_derror[ntotal-nout+i];
            network.m_nwbuf[nout+i]=(-v+deown*fown+deown*(net-fown))*fown/CMath::Sqr(net);
           }
         for(i=0;i<=nout-1;i++)
            network.m_derror[ntotal-nout+i]=network.m_nwbuf[nout+i];
        }
     }
   else
     {
      //--- Un-standardisation
      for(i=0;i<=nout-1;i++)
         network.m_derror[ntotal-nout+i]=network.m_derror[ntotal-nout+i]*network.m_columnsigmas[nin+i];
     }
//--- Backpropagation
   for(i=ntotal-1;i>=0;i--)
     {
      //--- Extract info
      offs=istart+i*m_nfieldwidth;
      //--- check
      if(network.m_structinfo[offs+0]>0 || network.m_structinfo[offs+0]==-5)
        {
         //--- Activation function
         dedf=network.m_derror[i];
         dfdnet=network.m_dfdnet[i];
         derror[network.m_structinfo[offs+2]]=derror[network.m_structinfo[offs+2]]+dedf*dfdnet;
         continue;
        }
      //--- check
      if(network.m_structinfo[offs+0]==0)
        {
         //--- Adaptive summator
         n1=network.m_structinfo[offs+2];
         n2=n1+network.m_structinfo[offs+1]-1;
         w1=network.m_structinfo[offs+3];
         w2=w1+network.m_structinfo[offs+1]-1;
         dedf=network.m_derror[i];
         dfdnet=1.0;
         v=dedf*dfdnet;
         i1_=n1-w1;
         //--- calculation
         for(i_=w1;i_<=w2;i_++)
            grad[i_]=v*neurons[i_+i1_];
         i1_=w1-n1;
         for(i_=n1;i_<=n2;i_++)
            derror[i_]=derror[i_]+v*weights[i_+i1_];
         continue;
        }
      //--- check
      if(network.m_structinfo[offs+0]<0)
        {
         bflag=false;
         //--- check
         if((network.m_structinfo[offs+0]==-2 || network.m_structinfo[offs+0]==-3) || network.m_structinfo[offs+0]==-4)
           {
            //--- Special neuron type,no back-propagation required
            bflag=true;
           }
         //--- check
         if(!CAp::Assert(bflag,__FUNCTION__+": unknown neuron type!"))
            return;
         continue;
        }
     }
  }
//+------------------------------------------------------------------+
//| Internal subroutine, chunked gradient                            |
//+------------------------------------------------------------------+
static void CMLPBase::MLPChunkedGradient(CMultilayerPerceptron &network,
                                         CMatrixDouble &xy,const int cstart,
                                         const int csize,double &e,
                                         double &grad[],const bool naturalerrorfunc)
  {
//--- create variables
   int    i=0;
   int    j=0;
   int    k=0;
   int    kl=0;
   int    n1=0;
   int    n2=0;
   int    w1=0;
   int    w2=0;
   int    c1=0;
   int    c2=0;
   int    ntotal=0;
   int    nin=0;
   int    nout=0;
   int    offs=0;
   double f=0;
   double df=0;
   double d2f=0;
   double v=0;
   double s=0;
   double fown=0;
   double deown=0;
   double net=0;
   double lnnet=0;
   double mx=0;
   bool   bflag;
   int    istart=0;
   int    ineurons=0;
   int    idfdnet=0;
   int    iderror=0;
   int    izeros=0;
   int    i_=0;
   int    i1_=0;
//--- Read network geometry,prepare data
   nin=network.m_structinfo[1];
   nout=network.m_structinfo[2];
   ntotal=network.m_structinfo[3];
   istart=network.m_structinfo[5];
   c1=cstart;
   c2=cstart+csize-1;
   ineurons=0;
   idfdnet=ntotal;
   iderror=2*ntotal;
   izeros=3*ntotal;
   for(j=0;j<=csize-1;j++)
      network.m_chunks[izeros].Set(j,0);
//--- Forward pass:
//--- 1. Load inputs from XY to Chunks[0:NIn-1,0:CSize-1]
//--- 2. Forward pass
   for(i=0;i<=nin-1;i++)
     {
      for(j=0;j<=csize-1;j++)
        {
         //--- check
         if(network.m_columnsigmas[i]!=0.0)
            network.m_chunks[i].Set(j,(xy[c1+j][i]-network.m_columnmeans[i])/network.m_columnsigmas[i]);
         else
            network.m_chunks[i].Set(j,xy[c1+j][i]-network.m_columnmeans[i]);
        }
     }
   for(i=0;i<=ntotal-1;i++)
     {
      offs=istart+i*m_nfieldwidth;
      //--- check
      if(network.m_structinfo[offs+0]>0 || network.m_structinfo[offs+0]==-5)
        {
         //--- Activation function:
         //--- * calculate F vector,F(i)=F(NET(i))
         n1=network.m_structinfo[offs+2];
         for(i_=0;i_<=csize-1;i_++)
            network.m_chunks[i].Set(i_,network.m_chunks[n1][i_]);
         for(j=0;j<=csize-1;j++)
           {
            //--- function call
            MLPActivationFunction(network.m_chunks[i][j],network.m_structinfo[offs+0],f,df,d2f);
            //--- change values
            network.m_chunks[i].Set(j,f);
            network.m_chunks[idfdnet+i].Set(j,df);
           }
         continue;
        }
      //--- check
      if(network.m_structinfo[offs+0]==0)
        {
         //--- Adaptive summator:
         //--- * calculate NET vector,NET(i)=SUM(W(j,i)*Neurons(j),j=N1..N2)
         n1=network.m_structinfo[offs+2];
         n2=n1+network.m_structinfo[offs+1]-1;
         w1=network.m_structinfo[offs+3];
         w2=w1+network.m_structinfo[offs+1]-1;
         //--- calculation
         for(i_=0;i_<=csize-1;i_++)
            network.m_chunks[i].Set(i_,network.m_chunks[izeros][i_]);
         for(j=n1;j<=n2;j++)
           {
            v=network.m_weights[w1+j-n1];
            for(i_=0;i_<=csize-1;i_++)
               network.m_chunks[i].Set(i_,network.m_chunks[i][i_]+v*network.m_chunks[j][i_]);
           }
         continue;
        }
      //--- check
      if(network.m_structinfo[offs+0]<0)
        {
         bflag=false;
         //--- check
         if(network.m_structinfo[offs+0]==-2)
           {
            //--- input neuron,left unchanged
            bflag=true;
           }
         //--- check
         if(network.m_structinfo[offs+0]==-3)
           {
            //--- "-1" neuron
            for(k=0;k<=csize-1;k++)
               network.m_chunks[i].Set(k,-1);
            bflag=true;
           }
         //--- check
         if(network.m_structinfo[offs+0]==-4)
           {
            //--- "0" neuron
            for(k=0;k<=csize-1;k++)
               network.m_chunks[i].Set(k,0);
            bflag=true;
           }
         //--- check
         if(!CAp::Assert(bflag,__FUNCTION__+": internal error - unknown neuron type!"))
            return;
         continue;
        }
     }
//--- Post-processing,error,dError/dOut
   for(i=0;i<=ntotal-1;i++)
     {
      for(i_=0;i_<=csize-1;i_++)
         network.m_chunks[iderror+i].Set(i_,network.m_chunks[izeros][i_]);
     }
//--- check
   if(!CAp::Assert(network.m_structinfo[6]==0 || network.m_structinfo[6]==1,__FUNCTION__+": unknown normalization type!"))
      return;
//--- check
   if(network.m_structinfo[6]==1)
     {
      //--- Softmax output,classification network.
      //--- For each K=0..CSize-1 do:
      //--- 1. place exp(outputs[k]) to NWBuf[0:NOut-1]
      //--- 2. place sum(exp(..)) to NET
      //--- 3. calculate dError/dOut and place it to the second block of Chunks
      for(k=0;k<=csize-1;k++)
        {
         //--- Normalize
         mx=network.m_chunks[ntotal-nout][k];
         for(i=1;i<=nout-1;i++)
            mx=MathMax(mx,network.m_chunks[ntotal-nout+i][k]);
         net=0;
         for(i=0;i<=nout-1;i++)
           {
            network.m_nwbuf[i]=MathExp(network.m_chunks[ntotal-nout+i][k]-mx);
            net=net+network.m_nwbuf[i];
           }
         //--- Calculate error function and dError/dOut
         if(naturalerrorfunc)
           {
            //--- Natural error func.
            s=1;
            lnnet=MathLog(net);
            kl=(int)MathRound(xy[cstart+k][nin]);
            //--- calculation
            for(i=0;i<=nout-1;i++)
              {
               //--- check
               if(i==kl)
                  v=1;
               else
                  v=0;
               network.m_chunks[iderror+ntotal-nout+i].Set(k,s*network.m_nwbuf[i]/net-v);
               e=e+SafeCrossEntropy(v,network.m_nwbuf[i]/net);
              }
           }
         else
           {
            //--- Least squares error func
            //--- Error,dError/dOut(normalized)
            kl=(int)MathRound(xy[cstart+k][nin]);
            for(i=0;i<=nout-1;i++)
              {
               //--- check
               if(i==kl)
                  v=network.m_nwbuf[i]/net-1;
               else
                  v=network.m_nwbuf[i]/net;
               network.m_nwbuf[nout+i]=v;
               e=e+CMath::Sqr(v)/2;
              }
            //--- From dError/dOut(normalized) to dError/dOut(non-normalized)
            i1_=-nout;
            v=0.0;
            for(i_=nout;i_<=2*nout-1;i_++)
               v+=network.m_nwbuf[i_]*network.m_nwbuf[i_+i1_];
            //--- calculation
            for(i=0;i<=nout-1;i++)
              {
               fown=network.m_nwbuf[i];
               deown=network.m_nwbuf[nout+i];
               network.m_chunks[iderror+ntotal-nout+i].Set(k,(-v+deown*fown+deown*(net-fown))*fown/CMath::Sqr(net));
              }
           }
        }
     }
   else
     {
      //--- Normal output,regression network
      //--- For each K=0..CSize-1 do:
      //--- 1. calculate dError/dOut and place it to the second block of Chunks
      for(i=0;i<=nout-1;i++)
        {
         for(j=0;j<=csize-1;j++)
           {
            v=network.m_chunks[ntotal-nout+i][j]*network.m_columnsigmas[nin+i]+network.m_columnmeans[nin+i]-xy[cstart+j][nin+i];
            network.m_chunks[iderror+ntotal-nout+i].Set(j,v*network.m_columnsigmas[nin+i]);
            e=e+CMath::Sqr(v)/2;
           }
        }
     }
//--- Backpropagation
   for(i=ntotal-1;i>=0;i--)
     {
      //--- Extract info
      offs=istart+i*m_nfieldwidth;
      //--- check
      if(network.m_structinfo[offs+0]>0 || network.m_structinfo[offs+0]==-5)
        {
         //--- Activation function
         n1=network.m_structinfo[offs+2];
         for(k=0;k<=csize-1;k++)
            network.m_chunks[iderror+i].Set(k,network.m_chunks[iderror+i][k]*network.m_chunks[idfdnet+i][k]);
         for(i_=0;i_<=csize-1;i_++)
            network.m_chunks[iderror+n1].Set(i_,network.m_chunks[iderror+n1][i_]+network.m_chunks[iderror+i][i_]);
         continue;
        }
      //--- check
      if(network.m_structinfo[offs+0]==0)
        {
         //--- "Normal" activation function
         n1=network.m_structinfo[offs+2];
         n2=n1+network.m_structinfo[offs+1]-1;
         w1=network.m_structinfo[offs+3];
         w2=w1+network.m_structinfo[offs+1]-1;
         //--- calculation
         for(j=w1;j<=w2;j++)
           {
            v=0.0;
            for(i_=0;i_<=csize-1;i_++)
               v+=network.m_chunks[n1+j-w1][i_]*network.m_chunks[iderror+i][i_];
            grad[j]=grad[j]+v;
           }
         //--- calculation
         for(j=n1;j<=n2;j++)
           {
            v=network.m_weights[w1+j-n1];
            for(i_=0;i_<=csize-1;i_++)
               network.m_chunks[iderror+j].Set(i_,network.m_chunks[iderror+j][i_]+v*network.m_chunks[iderror+i][i_]);
           }
         continue;
        }
      //--- check
      if(network.m_structinfo[offs+0]<0)
        {
         bflag=false;
         //--- check
         if((network.m_structinfo[offs+0]==-2 || network.m_structinfo[offs+0]==-3) || network.m_structinfo[offs+0]==-4)
           {
            //--- Special neuron type,no back-propagation required
            bflag=true;
           }
         //--- check
         if(!CAp::Assert(bflag,__FUNCTION__+": unknown neuron type!"))
            return;
         continue;
        }
     }
  }
//+------------------------------------------------------------------+
//| Returns T*Ln(T/Z), guarded against overflow/underflow.           |
//| Internal subroutine.                                             |
//+------------------------------------------------------------------+
static double CMLPBase::SafeCrossEntropy(const double t,const double z)
  {
//--- create variables
   double result=0;
   double r=0;
//--- check
   if(t==0.0)
      result=0;
   else
     {
      //--- check
      if(MathAbs(z)>1.0)
        {
         //--- Shouldn't be the case with softmax,
         //--- but we just want to be sure.
         if(t/z==0.0)
            r=CMath::m_minrealnumber;
         else
            r=t/z;
        }
      else
        {
         //--- Normal case
         if(z==0.0 || MathAbs(t)>=CMath::m_maxrealnumber*MathAbs(z))
            r=CMath::m_maxrealnumber;
         else
            r=t/z;
        }
      //--- get result
      result=t*MathLog(r);
     }
//--- return result
   return(result);
  }
//+------------------------------------------------------------------+
//| Auxiliary class for CLogit                                       |
//+------------------------------------------------------------------+
class CLogitModel
  {
public:
   double            m_w[];
   //--- constructor, destructor
                     CLogitModel(void);
                    ~CLogitModel(void);
   //--- copy
   void              Copy(CLogitModel &obj);
  };
//+------------------------------------------------------------------+
//| Constructor without parameters                                   |
//+------------------------------------------------------------------+
CLogitModel::CLogitModel(void)
  {

  }
//+------------------------------------------------------------------+
//| Destructor                                                       |
//+------------------------------------------------------------------+
CLogitModel::~CLogitModel(void)
  {

  }
//+------------------------------------------------------------------+
//| Copy                                                             |
//+------------------------------------------------------------------+
void CLogitModel::Copy(CLogitModel &obj)
  {
//--- copy array
   ArrayCopy(m_w,obj.m_w);
  }
//+------------------------------------------------------------------+
//| This class is a shell for class CLogitModel                      |
//+------------------------------------------------------------------+
class CLogitModelShell
  {
private:
   CLogitModel       m_innerobj;
public:
   //--- constructors, destructor
                     CLogitModelShell(void);
                     CLogitModelShell(CLogitModel &obj);
                    ~CLogitModelShell(void);
   //--- method
   CLogitModel      *GetInnerObj(void);
  };
//+------------------------------------------------------------------+
//| Constructor without parameters                                   |
//+------------------------------------------------------------------+
CLogitModelShell::CLogitModelShell(void)
  {

  }
//+------------------------------------------------------------------+
//| Copy constructor                                                 |
//+------------------------------------------------------------------+
CLogitModelShell::CLogitModelShell(CLogitModel &obj)
  {
//--- copy
   m_innerobj.Copy(obj);
  }
//+------------------------------------------------------------------+
//| Destructor                                                       |
//+------------------------------------------------------------------+
CLogitModelShell::~CLogitModelShell(void)
  {

  }
//+------------------------------------------------------------------+
//| Return object of class                                           |
//+------------------------------------------------------------------+
CLogitModel *CLogitModelShell::GetInnerObj(void)
  {
//--- return result
   return(GetPointer(m_innerobj));
  }
//+------------------------------------------------------------------+
//| Auxiliary class for CLogit                                       |
//+------------------------------------------------------------------+
class CLogitMCState
  {
public:
   //--- variables
   bool              m_brackt;
   bool              m_stage1;
   int               m_infoc;
   double            m_dg;
   double            m_dgm;
   double            m_dginit;
   double            m_dgtest;
   double            m_dgx;
   double            m_dgxm;
   double            m_dgy;
   double            m_dgym;
   double            m_finit;
   double            m_ftest1;
   double            m_fm;
   double            m_fx;
   double            m_fxm;
   double            m_fy;
   double            m_fym;
   double            m_stx;
   double            m_sty;
   double            m_stmin;
   double            m_stmax;
   double            m_width;
   double            m_width1;
   double            m_xtrapf;
   //--- constructor, destructor
                     CLogitMCState(void);
                    ~CLogitMCState(void);
  };
//+------------------------------------------------------------------+
//| Constructor without parameters                                   |
//+------------------------------------------------------------------+
CLogitMCState::CLogitMCState(void)
  {

  }
//+------------------------------------------------------------------+
//| Destructor                                                       |
//+------------------------------------------------------------------+
CLogitMCState::~CLogitMCState(void)
  {

  }
//+------------------------------------------------------------------+
//| MNLReport structure contains information about training process: |
//| * NGrad     -   number of gradient calculations                  |
//| * NHess     -   number of Hessian calculations                   |
//+------------------------------------------------------------------+
class CMNLReport
  {
public:
   //--- variables
   int               m_ngrad;
   int               m_nhess;
   //--- constructor, destructor
                     CMNLReport(void);
                    ~CMNLReport(void);
   //--- copy
   void              Copy(CMNLReport &obj);
  };
//+------------------------------------------------------------------+
//| Constructor without parameters                                   |
//+------------------------------------------------------------------+
CMNLReport::CMNLReport(void)
  {

  }
//+------------------------------------------------------------------+
//| Destructor                                                       |
//+------------------------------------------------------------------+
CMNLReport::~CMNLReport(void)
  {

  }
//+------------------------------------------------------------------+
//| Copy                                                             |
//+------------------------------------------------------------------+
void CMNLReport::Copy(CMNLReport &obj)
  {
//--- copy variables
   m_ngrad=obj.m_ngrad;
   m_nhess=obj.m_nhess;
  }
//+------------------------------------------------------------------+
//| MNLReport structure contains information about training process: |
//| * NGrad     -   number of gradient calculations                  |
//| * NHess     -   number of Hessian calculations                   |
//+------------------------------------------------------------------+
class CMNLReportShell
  {
private:
   CMNLReport        m_innerobj;
public:
   //--- constructors, destructor
                     CMNLReportShell(void);
                     CMNLReportShell(CMNLReport &obj);
                    ~CMNLReportShell(void);
   //--- methods
   int               GetNGrad(void);
   void              SetNGrad(const int i);
   int               GetNHess(void);
   void              SetNHess(const int i);
   CMNLReport       *GetInnerObj(void);
  };
//+------------------------------------------------------------------+
//| Constructor without parameters                                   |
//+------------------------------------------------------------------+
CMNLReportShell::CMNLReportShell(void)
  {

  }
//+------------------------------------------------------------------+
//| Copy constructor                                                 |
//+------------------------------------------------------------------+
CMNLReportShell::CMNLReportShell(CMNLReport &obj)
  {
//--- copy
   m_innerobj.Copy(obj);
  }
//+------------------------------------------------------------------+
//| Destructor                                                       |
//+------------------------------------------------------------------+
CMNLReportShell::~CMNLReportShell(void)
  {

  }
//+------------------------------------------------------------------+
//| Returns the value of the variable ngrad                          |
//+------------------------------------------------------------------+
int CMNLReportShell::GetNGrad(void)
  {
//--- return result
   return(m_innerobj.m_ngrad);
  }
//+------------------------------------------------------------------+
//| Changing the value of the variable ngrad                         |
//+------------------------------------------------------------------+
void CMNLReportShell::SetNGrad(const int i)
  {
//--- change value
   m_innerobj.m_ngrad=i;
  }
//+------------------------------------------------------------------+
//| Returns the value of the variable nhess                          |
//+------------------------------------------------------------------+
int CMNLReportShell::GetNHess(void)
  {
//--- return result
   return(m_innerobj.m_nhess);
  }
//+------------------------------------------------------------------+
//| Changing the value of the variable nhess                         |
//+------------------------------------------------------------------+
void CMNLReportShell::SetNHess(const int i)
  {
//--- change value
   m_innerobj.m_nhess=i;
  }
//+------------------------------------------------------------------+
//| Return object of class                                           |
//+------------------------------------------------------------------+
CMNLReport *CMNLReportShell::GetInnerObj(void)
  {
//--- return result
   return(GetPointer(m_innerobj));
  }
//+------------------------------------------------------------------+
//| Class logit model                                                |
//+------------------------------------------------------------------+
class CLogit
  {
private:
   //--- private methods
   static void       MNLIExp(double &w[],double &x[]);
   static void       MNLAllErrors(CLogitModel &lm,CMatrixDouble &xy,const int npoints,double &relcls,double &avgce,double &rms,double &avg,double &avgrel);
   static void       MNLMCSrch(const int n,double &x[],double &f,double &g[],double &s[],double &stp,int &info,int &nfev,double &wa[],CLogitMCState &state,int &stage);
   static void       MNLMCStep(double &stx,double &fx,double &dx,double &sty,double &fy,double &dy,double &stp,const double fp,const double dp,bool &brackt,const double stmin,const double stmax,int &info);
public:
   //--- variables
   static const double m_xtol;
   static const double m_ftol;
   static const double m_gtol;
   static const int  m_maxfev;
   static const double m_stpmin;
   static const double m_stpmax;
   static const int  m_logitvnum;
   //--- constructor, destructor
                     CLogit(void);
                    ~CLogit(void);
   //--- public methods
   static void       MNLTrainH(CMatrixDouble &xy,const int npoints,const int nvars,const int nclasses,int &info,CLogitModel &lm,CMNLReport &rep);
   static void       MNLProcess(CLogitModel &lm,double &x[],double &y[]);
   static void       MNLProcessI(CLogitModel &lm,double &x[],double &y[]);
   static void       MNLUnpack(CLogitModel &lm,CMatrixDouble &a,int &nvars,int &nclasses);
   static void       MNLPack(CMatrixDouble &a,const int nvars,const int nclasses,CLogitModel &lm);
   static void       MNLCopy(CLogitModel &lm1,CLogitModel &lm2);
   static double     MNLAvgCE(CLogitModel &lm,CMatrixDouble &xy,const int npoints);
   static double     MNLRelClsError(CLogitModel &lm,CMatrixDouble &xy,const int npoints);
   static double     MNLRMSError(CLogitModel &lm,CMatrixDouble &xy,const int npoints);
   static double     MNLAvgError(CLogitModel &lm,CMatrixDouble &xy,const int npoints);
   static double     MNLAvgRelError(CLogitModel &lm,CMatrixDouble &xy,const int ssize);
   static int        MNLClsError(CLogitModel &lm,CMatrixDouble &xy,const int npoints);
  };
//+------------------------------------------------------------------+
//| Initialize constants                                             |
//+------------------------------------------------------------------+
const double CLogit::m_xtol=100*CMath::m_machineepsilon;
const double CLogit::m_ftol=0.0001;
const double CLogit::m_gtol=0.3;
const int    CLogit::m_maxfev=20;
const double CLogit::m_stpmin=1.0E-2;
const double CLogit::m_stpmax=1.0E5;
const int    CLogit::m_logitvnum=6;
//+------------------------------------------------------------------+
//| Constructor without parameters                                   |
//+------------------------------------------------------------------+
CLogit::CLogit(void)
  {

  }
//+------------------------------------------------------------------+
//| Destructor                                                       |
//+------------------------------------------------------------------+
CLogit::~CLogit(void)
  {

  }
//+------------------------------------------------------------------+
//| This subroutine trains logit model.                              |
//| INPUT PARAMETERS:                                                |
//|     XY          -   training set, array[0..NPoints-1,0..NVars]   |
//|                     First NVars columns store values of          |
//|                     independent variables, next column stores    |
//|                     number of class (from 0 to NClasses-1) which |
//|                     dataset element belongs to. Fractional values|
//|                     are rounded to nearest integer.              |
//|     NPoints     -   training set size, NPoints>=1                |
//|     NVars       -   number of independent variables, NVars>=1    |
//|     NClasses    -   number of classes, NClasses>=2               |
//| OUTPUT PARAMETERS:                                               |
//|     Info        -   return code:                                 |
//|                     * -2, if there is a point with class number  |
//|                           outside of [0..NClasses-1].            |
//|                     * -1, if incorrect parameters was passed     |
//|                           (NPoints<NVars+2, NVars<1, NClasses<2).|
//|                     *  1, if task has been solved                |
//|     LM          -   model built                                  |
//|     Rep         -   training report                              |
//+------------------------------------------------------------------+
static void CLogit::MNLTrainH(CMatrixDouble &xy,const int npoints,
                              const int nvars,const int nclasses,
                              int &info,CLogitModel &lm,CMNLReport &rep)
  {
//--- create variables
   int    i=0;
   int    j=0;
   int    k=0;
   int    ssize=0;
   bool   allsame;
   int    offs=0;
   double threshold=0;
   double wminstep=0;
   double decay=0;
   int    wdim=0;
   int    expoffs=0;
   double v=0;
   double s=0;
   int    nin=0;
   int    nout=0;
   int    wcount=0;
   double e=0;
   bool   spd;
   double wstep=0;
   int    mcstage=0;
   int    mcinfo=0;
   int    mcnfev=0;
   int    solverinfo=0;
   int    i_=0;
   int    i1_=0;
//--- creating arrays
   double g[];
   double x[];
   double y[];
   double wbase[];
   double wdir[];
   double work[];
//--- create matrix
   CMatrixDouble h;
//--- create objects of classes
   CLogitMCState         mcstate;
   CDenseSolverReport    solverrep;
   CMultilayerPerceptron network;
//--- initialization
   info=0;
   threshold=1000*CMath::m_machineepsilon;
   wminstep=0.001;
   decay=0.001;
//--- Test for inputs
   if((npoints<nvars+2 || nvars<1) || nclasses<2)
     {
      info=-1;
      return;
     }
   for(i=0;i<=npoints-1;i++)
     {
      //--- check
      if((int)MathRound(xy[i][nvars])<0 || (int)MathRound(xy[i][nvars])>=nclasses)
        {
         info=-2;
         return;
        }
     }
//--- change value
   info=1;
//--- Initialize data
   rep.m_ngrad=0;
   rep.m_nhess=0;
//--- Allocate array
   wdim=(nvars+1)*(nclasses-1);
   offs=5;
   expoffs=offs+wdim;
   ssize=5+(nvars+1)*(nclasses-1)+nclasses;
//--- allocation
   ArrayResizeAL(lm.m_w,ssize);
//--- change values
   lm.m_w[0]=ssize;
   lm.m_w[1]=m_logitvnum;
   lm.m_w[2]=nvars;
   lm.m_w[3]=nclasses;
   lm.m_w[4]=offs;
//--- Degenerate case: all outputs are equal
   allsame=true;
   for(i=1;i<=npoints-1;i++)
     {
      //--- check
      if((int)MathRound(xy[i][nvars])!=(int)MathRound(xy[i-1][nvars]))
         allsame=false;
     }
//--- check
   if(allsame)
     {
      for(i=0;i<=(nvars+1)*(nclasses-1)-1;i++)
         lm.m_w[offs+i]=0;
      //--- change values
      v=-(2*MathLog(CMath::m_minrealnumber));
      k=(int)MathRound(xy[0][nvars]);
      //--- check
      if(k==nclasses-1)
        {
         for(i=0;i<=nclasses-2;i++)
            lm.m_w[offs+i*(nvars+1)+nvars]=-v;
        }
      else
        {
         for(i=0;i<=nclasses-2;i++)
           {
            //--- check
            if(i==k)
               lm.m_w[offs+i*(nvars+1)+nvars]=v;
            else
               lm.m_w[offs+i*(nvars+1)+nvars]=0;
           }
        }
      //--- exit the function
      return;
     }
//--- General case.
//--- Prepare task and network. Allocate space.
   CMLPBase::MLPCreateC0(nvars,nclasses,network);
//--- function call
   CMLPBase::MLPInitPreprocessor(network,xy,npoints);
//--- function call
   CMLPBase::MLPProperties(network,nin,nout,wcount);
   for(i=0;i<=wcount-1;i++)
      network.m_weights[i]=(2*CMath::RandomReal()-1)/nvars;
//--- allocation
   ArrayResizeAL(g,wcount);
   h.Resize(wcount,wcount);
   ArrayResizeAL(wbase,wcount);
   ArrayResizeAL(wdir,wcount);
   ArrayResizeAL(work,wcount);
//--- First stage: optimize in gradient direction.
   for(k=0;k<=wcount/3+10;k++)
     {
      //--- Calculate gradient in starting point
      CMLPBase::MLPGradNBatch(network,xy,npoints,e,g);
      v=0.0;
      for(i_=0;i_<=wcount-1;i_++)
         v+=network.m_weights[i_]*network.m_weights[i_];
      //--- change value
      e=e+0.5*decay*v;
      for(i_=0;i_<=wcount-1;i_++)
         g[i_]=g[i_]+decay*network.m_weights[i_];
      rep.m_ngrad=rep.m_ngrad+1;
      //--- Setup optimization scheme
      for(i_=0;i_<=wcount-1;i_++)
         wdir[i_]=-g[i_];
      v=0.0;
      for(i_=0;i_<=wcount-1;i_++)
         v+=wdir[i_]*wdir[i_];
      //--- change values
      wstep=MathSqrt(v);
      v=1/MathSqrt(v);
      for(i_=0;i_<=wcount-1;i_++)
         wdir[i_]=v*wdir[i_];
      mcstage=0;
      //--- function call
      MNLMCSrch(wcount,network.m_weights,e,g,wdir,wstep,mcinfo,mcnfev,work,mcstate,mcstage);
      //--- cycle
      while(mcstage!=0)
        {
         //--- function call
         CMLPBase::MLPGradNBatch(network,xy,npoints,e,g);
         v=0.0;
         for(i_=0;i_<=wcount-1;i_++)
            v+=network.m_weights[i_]*network.m_weights[i_];
         //--- change value
         e=e+0.5*decay*v;
         for(i_=0;i_<=wcount-1;i_++)
            g[i_]=g[i_]+decay*network.m_weights[i_];
         rep.m_ngrad=rep.m_ngrad+1;
         //--- function call
         MNLMCSrch(wcount,network.m_weights,e,g,wdir,wstep,mcinfo,mcnfev,work,mcstate,mcstage);
        }
     }
//--- Second stage: use Hessian when we are close to the minimum
   while(true)
     {
      //--- Calculate and update E/G/H
      CMLPBase::MLPHessianNBatch(network,xy,npoints,e,g,h);
      v=0.0;
      for(i_=0;i_<=wcount-1;i_++)
         v+=network.m_weights[i_]*network.m_weights[i_];
      //--- change value
      e=e+0.5*decay*v;
      for(i_=0;i_<=wcount-1;i_++)
         g[i_]=g[i_]+decay*network.m_weights[i_];
      for(k=0;k<=wcount-1;k++)
         h[k].Set(k,h[k][k]+decay);
      rep.m_nhess=rep.m_nhess+1;
      //--- Select step direction
      //--- NOTE: it is important to use lower-triangle Cholesky
      //--- factorization since it is much faster than higher-triangle version.
      spd=CTrFac::SPDMatrixCholesky(h,wcount,false);
      //--- function call
      CDenseSolver::SPDMatrixCholeskySolve(h,wcount,false,g,solverinfo,solverrep,wdir);
      spd=solverinfo>0;
      //--- check
      if(spd)
        {
         //--- H is positive definite.
         //--- Step in Newton direction.
         for(i_=0;i_<=wcount-1;i_++)
            wdir[i_]=-1*wdir[i_];
         spd=true;
        }
      else
        {
         //--- H is indefinite.
         //--- Step in gradient direction.
         for(i_=0;i_<=wcount-1;i_++)
            wdir[i_]=-g[i_];
         spd=false;
        }
      //--- Optimize in WDir direction
      v=0.0;
      for(i_=0;i_<=wcount-1;i_++)
         v+=wdir[i_]*wdir[i_];
      //--- change values
      wstep=MathSqrt(v);
      v=1/MathSqrt(v);
      for(i_=0;i_<=wcount-1;i_++)
         wdir[i_]=v*wdir[i_];
      mcstage=0;
      //--- function call
      MNLMCSrch(wcount,network.m_weights,e,g,wdir,wstep,mcinfo,mcnfev,work,mcstate,mcstage);
      //--- cycle
      while(mcstage!=0)
        {
         //--- function call
         CMLPBase::MLPGradNBatch(network,xy,npoints,e,g);
         v=0.0;
         for(i_=0;i_<=wcount-1;i_++)
            v+=network.m_weights[i_]*network.m_weights[i_];
         //--- change value
         e=e+0.5*decay*v;
         for(i_=0;i_<=wcount-1;i_++)
            g[i_]=g[i_]+decay*network.m_weights[i_];
         rep.m_ngrad=rep.m_ngrad+1;
         //--- function call
         MNLMCSrch(wcount,network.m_weights,e,g,wdir,wstep,mcinfo,mcnfev,work,mcstate,mcstage);
        }
      //--- check
      if(spd && ((mcinfo==2 || mcinfo==4) || mcinfo==6))
         break;
     }
//--- Convert from NN format to MNL format
   i1_=-offs;
   for(i_=offs;i_<=offs+wcount-1;i_++)
      lm.m_w[i_]=network.m_weights[i_+i1_];
   for(k=0;k<=nvars-1;k++)
     {
      for(i=0;i<=nclasses-2;i++)
        {
         s=network.m_columnsigmas[k];
         //--- check
         if(s==0.0)
            s=1;
         //--- change values
         j=offs+(nvars+1)*i;
         v=lm.m_w[j+k];
         lm.m_w[j+k]=v/s;
         lm.m_w[j+nvars]=lm.m_w[j+nvars]+v*network.m_columnmeans[k]/s;
        }
     }
//--- calculation
   for(k=0;k<=nclasses-2;k++)
      lm.m_w[offs+(nvars+1)*k+nvars]=-lm.m_w[offs+(nvars+1)*k+nvars];
  }
//+------------------------------------------------------------------+
//| Procesing                                                        |
//| INPUT PARAMETERS:                                                |
//|     LM      -   logit model, passed by non-constant reference    |
//|                 (some fields of structure are used as temporaries|
//|                 when calculating model output).                  |
//|     X       -   input vector,  array[0..NVars-1].                |
//|     Y       -   (possibly) preallocated buffer; if size of Y is  |
//|                 less than NClasses, it will be reallocated.If it |
//|                 is large enough, it is NOT reallocated, so we    |
//|                 can save some time on reallocation.              |
//| OUTPUT PARAMETERS:                                               |
//|     Y       -   result, array[0..NClasses-1]                     |
//|                 Vector of posterior probabilities for            |
//|                 classification task.                             |
//+------------------------------------------------------------------+
static void CLogit::MNLProcess(CLogitModel &lm,double &x[],double &y[])
  {
//--- create variables
   int    nvars=0;
   int    nclasses=0;
   int    offs=0;
   int    i=0;
   int    i1=0;
   double s=0;
//--- check
   if(!CAp::Assert(lm.m_w[1]==m_logitvnum,__FUNCTION__+": unexpected model version"))
      return;
//--- initialization
   nvars=(int)MathRound(lm.m_w[2]);
   nclasses=(int)MathRound(lm.m_w[3]);
   offs=(int)MathRound(lm.m_w[4]);
//--- function call
   MNLIExp(lm.m_w,x);
   s=0;
//--- calculation
   i1=offs+(nvars+1)*(nclasses-1);
   for(i=i1;i<=i1+nclasses-1;i++)
      s=s+lm.m_w[i];
//--- check
   if(CAp::Len(y)<nclasses)
      ArrayResizeAL(y,nclasses);
//--- change values
   for(i=0;i<=nclasses-1;i++)
      y[i]=lm.m_w[i1+i]/s;
  }
//+------------------------------------------------------------------+
//| 'interactive' variant of MNLProcess for languages like Python    |
//| which support constructs like "Y=MNLProcess(LM,X)" and           |
//| interactive mode of the interpreter                              |
//| This function allocates new array on each call, so it is         |
//| significantly slower than its 'non-interactive' counterpart,     |
//| but it is  more  convenient when you call it from command line.  |
//+------------------------------------------------------------------+
static void CLogit::MNLProcessI(CLogitModel &lm,double &x[],double &y[])
  {
//--- function call
   MNLProcess(lm,x,y);
  }
//+------------------------------------------------------------------+
//| Unpacks coefficients of logit model. Logit model have form:      |
//|     P(class=i) = S(i) / (S(0) + S(1) + ... +S(M-1))              |
//|     S(i) = Exp(A[i,0]*X[0] + ... + A[i,N-1]*X[N-1] + A[i,N]),    |
//|            when i<M-1                                            |
//|     S(M-1) = 1                                                   |
//| INPUT PARAMETERS:                                                |
//|     LM          -   logit model in ALGLIB format                 |
//| OUTPUT PARAMETERS:                                               |
//|     V           -   coefficients, array[0..NClasses-2,0..NVars]  |
//|     NVars       -   number of independent variables              |
//|     NClasses    -   number of classes                            |
//+------------------------------------------------------------------+
static void CLogit::MNLUnpack(CLogitModel &lm,CMatrixDouble &a,int &nvars,
                              int &nclasses)
  {
//--- create variables
   int offs=0;
   int i=0;
   int i_=0;
   int i1_=0;
//--- initialization
   nvars=0;
   nclasses=0;
//--- check
   if(!CAp::Assert(lm.m_w[1]==m_logitvnum,__FUNCTION__+": unexpected model version"))
      return;
//--- initialization
   nvars=(int)MathRound(lm.m_w[2]);
   nclasses=(int)MathRound(lm.m_w[3]);
   offs=(int)MathRound(lm.m_w[4]);
//--- allocation
   a.Resize(nclasses-1,nvars+1);
//--- calculation
   for(i=0;i<=nclasses-2;i++)
     {
      i1_=offs+i*(nvars+1);
      for(i_=0;i_<=nvars;i_++)
         a[i].Set(i_,lm.m_w[i_+i1_]);
     }
  }
//+------------------------------------------------------------------+
//| "Packs" coefficients and creates logit model in ALGLIB format    |
//| (MNLUnpack reversed).                                            |
//| INPUT PARAMETERS:                                                |
//|     A           -   model (see MNLUnpack)                        |
//|     NVars       -   number of independent variables              |
//|     NClasses    -   number of classes                            |
//| OUTPUT PARAMETERS:                                               |
//|     LM          -   logit model.                                 |
//+------------------------------------------------------------------+
static void CLogit::MNLPack(CMatrixDouble &a,const int nvars,const int nclasses,
                            CLogitModel &lm)
  {
//--- create variables
   int offs=0;
   int i=0;
   int wdim=0;
   int ssize=0;
   int i_=0;
   int i1_=0;
//--- initialization
   wdim=(nvars+1)*(nclasses-1);
   offs=5;
   ssize=5+(nvars+1)*(nclasses-1)+nclasses;
//--- allocation
   ArrayResizeAL(lm.m_w,ssize);
//--- initialization
   lm.m_w[0]=ssize;
   lm.m_w[1]=m_logitvnum;
   lm.m_w[2]=nvars;
   lm.m_w[3]=nclasses;
   lm.m_w[4]=offs;
//--- calculation
   for(i=0;i<=nclasses-2;i++)
     {
      i1_=-(offs+i*(nvars+1));
      for(i_=offs+i*(nvars+1);i_<=offs+i*(nvars+1)+nvars;i_++)
         lm.m_w[i_]=a[i][i_+i1_];
     }
  }
//+------------------------------------------------------------------+
//| Copying of LogitModel strucure                                   |
//| INPUT PARAMETERS:                                                |
//|     LM1 -   original                                             |
//| OUTPUT PARAMETERS:                                               |
//|     LM2 -   copy                                                 |
//+------------------------------------------------------------------+
static void CLogit::MNLCopy(CLogitModel &lm1,CLogitModel &lm2)
  {
//--- create variables
   int k=0;
   int i_=0;
//--- initialization
   k=(int)MathRound(lm1.m_w[0]);
//--- allocation
   ArrayResizeAL(lm2.m_w,k);
//--- copy
   for(i_=0;i_<=k-1;i_++)
      lm2.m_w[i_]=lm1.m_w[i_];
  }
//+------------------------------------------------------------------+
//| Average cross-entropy (in bits per element) on the test set      |
//| INPUT PARAMETERS:                                                |
//|     LM      -   logit model                                      |
//|     XY      -   test set                                         |
//|     NPoints -   test set size                                    |
//| RESULT:                                                          |
//|     CrossEntropy/(NPoints*ln(2)).                                |
//+------------------------------------------------------------------+
static double CLogit::MNLAvgCE(CLogitModel &lm,CMatrixDouble &xy,const int npoints)
  {
//--- create variables
   double result=0;
   int    nvars=0;
   int    nclasses=0;
   int    i=0;
   int    i_=0;
//--- creating arrays
   double workx[];
   double worky[];
//--- check
   if(!CAp::Assert(lm.m_w[1]==m_logitvnum,__FUNCTION__+": unexpected model version"))
      return(EMPTY_VALUE);
//--- initialization
   nvars=(int)MathRound(lm.m_w[2]);
   nclasses=(int)MathRound(lm.m_w[3]);
//--- allocation
   ArrayResizeAL(workx,nvars);
   ArrayResizeAL(worky,nclasses);
//--- calculation
   for(i=0;i<=npoints-1;i++)
     {
      //--- check
      if(!CAp::Assert((int)MathRound(xy[i][nvars])>=0 &&(int)MathRound(xy[i][nvars])<nclasses,__FUNCTION__+": incorrect class number!"))
         return(EMPTY_VALUE);
      //--- Process
      for(i_=0;i_<=nvars-1;i_++)
         workx[i_]=xy[i][i_];
      //--- function call
      MNLProcess(lm,workx,worky);
      //--- check
      if(worky[(int)MathRound(xy[i][nvars])]>0.0)
         result=result-MathLog(worky[(int)MathRound(xy[i][nvars])]);
      else
         result=result-MathLog(CMath::m_minrealnumber);
     }
//--- return result
   return(result/(npoints*MathLog(2)));
  }
//+------------------------------------------------------------------+
//| Relative classification error on the test set                    |
//| INPUT PARAMETERS:                                                |
//|     LM      -   logit model                                      |
//|     XY      -   test set                                         |
//|     NPoints -   test set size                                    |
//| RESULT:                                                          |
//|     percent of incorrectly classified cases.                     |
//+------------------------------------------------------------------+
static double CLogit::MNLRelClsError(CLogitModel &lm,CMatrixDouble &xy,
                                     const int npoints)
  {
//--- return result
   return((double)MNLClsError(lm,xy,npoints)/(double)npoints);
  }
//+------------------------------------------------------------------+
//| RMS error on the test set                                        |
//| INPUT PARAMETERS:                                                |
//|     LM      -   logit model                                      |
//|     XY      -   test set                                         |
//|     NPoints -   test set size                                    |
//| RESULT:                                                          |
//|     root mean square error (error when estimating posterior      |
//|     probabilities).                                              |
//+------------------------------------------------------------------+
static double CLogit::MNLRMSError(CLogitModel &lm,CMatrixDouble &xy,
                                  const int npoints)
  {
//--- create variables
   double relcls=0;
   double avgce=0;
   double rms=0;
   double avg=0;
   double avgrel=0;
//--- check
   if(!CAp::Assert((int)MathRound(lm.m_w[1])==m_logitvnum,__FUNCTION__+": Incorrect MNL version!"))
      return(EMPTY_VALUE);
//--- function call
   MNLAllErrors(lm,xy,npoints,relcls,avgce,rms,avg,avgrel);
//--- return result
   return(rms);
  }
//+------------------------------------------------------------------+
//| Average error on the test set                                    |
//| INPUT PARAMETERS:                                                |
//|     LM      -   logit model                                      |
//|     XY      -   test set                                         |
//|     NPoints -   test set size                                    |
//| RESULT:                                                          |
//|     average error (error when estimating posterior               |
//|     probabilities).                                              |
//+------------------------------------------------------------------+
static double CLogit::MNLAvgError(CLogitModel &lm,CMatrixDouble &xy,
                                  const int npoints)
  {
//--- create variables
   double relcls=0;
   double avgce=0;
   double rms=0;
   double avg=0;
   double avgrel=0;
//--- check
   if(!CAp::Assert((int)MathRound(lm.m_w[1])==m_logitvnum,__FUNCTION__+": Incorrect MNL version!"))
      return(EMPTY_VALUE);
//--- function call
   MNLAllErrors(lm,xy,npoints,relcls,avgce,rms,avg,avgrel);
//--- return result
   return(avg);
  }
//+------------------------------------------------------------------+
//| Average relative error on the test set                           |
//| INPUT PARAMETERS:                                                |
//|     LM      -   logit model                                      |
//|     XY      -   test set                                         |
//|     NPoints -   test set size                                    |
//| RESULT:                                                          |
//|     average relative error (error when estimating posterior      |
//|     probabilities).                                              |
//+------------------------------------------------------------------+
static double CLogit::MNLAvgRelError(CLogitModel &lm,CMatrixDouble &xy,
                                     const int ssize)
  {
//--- create variables
   double relcls=0;
   double avgce=0;
   double rms=0;
   double avg=0;
   double avgrel=0;
//--- check
   if(!CAp::Assert((int)MathRound(lm.m_w[1])==m_logitvnum,__FUNCTION__+": Incorrect MNL version!"))
      return(EMPTY_VALUE);
//--- function call
   MNLAllErrors(lm,xy,ssize,relcls,avgce,rms,avg,avgrel);
//--- return result
   return(avgrel);
  }
//+------------------------------------------------------------------+
//| Classification error on test set = MNLRelClsError*NPoints        |
//+------------------------------------------------------------------+
static int CLogit::MNLClsError(CLogitModel &lm,CMatrixDouble &xy,const int npoints)
  {
//--- create variables
   int result=0;
   int nvars=0;
   int nclasses=0;
   int i=0;
   int j=0;
   int nmax=0;
   int i_=0;
//--- creating arrays
   double workx[];
   double worky[];
//--- check
   if(!CAp::Assert(lm.m_w[1]==m_logitvnum,__FUNCTION__+": unexpected model version"))
      return(-1);
//--- initialization
   nvars=(int)MathRound(lm.m_w[2]);
   nclasses=(int)MathRound(lm.m_w[3]);
//--- allocation
   ArrayResizeAL(workx,nvars);
   ArrayResizeAL(worky,nclasses);
//--- calculation
   for(i=0;i<=npoints-1;i++)
     {
      //--- Process
      for(i_=0;i_<=nvars-1;i_++)
         workx[i_]=xy[i][i_];
      //--- function call
      MNLProcess(lm,workx,worky);
      //--- Logit version of the answer
      nmax=0;
      for(j=0;j<=nclasses-1;j++)
        {
         //--- check
         if(worky[j]>worky[nmax])
            nmax=j;
        }
      //--- compare
      if(nmax!=(int)MathRound(xy[i][nvars]))
         result=result+1;
     }
//--- return result
   return(result);
  }
//+------------------------------------------------------------------+
//| Internal subroutine. Places exponents of the anti-overflow       |
//| shifted internal linear outputs into the service part of the W   |
//| array.                                                           |
//+------------------------------------------------------------------+
static void CLogit::MNLIExp(double &w[],double &x[])
  {
//--- create variables
   int    nvars=0;
   int    nclasses=0;
   int    offs=0;
   int    i=0;
   int    i1=0;
   double v=0;
   double mx=0;
   int    i_=0;
   int    i1_=0;
//--- check
   if(!CAp::Assert(w[1]==m_logitvnum,__FUNCTION__+": unexpected model version"))
      return;
//--- initialization
   nvars=(int)MathRound(w[2]);
   nclasses=(int)MathRound(w[3]);
   offs=(int)MathRound(w[4]);
//--- calculation
   i1=offs+(nvars+1)*(nclasses-1);
   for(i=0;i<=nclasses-2;i++)
     {
      //--- change values
      i1_=-(offs+i*(nvars+1));
      v=0.0;
      for(i_=offs+i*(nvars+1);i_<=offs+i*(nvars+1)+nvars-1;i_++)
         v+=w[i_]*x[i_+i1_];
      w[i1+i]=v+w[offs+i*(nvars+1)+nvars];
     }
//--- change values
   w[i1+nclasses-1]=0;
   mx=0;
//--- calculation
   for(i=i1;i<=i1+nclasses-1;i++)
      mx=MathMax(mx,w[i]);
   for(i=i1;i<=i1+nclasses-1;i++)
      w[i]=MathExp(w[i]-mx);
  }
//+------------------------------------------------------------------+
//| Calculation of all types of errors                               |
//+------------------------------------------------------------------+
static void CLogit::MNLAllErrors(CLogitModel &lm,CMatrixDouble &xy,
                                 const int npoints,double &relcls,
                                 double &avgce,double &rms,double &avg,
                                 double &avgrel)
  {
//--- create variables
   int nvars=0;
   int nclasses=0;
   int i=0;
   int i_=0;
//--- creating arrays
   double buf[];
   double workx[];
   double y[];
   double dy[];
//--- initialization
   relcls=0;
   avgce=0;
   rms=0;
   avg=0;
   avgrel=0;
//--- check
   if(!CAp::Assert((int)MathRound(lm.m_w[1])==m_logitvnum,__FUNCTION__+": Incorrect MNL version!"))
      return;
//--- initialization
   nvars=(int)MathRound(lm.m_w[2]);
   nclasses=(int)MathRound(lm.m_w[3]);
//--- allocation
   ArrayResizeAL(workx,nvars);
   ArrayResizeAL(y,nclasses);
   ArrayResizeAL(dy,1);
//--- function call
   CBdSS::DSErrAllocate(nclasses,buf);
   for(i=0;i<=npoints-1;i++)
     {
      for(i_=0;i_<=nvars-1;i_++)
         workx[i_]=xy[i][i_];
      //--- function call
      MNLProcess(lm,workx,y);
      //--- change value
      dy[0]=xy[i][nvars];
      //--- function call
      CBdSS::DSErrAccumulate(buf,y,dy);
     }
//--- function call
   CBdSS::DSErrFinish(buf);
//--- change values
   relcls=buf[0];
   avgce=buf[1];
   rms=buf[2];
   avg=buf[3];
   avgrel=buf[4];
  }
//+------------------------------------------------------------------+
//| The purpose of mcsrch is to find a step which satisfies a        |
//| sufficient decrease condition and a curvature condition.         |
//| At each stage the subroutine updates an interval of uncertainty  |
//| with endpoints stx and sty. The interval of uncertainty is       |
//| initially chosen so that it contains a minimizer of the modified |
//| function                                                         |
//|     f(x+stp*s) - f(x) - ftol*stp*(gradf(x)'s).                   |
//| If a step is obtained for  which the modified function has a     |
//| nonpositive function value and nonnegative derivative, then the  |
//| interval of uncertainty is chosen so that it contains a minimizer|
//| of f(x+stp*s).                                                   |
//| The algorithm is designed to find a step which satisfies the     |
//| sufficient decrease condition                                    |
//|     f(x+stp*s) .le. f(x) + ftol*stp*(gradf(x)'s),                |
//| and the curvature condition                                      |
//|     abs(gradf(x+stp*s)'s)) .le. gtol*abs(gradf(x)'s).            |
//| If ftol is less than gtol and if, for example, the function is   |
//| bounded below, then there is always a step which satisfies both  |
//| conditions. If no step can be found which satisfies both         |
//| conditions, then the algorithm usually stops when rounding       |
//| errors prevent further progress. In this case stp only satisfies |
//| the sufficient decrease condition.                               |
//| Parameters descriprion                                           |
//| N is a positive integer input variable set to the number of      |
//| variables.                                                       |
//| X is an array of length n. on input it must contain the base     |
//| point for the line search. on output it contains x+stp*s.        |
//| F is a variable. on input it must contain the value of f at x. On|
//| output it contains the value of f at x + stp*s.                  |
//| G is an array of length n. on input it must contain the gradient |
//| of f at x. On output it contains the gradient of f at x + stp*s. |
//| s is an input array of length n which specifies the search       |
//| direction.                                                       |
//| Stp  is  a nonnegative variable. on input stp contains an initial|
//| estimate of a satisfactory step. on output stp contains the final|
//| estimate.                                                        |
//| Ftol and gtol are nonnegative input variables. termination occurs|
//| when the sufficient decrease condition and the directional       |
//| derivative condition are satisfied.                              |
//| Xtol is a nonnegative input variable. termination occurs when the|
//| relative width of the interval of uncertainty is at most xtol.   |
//| Stpmin and stpmax are nonnegative input variables which specify  |
//| lower and upper bounds for the step.                             |
//| Maxfev is a positive integer input variable. termination occurs  |
//| when the number of calls to fcn is at least maxfev by the end of |
//| an iteration.                                                    |
//| Info is an integer output variable set as follows:               |
//|     info = 0  improper input parameters.                         |
//|     info = 1  the sufficient decrease condition and the          |
//|               directional derivative condition hold.             |
//|     info = 2  relative width of the interval of uncertainty      |
//|               is at most xtol.                                   |
//|     info = 3  number of calls to fcn has reached maxfev.         |
//|     info = 4  the step is at the lower bound stpmin.             |
//|     info = 5  the step is at the upper bound stpmax.             |
//|     info = 6  rounding errors prevent further progress.          |
//|               there may not be a step which satisfies the        |
//|               sufficient decrease and curvature conditions.      |
//|               tolerances may be too small.                       |
//| Nfev is an integer output variable set to the number of calls to |
//| fcn.                                                             |
//| wa is a work array of length n.                                  |
//| argonne national laboratory. minpack project. june 1983          |
//| Jorge J. More', David J. Thuente                                 |
//+------------------------------------------------------------------+
static void CLogit::MNLMCSrch(const int n,double &x[],double &f,double &g[],
                              double &s[],double &stp,int &info,int &nfev,
                              double &wa[],CLogitMCState &state,int &stage)
  {
//--- create variables
   double v=0;
   double p5=0;
   double p66=0;
   double zero=0;
   int    i_=0;
//--- init
   p5=0.5;
   p66=0.66;
   state.m_xtrapf=4.0;
   zero=0;
//--- Main cycle
   while(true)
     {
      //--- check
      if(stage==0)
        {
         //--- NEXT
         stage=2;
         continue;
        }
      //--- check
      if(stage==2)
        {
         state.m_infoc=1;
         info=0;
         //--- CHECK THE INPUT PARAMETERS FOR ERRORS.
         if(n<=0 || stp<=0.0 || m_ftol<0.0 || m_gtol<zero || m_xtol<zero || m_stpmin<zero || m_stpmax<m_stpmin || m_maxfev<=0)
           {
            stage=0;
            return;
           }
         //--- compute the initial gradient in the search direction
         //--- and check that s is a descent direction.
         v=0.0;
         for(i_=0;i_<=n-1;i_++)
            v+=g[i_]*s[i_];
         state.m_dginit=v;
         //--- check
         if(state.m_dginit>=0.0)
           {
            stage=0;
            return;
           }
         //--- initialize local variables.
         state.m_brackt=false;
         state.m_stage1=true;
         nfev=0;
         state.m_finit=f;
         state.m_dgtest=m_ftol*state.m_dginit;
         state.m_width=m_stpmax-m_stpmin;
         state.m_width1=state.m_width/p5;
         for(i_=0;i_<=n-1;i_++)
            wa[i_]=x[i_];
         //--- the variables stx,fx,dgx contain the values of the step,
         //--- function,and directional derivative at the best step.
         //--- the variables sty,fy,dgy contain the value of the step,
         //--- function,and derivative at the other endpoint of
         //--- the interval of uncertainty.
         //--- the variables stp,f,dg contain the values of the step,
         //--- function,and derivative at the current step.
         state.m_stx=0;
         state.m_fx=state.m_finit;
         state.m_dgx=state.m_dginit;
         state.m_sty=0;
         state.m_fy=state.m_finit;
         state.m_dgy=state.m_dginit;
         //--- NEXT
         stage=3;
         continue;
        }
      //--- check
      if(stage==3)
        {
         //--- start of iteration.
         //--- set the minimum and maximum steps to correspond
         //--- to the present interval of uncertainty.
         if(state.m_brackt)
           {
            //--- check
            if(state.m_stx<state.m_sty)
              {
               state.m_stmin=state.m_stx;
               state.m_stmax=state.m_sty;
              }
            else
              {
               state.m_stmin=state.m_sty;
               state.m_stmax=state.m_stx;
              }
           }
         else
           {
            state.m_stmin=state.m_stx;
            state.m_stmax=stp+state.m_xtrapf*(stp-state.m_stx);
           }
         //--- force the step to be within the bounds stpmax and stpmin.
         if(stp>m_stpmax)
            stp=m_stpmax;
         //--- check
         if(stp<m_stpmin)
            stp=m_stpmin;
         //--- if an unusual termination is to occur then let
         //--- stp be the lowest point obtained so far.
         if((((state.m_brackt && (stp<=state.m_stmin || stp>=state.m_stmax)) || nfev>=m_maxfev-1) || state.m_infoc==0) || (state.m_brackt && state.m_stmax-state.m_stmin<=m_xtol*state.m_stmax))
            stp=state.m_stx;
         //--- evaluate the function and gradient at stp
         //--- and compute the directional derivative.
         for(i_=0;i_<=n-1;i_++)
            x[i_]=wa[i_];
         for(i_=0;i_<=n-1;i_++)
            x[i_]=x[i_]+stp*s[i_];
         //--- next
         stage=4;
         return;
        }
      //--- check
      if(stage==4)
        {
         info=0;
         nfev=nfev+1;
         v=0.0;
         //--- calculation
         for(i_=0;i_<=n-1;i_++)
            v+=g[i_]*s[i_];
         state.m_dg=v;
         state.m_ftest1=state.m_finit+stp*state.m_dgtest;
         //--- test for convergence.
         if((state.m_brackt && (stp<=state.m_stmin || stp>=state.m_stmax)) || state.m_infoc==0)
            info=6;
         //--- check
         if((stp==m_stpmax &&f<=state.m_ftest1) &&state.m_dg<=state.m_dgtest)
            info=5;
         //--- check
         if(stp==m_stpmin && (f>state.m_ftest1 || state.m_dg>=state.m_dgtest))
            info=4;
         //--- check
         if(nfev>=m_maxfev)
            info=3;
         //--- check
         if(state.m_brackt && state.m_stmax-state.m_stmin<=m_xtol*state.m_stmax)
            info=2;
         //--- check
         if(f<=state.m_ftest1 && MathAbs(state.m_dg)<=-(m_gtol*state.m_dginit))
            info=1;
         //--- check for termination.
         if(info!=0)
           {
            stage=0;
            return;
           }
         //--- in the first stage we seek a step for which the modified
         //--- function has a nonpositive value and nonnegative derivative.
         if((state.m_stage1 && f<=state.m_ftest1) && state.m_dg>=MathMin(m_ftol,m_gtol)*state.m_dginit)
            state.m_stage1=false;
         //--- a modified function is used to predict the step only if
         //--- we have not obtained a step for which the modified
         //--- function has a nonpositive function value and nonnegative
         //--- derivative,and if a lower function value has been
         //--- obtained but the decrease is not sufficient.
         if((state.m_stage1 && f<=state.m_fx) && f>state.m_ftest1)
           {
            //--- define the modified function and derivative values.
            state.m_fm=f-stp*state.m_dgtest;
            state.m_fxm=state.m_fx-state.m_stx*state.m_dgtest;
            state.m_fym=state.m_fy-state.m_sty*state.m_dgtest;
            state.m_dgm=state.m_dg-state.m_dgtest;
            state.m_dgxm=state.m_dgx-state.m_dgtest;
            state.m_dgym=state.m_dgy-state.m_dgtest;
            //--- call cstep to update the interval of uncertainty
            //--- and to compute the new step.
            MNLMCStep(state.m_stx,state.m_fxm,state.m_dgxm,state.m_sty,state.m_fym,state.m_dgym,stp,state.m_fm,state.m_dgm,state.m_brackt,state.m_stmin,state.m_stmax,state.m_infoc);
            //--- reset the function and gradient values for f.
            state.m_fx=state.m_fxm+state.m_stx*state.m_dgtest;
            state.m_fy=state.m_fym+state.m_sty*state.m_dgtest;
            state.m_dgx=state.m_dgxm+state.m_dgtest;
            state.m_dgy=state.m_dgym+state.m_dgtest;
           }
         else
           {
            //--- call mcstep to update the interval of uncertainty
            //--- and to compute the new step.
            MNLMCStep(state.m_stx,state.m_fx,state.m_dgx,state.m_sty,state.m_fy,state.m_dgy,stp,f,state.m_dg,state.m_brackt,state.m_stmin,state.m_stmax,state.m_infoc);
           }
         //--- force a sufficient decrease in the size of the
         //--- interval of uncertainty.
         if(state.m_brackt)
           {
            //--- check
            if(MathAbs(state.m_sty-state.m_stx)>=p66*state.m_width1)
               stp=state.m_stx+p5*(state.m_sty-state.m_stx);
            state.m_width1=state.m_width;
            state.m_width=MathAbs(state.m_sty-state.m_stx);
           }
         //--- next.
         stage=3;
         continue;
        }
     }
  }
//+------------------------------------------------------------------+
//| Auxiliary function for MNLMCSrch                                 |
//+------------------------------------------------------------------+
static void CLogit::MNLMCStep(double &stx,double &fx,double &dx,double &sty,
                              double &fy,double &dy,double &stp,const double fp,
                              const double dp,bool &brackt,const double stmin,
                              const double stmax,int &info)
  {
//--- create variables
   bool   bound;
   double gamma=0;
   double p=0;
   double q=0;
   double r=0;
   double s=0;
   double sgnd=0;
   double stpc=0;
   double stpf=0;
   double stpq=0;
   double theta=0;
//--- initialization
   info=0;
//--- check the input parameters for errors.
   if(((brackt && (stp<=MathMin(stx,sty) || stp>=MathMax(stx,sty))) || dx*(stp-stx)>=0.0) || stmax<stmin)
      return;
//--- determine if the derivatives have opposite sign.
   sgnd=dp*(dx/MathAbs(dx));
//--- first case. a higher function value.
//--- the minimum is bracketed. if the cubic step is closer
//--- to stx than the quadratic step,the cubic step is taken,
//--- else the average of the cubic and quadratic steps is taken.
   if(fp>fx)
     {
      //--- change value
      info=1;
      bound=true;
      theta=3*(fx-fp)/(stp-stx)+dx+dp;
      s=MathMax(MathAbs(theta),MathMax(MathAbs(dx),MathAbs(dp)));
      gamma=s*MathSqrt(CMath::Sqr(theta/s)-dx/s*(dp/s));
      //--- check
      if(stp<stx)
         gamma=-gamma;
      //--- change value
      p=gamma-dx+theta;
      q=gamma-dx+gamma+dp;
      r=p/q;
      stpc=stx+r*(stp-stx);
      stpq=stx+dx/((fx-fp)/(stp-stx)+dx)/2*(stp-stx);
      //--- check
      if(MathAbs(stpc-stx)<MathAbs(stpq-stx))
         stpf=stpc;
      else
         stpf=stpc+(stpq-stpc)/2;
      brackt=true;
     }
   else
     {
      //--- check
      if(sgnd<0.0)
        {
         //--- second case. a lower function value and derivatives of
         //--- opposite sign. the minimum is bracketed. if the cubic
         //--- step is closer to stx than the quadratic (secant) step,
         //--- the cubic step is taken,else the quadratic step is taken.
         info=2;
         bound=false;
         theta=3*(fx-fp)/(stp-stx)+dx+dp;
         s=MathMax(MathAbs(theta),MathMax(MathAbs(dx),MathAbs(dp)));
         gamma=s*MathSqrt(CMath::Sqr(theta/s)-dx/s*(dp/s));
         //--- check
         if(stp>stx)
            gamma=-gamma;
         //--- change values
         p=gamma-dp+theta;
         q=gamma-dp+gamma+dx;
         r=p/q;
         stpc=stp+r*(stx-stp);
         stpq=stp+dp/(dp-dx)*(stx-stp);
         //--- check
         if(MathAbs(stpc-stp)>MathAbs(stpq-stp))
            stpf=stpc;
         else
            stpf=stpq;
         brackt=true;
        }
      else
        {
         //--- check
         if(MathAbs(dp)<MathAbs(dx))
           {
            //--- third case. a lower function value,derivatives of the
            //--- same sign,and the magnitude of the derivative decreases.
            //--- the cubic step is only used if the cubic tends to infinity
            //--- in the direction of the step or if the minimum of the cubic
            //--- is beyond stp. otherwise the cubic step is defined to be
            //--- either stpmin or stpmax. the quadratic (secant) step is also
            //--- computed and if the minimum is bracketed then the the step
            //--- closest to stx is taken,else the step farthest away is taken.
            info=3;
            bound=true;
            theta=3*(fx-fp)/(stp-stx)+dx+dp;
            s=MathMax(MathAbs(theta),MathMax(MathAbs(dx),MathAbs(dp)));
            //--- the case gamma=0 only arises if the cubic does not tend
            //--- to infinity in the direction of the step.
            gamma=s*MathSqrt(MathMax(0,CMath::Sqr(theta/s)-dx/s*(dp/s)));
            //--- check
            if(stp>stx)
               gamma=-gamma;
            p=gamma-dp+theta;
            q=gamma+(dx-dp)+gamma;
            r=p/q;
            //--- check
            if(r<0.0 && gamma!=0.0)
               stpc=stp+r*(stx-stp);
            else
              {
               //--- check
               if(stp>stx)
                  stpc=stmax;
               else
                  stpc=stmin;
              }
            stpq=stp+dp/(dp-dx)*(stx-stp);
            //--- check
            if(brackt)
              {
               //--- check
               if(MathAbs(stp-stpc)<MathAbs(stp-stpq))
                  stpf=stpc;
               else
                  stpf=stpq;
              }
            else
              {
               //--- check
               if(MathAbs(stp-stpc)>MathAbs(stp-stpq))
                  stpf=stpc;
               else
                  stpf=stpq;
              }
           }
         else
           {
            //--- fourth case. a lower function value,derivatives of the
            //--- same sign,and the magnitude of the derivative does
            //--- not decrease. if the minimum is not bracketed,the step
            //--- is either stpmin or stpmax,else the cubic step is taken.
            info=4;
            bound=false;
            //--- check
            if(brackt)
              {
               //--- change values
               theta=3*(fp-fy)/(sty-stp)+dy+dp;
               s=MathMax(MathAbs(theta),MathMax(MathAbs(dy),MathAbs(dp)));
               gamma=s*MathSqrt(CMath::Sqr(theta/s)-dy/s*(dp/s));
               //--- check
               if(stp>sty)
                  gamma=-gamma;
               //--- change values
               p=gamma-dp+theta;
               q=gamma-dp+gamma+dy;
               r=p/q;
               stpc=stp+r*(sty-stp);
               stpf=stpc;
              }
            else
              {
               //--- check
               if(stp>stx)
                  stpf=stmax;
               else
                  stpf=stmin;
              }
           }
        }
     }
//--- update the interval of uncertainty. this update does not
//--- depend on the new step or the case analysis above.
   if(fp>fx)
     {
      sty=stp;
      fy=fp;
      dy=dp;
     }
   else
     {
      //--- check
      if(sgnd<0.0)
        {
         sty=stx;
         fy=fx;
         dy=dx;
        }
      //--- change values
      stx=stp;
      fx=fp;
      dx=dp;
     }
//--- compute the new step and safeguard it.
   stpf=MathMin(stmax,stpf);
   stpf=MathMax(stmin,stpf);
   stp=stpf;
//--- check
   if(brackt && bound)
     {
      //--- check
      if(sty>stx)
         stp=MathMin(stx+0.66*(sty-stx),stp);
      else
         stp=MathMax(stx+0.66*(sty-stx),stp);
     }
  }
//+------------------------------------------------------------------+
//| This structure is a MCPD (Markov Chains for Population Data)     |
//| solver. You should use ALGLIB functions in order to work with    |
//| this object.                                                     |
//+------------------------------------------------------------------+
class CMCPDState
  {
public:
   //--- variables
   int               m_n;
   int               m_npairs;
   int               m_ccnt;
   double            m_regterm;
   int               m_repinneriterationscount;
   int               m_repouteriterationscount;
   int               m_repnfev;
   int               m_repterminationtype;
   CMinBLEICState    m_bs;
   CMinBLEICReport   m_br;
   //--- arrays
   int               m_states[];
   int               m_ct[];
   double            m_pw[];
   double            m_tmpp[];
   double            m_effectivew[];
   double            m_effectivebndl[];
   double            m_effectivebndu[];
   int               m_effectivect[];
   double            m_h[];
   //--- matrices
   CMatrixDouble     m_data;
   CMatrixDouble     m_ec;
   CMatrixDouble     m_bndl;
   CMatrixDouble     m_bndu;
   CMatrixDouble     m_c;
   CMatrixDouble     m_priorp;
   CMatrixDouble     m_effectivec;
   CMatrixDouble     m_p;
   //--- constructor, destructor
                     CMCPDState(void);
                    ~CMCPDState(void);
   //--- copy
   void              Copy(CMCPDState &obj);
  };
//+------------------------------------------------------------------+
//| Constructor without parameters                                   |
//+------------------------------------------------------------------+
CMCPDState::CMCPDState(void)
  {

  }
//+------------------------------------------------------------------+
//| Destructor                                                       |
//+------------------------------------------------------------------+
CMCPDState::~CMCPDState(void)
  {

  }
//+------------------------------------------------------------------+
//| Copy                                                             |
//+------------------------------------------------------------------+
void CMCPDState::Copy(CMCPDState &obj)
  {
//--- copy variables
   m_n=obj.m_n;
   m_npairs=obj.m_npairs;
   m_ccnt=obj.m_ccnt;
   m_regterm=obj.m_regterm;
   m_repinneriterationscount=obj.m_repinneriterationscount;
   m_repouteriterationscount=obj.m_repouteriterationscount;
   m_repnfev=obj.m_repnfev;
   m_repterminationtype=obj.m_repterminationtype;
   m_bs.Copy(obj.m_bs);
   m_br.Copy(obj.m_br);
//--- copy arrays
   ArrayCopy(m_states,obj.m_states);
   ArrayCopy(m_ct,obj.m_ct);
   ArrayCopy(m_pw,obj.m_pw);
   ArrayCopy(m_tmpp,obj.m_tmpp);
   ArrayCopy(m_effectivew,obj.m_effectivew);
   ArrayCopy(m_effectivebndl,obj.m_effectivebndl);
   ArrayCopy(m_effectivebndu,obj.m_effectivebndu);
   ArrayCopy(m_effectivect,obj.m_effectivect);
   ArrayCopy(m_h,obj.m_h);
//--- copy matrices
   m_data=obj.m_data;
   m_ec=obj.m_ec;
   m_bndl=obj.m_bndl;
   m_bndu=obj.m_bndu;
   m_c=obj.m_c;
   m_priorp=obj.m_priorp;
   m_effectivec=obj.m_effectivec;
   m_p=obj.m_p;
  }
//+------------------------------------------------------------------+
//| This structure is a MCPD (Markov Chains for Population Data)     |
//| solver.                                                          |
//| You should use ALGLIB functions in order to work with this object|
//+------------------------------------------------------------------+
class CMCPDStateShell
  {
private:
   CMCPDState        m_innerobj;
public:
   //--- constructors, destructor
                     CMCPDStateShell(void);
                     CMCPDStateShell(CMCPDState &obj);
                    ~CMCPDStateShell(void);
   //--- method
   CMCPDState       *GetInnerObj(void);
  };
//+------------------------------------------------------------------+
//| Constructor without parameters                                   |
//+------------------------------------------------------------------+
CMCPDStateShell::CMCPDStateShell(void)
  {

  }
//+------------------------------------------------------------------+
//| Copy constructor                                                 |
//+------------------------------------------------------------------+
CMCPDStateShell::CMCPDStateShell(CMCPDState &obj)
  {
//--- copy
   m_innerobj.Copy(obj);
  }
//+------------------------------------------------------------------+
//| Destructor                                                       |
//+------------------------------------------------------------------+
CMCPDStateShell::~CMCPDStateShell(void)
  {

  }
//+------------------------------------------------------------------+
//| Return object of class                                           |
//+------------------------------------------------------------------+
CMCPDState *CMCPDStateShell::GetInnerObj(void)
  {
//--- return result
   return(GetPointer(m_innerobj));
  }
//+------------------------------------------------------------------+
//| This structure is a MCPD training report:                        |
//|     InnerIterationsCount    -   number of inner iterations of the|
//|                                 underlying optimization algorithm|
//|     OuterIterationsCount    -   number of outer iterations of the|
//|                                 underlying optimization algorithm|
//|     NFEV                    -   number of merit function         |
//|                                 evaluations                      |
//|     TerminationType         -   termination type                 |
//|                                 (same as for MinBLEIC optimizer, |
//|                                 positive values denote success,  | 
//|                                 negative ones - failure)         |
//+------------------------------------------------------------------+
class CMCPDReport
  {
public:
   //--- variables
   int               m_inneriterationscount;
   int               m_outeriterationscount;
   int               m_nfev;
   int               m_terminationtype;
   //--- constructor, destructor
                     CMCPDReport(void);
                    ~CMCPDReport(void);
   //--- copy
   void              Copy(CMCPDReport &obj);
  };
//+------------------------------------------------------------------+
//| Constructor without parameters                                   |
//+------------------------------------------------------------------+
CMCPDReport::CMCPDReport(void)
  {

  }
//+------------------------------------------------------------------+
//| Destructor                                                       |
//+------------------------------------------------------------------+
CMCPDReport::~CMCPDReport(void)
  {

  }
//+------------------------------------------------------------------+
//| Copy                                                             |
//+------------------------------------------------------------------+
void CMCPDReport::Copy(CMCPDReport &obj)
  {
//--- copy variables
   m_inneriterationscount=obj.m_inneriterationscount;
   m_outeriterationscount=obj.m_outeriterationscount;
   m_nfev=obj.m_nfev;
   m_terminationtype=obj.m_terminationtype;
  }
//+------------------------------------------------------------------+
//| This structure is a MCPD training report:                        |
//|     InnerIterationsCount    -   number of inner iterations of the|
//|                                 underlying optimization algorithm|
//|     OuterIterationsCount    -   number of outer iterations of the|
//|                                 underlying optimization algorithm|
//|     NFEV                    -   number of merit function         |
//|                                 evaluations                      |
//|     TerminationType         -   termination type                 |
//|                                 (same as for MinBLEIC optimizer, |
//|                                 positive values denote success,  | 
//|                                 negative ones - failure)         |
//+------------------------------------------------------------------+
class CMCPDReportShell
  {
private:
   CMCPDReport       m_innerobj;
public:
   //--- constructors, destructor
                     CMCPDReportShell(void);
                     CMCPDReportShell(CMCPDReport &obj);
                    ~CMCPDReportShell(void);
   //--- methods
   int               GetInnerIterationsCount(void);
   void              SetInnerIterationsCount(const int i);
   int               GetOuterIterationsCount(void);
   void              SetOuterIterationsCount(const int i);
   int               GetNFev(void);
   void              SetNFev(const int i);
   int               GetTerminationType(void);
   void              SetTerminationType(const int i);
   CMCPDReport      *GetInnerObj(void);
  };
//+------------------------------------------------------------------+
//| Constructor without parameters                                   |
//+------------------------------------------------------------------+
CMCPDReportShell::CMCPDReportShell(void)
  {

  }
//+------------------------------------------------------------------+
//| Copy constructor                                                 |
//+------------------------------------------------------------------+
CMCPDReportShell::CMCPDReportShell(CMCPDReport &obj)
  {
//--- copy
   m_innerobj.Copy(obj);
  }
//+------------------------------------------------------------------+
//| Destructor                                                       |
//+------------------------------------------------------------------+
CMCPDReportShell::~CMCPDReportShell(void)
  {

  }
//+------------------------------------------------------------------+
//| Returns the value of the variable inneriterationscount           |
//+------------------------------------------------------------------+
int CMCPDReportShell::GetInnerIterationsCount(void)
  {
//--- return result
   return(m_innerobj.m_inneriterationscount);
  }
//+------------------------------------------------------------------+
//| Changing the value of the variable inneriterationscount          |
//+------------------------------------------------------------------+
void CMCPDReportShell::SetInnerIterationsCount(const int i)
  {
//--- change value
   m_innerobj.m_inneriterationscount=i;
  }
//+------------------------------------------------------------------+
//| Returns the value of the variable outeriterationscount           |
//+------------------------------------------------------------------+
int CMCPDReportShell::GetOuterIterationsCount(void)
  {
//--- return result
   return(m_innerobj.m_outeriterationscount);
  }
//+------------------------------------------------------------------+
//| Changing the value of the variable outeriterationscount          |
//+------------------------------------------------------------------+
void CMCPDReportShell::SetOuterIterationsCount(const int i)
  {
//--- change value
   m_innerobj.m_outeriterationscount=i;
  }
//+------------------------------------------------------------------+
//| Returns the value of the variable nfev                           |
//+------------------------------------------------------------------+
int CMCPDReportShell::GetNFev(void)
  {
//--- return result
   return(m_innerobj.m_nfev);
  }
//+------------------------------------------------------------------+
//| Changing the value of the variable nfev                          |
//+------------------------------------------------------------------+
void CMCPDReportShell::SetNFev(const int i)
  {
//--- change value
   m_innerobj.m_nfev=i;
  }
//+------------------------------------------------------------------+
//| Returns the value of the variable terminationtype                |
//+------------------------------------------------------------------+
int CMCPDReportShell::GetTerminationType(void)
  {
//--- return result
   return(m_innerobj.m_terminationtype);
  }
//+------------------------------------------------------------------+
//| Changing the value of the variable terminationtype               |
//+------------------------------------------------------------------+
void CMCPDReportShell::SetTerminationType(const int i)
  {
//--- change value
   m_innerobj.m_terminationtype=i;
  }
//+------------------------------------------------------------------+
//| Return object of class                                           |
//+------------------------------------------------------------------+
CMCPDReport *CMCPDReportShell::GetInnerObj(void)
  {
//--- return result
   return(GetPointer(m_innerobj));
  }
//+------------------------------------------------------------------+
//| Markov chains class                                              |
//+------------------------------------------------------------------+
class CMarkovCPD
  {
private:
   //--- private method
   static void       MCPDInit(const int n,const int entrystate,const int exitstate,CMCPDState &s);
public:
   //--- constant
   static const double m_xtol;
   //--- constructor, destructor
                     CMarkovCPD(void);
                    ~CMarkovCPD(void);
   //--- public methods
   static void       MCPDCreate(const int n,CMCPDState &s);
   static void       MCPDCreateEntry(const int n,const int entrystate,CMCPDState &s);
   static void       MCPDCreateExit(const int n,const int exitstate,CMCPDState &s);
   static void       MCPDCreateEntryExit(const int n,const int entrystate,const int exitstate,CMCPDState &s);
   static void       MCPDAddTrack(CMCPDState &s,CMatrixDouble &xy,const int k);
   static void       MCPDSetEC(CMCPDState &s,CMatrixDouble &ec);
   static void       MCPDAddEC(CMCPDState &s,const int i,const int j,const double c);
   static void       MCPDSetBC(CMCPDState &s,CMatrixDouble &bndl,CMatrixDouble &bndu);
   static void       MCPDAddBC(CMCPDState &s,const int i,const int j,double bndl,double bndu);
   static void       MCPDSetLC(CMCPDState &s,CMatrixDouble &c,int &ct[],const int k);
   static void       MCPDSetTikhonovRegularizer(CMCPDState &s,const double v);
   static void       MCPDSetPrior(CMCPDState &s,CMatrixDouble &cpp);
   static void       MCPDSetPredictionWeights(CMCPDState &s,double &pw[]);
   static void       MCPDSolve(CMCPDState &s);
   static void       MCPDResults(CMCPDState &s,CMatrixDouble &p,CMCPDReport &rep);
  };
//+------------------------------------------------------------------+
//| Initialize constant                                              |
//+------------------------------------------------------------------+
const double CMarkovCPD::m_xtol=1.0E-8;
//+------------------------------------------------------------------+
//| Constructor without parameters                                   |
//+------------------------------------------------------------------+
CMarkovCPD::CMarkovCPD(void)
  {

  }
//+------------------------------------------------------------------+
//| Destructor                                                       |
//+------------------------------------------------------------------+
CMarkovCPD::~CMarkovCPD(void)
  {

  }
//+------------------------------------------------------------------+
//| DESCRIPTION:                                                     |
//| This function creates MCPD (Markov Chains for Population Data)   |
//| solver.                                                          |
//| This solver can be used to find transition matrix P for          |
//| N-dimensional prediction problem where transition from X[i] to   |
//|     X[i+1] is modelled as X[i+1] = P*X[i]                        |
//| where X[i] and X[i+1] are N-dimensional population vectors       |
//| (components of each X are non-negative), and P is a N*N          |
//| transition matrix (elements of   are non-negative, each column   |
//| sums to 1.0).                                                    |
//| Such models arise when when:                                     |
//| * there is some population of individuals                        |
//| * individuals can have different states                          |
//| * individuals can transit from one state to another              |
//| * population size is constant, i.e. there is no new individuals  |
//|   and no one leaves population                                   |
//| * you want to model transitions of individuals from one state    |
//|   into another                                                   |
//| USAGE:                                                           |
//| Here we give very brief outline of the MCPD. We strongly         |
//| recommend you to read examples in the ALGLIB Reference Manual    |
//| and to read ALGLIB User Guide on data analysis which is          |
//| available at http://www.alglib.net/dataanalysis/                 |
//| 1. User initializes algorithm state with MCPDCreate() call       |
//| 2. User adds one or more tracks -  sequences of states which     |
//|    describe evolution of a system being modelled from different  |
//|    starting conditions                                           |
//| 3. User may add optional boundary, equality and/or linear        |
//|    constraints on the coefficients of P by calling one of the    |
//|    following functions:                                          |
//|    * MCPDSetEC() to set equality constraints                     |
//|    * MCPDSetBC() to set bound constraints                        |
//|    * MCPDSetLC() to set linear constraints                       |
//| 4. Optionally, user may set custom weights for prediction errors |
//|    (by default, algorithm assigns non-equal, automatically chosen|
//|    weights for errors in the prediction of different components  |
//|    of X). It can be done with a call of                          |
//|    MCPDSetPredictionWeights() function.                          |
//| 5. User calls MCPDSolve() function which takes algorithm state   |
//|    and pointer (delegate, etc.) to callback function which       |
//|    calculates F/G.                                               |
//| 6. User calls MCPDResults() to get solution                      |
//| INPUT PARAMETERS:                                                |
//|     N       -   problem dimension, N>=1                          |
//| OUTPUT PARAMETERS:                                               |
//|     State   -   structure stores algorithm state                 |
//+------------------------------------------------------------------+
static void CMarkovCPD::MCPDCreate(const int n,CMCPDState &s)
  {
//--- check
   if(!CAp::Assert(n>=1,"MCPDCreate: N<1"))
      return;
//--- function call
   MCPDInit(n,-1,-1,s);
  }
//+------------------------------------------------------------------+
//| DESCRIPTION:                                                     |
//| This function is a specialized version of MCPDCreate() function, |
//| and we recommend  you  to read comments for this function for    |
//| general information about MCPD solver.                           |
//| This function creates MCPD (Markov Chains for Population Data)   |
//| solver for "Entry-state" model, i.e. model where transition from |
//| X[i] to X[i+1] is modelled as                                    |
//|     X[i+1] = P*X[i]                                              |
//| where                                                            |
//|     X[i] and X[i+1] are N-dimensional state vectors              |
//|     P is a N*N transition matrix                                 |
//| and  one  selected component of X[] is called "entry" state and  |
//| is treated in a special way:                                     |
//|     system state always transits from "entry" state to some      |
//|     another state                                                |
//|     system state can not transit from any state into "entry"     |
//|     state                                                        |
//| Such conditions basically mean that row of P which corresponds to|
//| "entry" state is zero.                                           |
//| Such models arise when:                                          |
//| * there is some population of individuals                        |
//| * individuals can have different states                          |
//| * individuals can transit from one state to another              |
//| * population size is NOT constant -  at every moment of time     |
//|   there is some (unpredictable) amount of "new" individuals,     |
//|   which can transit into one of the states at the next turn, but |
//|   still no one leaves population                                 |
//| * you want to model transitions of individuals from one state    |
//|   into another                                                   |
//| * but you do NOT want to predict amount of "new" individuals     |
//|   because it does not depends on individuals already present     |
//|   (hence system can not transit INTO entry state - it can only   |
//|   transit FROM it).                                              |
//| This model is discussed in more details in the ALGLIB User Guide |
//| (see http://www.alglib.net/dataanalysis/ for more data).         |
//| INPUT PARAMETERS:                                                |
//|     N       -   problem dimension, N>=2                          |
//|     EntryState- index of entry state, in 0..N-1                  |
//| OUTPUT PARAMETERS:                                               |
//|     State   -   structure stores algorithm state                 |
//+------------------------------------------------------------------+
static void CMarkovCPD::MCPDCreateEntry(const int n,const int entrystate,
                                        CMCPDState &s)
  {
//--- check
   if(!CAp::Assert(n>=2,__FUNCTION__+": N<2"))
      return;
//--- check
   if(!CAp::Assert(entrystate>=0,__FUNCTION__+": EntryState<0"))
      return;
//--- check
   if(!CAp::Assert(entrystate<n,__FUNCTION__+": EntryState>=N"))
      return;
//--- function call
   MCPDInit(n,entrystate,-1,s);
  }
//+------------------------------------------------------------------+
//| DESCRIPTION:                                                     |
//| This function is a specialized version of MCPDCreate() function, |
//| and we recommend  you  to read comments for this function for    |
//| general information about MCPD solver.                           |
//| This function creates MCPD (Markov Chains for Population Data)   |
//| solver for "Exit-state" model, i.e. model where transition from  |
//| X[i] to X[i+1] is modelled as                                    |
//|     X[i+1] = P*X[i]                                              |
//| where                                                            |
//|     X[i] and X[i+1] are N-dimensional state vectors              |
//|     P is a N*N transition matrix                                 |
//| and  one  selected component of X[] is called "exit" state and   |
//| is treated in a special way:                                     |
//|     system state can transit from any state into "exit" state    |
//|     system state can not transit from "exit" state into any other|
//|     state transition operator discards "exit" state (makes it    |
//|     zero at each turn)                                           |
//| Such conditions basically mean that column of P which            |
//| corresponds to "exit" state is zero. Multiplication by such P    |
//| may decrease sum of vector components.                           |
//| Such models arise when:                                          |
//| * there is some population of individuals                        |
//| * individuals can have different states                          |
//| * individuals can transit from one state to another              |
//| * population size is NOT constant - individuals can move into    |
//|   "exit" state and leave population at the next turn, but there  |
//|   are no new individuals                                         |
//| * amount of individuals which leave population can be predicted  |
//| * you want to model transitions of individuals from one state    |
//|   into another (including transitions into the "exit" state)     |
//| This model is discussed in more details in the ALGLIB User Guide |
//| (see http://www.alglib.net/dataanalysis/ for more data).         |
//| INPUT PARAMETERS:                                                |
//|     N       -   problem dimension, N>=2                          |
//|     ExitState-  index of exit state, in 0..N-1                   |
//| OUTPUT PARAMETERS:                                               |
//|     State   -   structure stores algorithm state                 |
//+------------------------------------------------------------------+
static void CMarkovCPD::MCPDCreateExit(const int n,const int exitstate,
                                       CMCPDState &s)
  {
//--- check
   if(!CAp::Assert(n>=2,__FUNCTION__+": N<2"))
      return;
//--- check
   if(!CAp::Assert(exitstate>=0,__FUNCTION__+": ExitState<0"))
      return;
//--- check
   if(!CAp::Assert(exitstate<n,__FUNCTION__+": ExitState>=N"))
      return;
//--- function call
   MCPDInit(n,-1,exitstate,s);
  }
//+------------------------------------------------------------------+
//| DESCRIPTION:                                                     |
//| This function is a specialized version of MCPDCreate() function, |
//| and we recommend you to read comments for this function for      |
//| general information about MCPD solver.                           |
//| This function creates MCPD (Markov Chains for Population Data)   |
//| solver for "Entry-Exit-states" model, i.e. model where transition|
//| from X[i] to X[i+1] is modelled as                               |
//|     X[i+1] = P*X[i]                                              |
//| where                                                            |
//|     X[i] and X[i+1] are N-dimensional state vectors              |
//|     P is a N*N transition matrix                                 |
//| one selected component of X[] is called "entry" state and is a   |
//| treated in special way:                                          |
//|     system state always transits from "entry" state to some      |
//|     another state                                                |
//|     system state can not transit from any state into "entry"     |
//|     state                                                        |
//| and another one component of X[] is called "exit" state and is   |
//| treated in a special way too:                                    |
//|     system state can transit from any state into "exit" state    |
//|     system state can not transit from "exit" state into any other|
//|     state transition operator discards "exit" state (makes it    |
//|     zero at each turn)                                           |
//| Such conditions basically mean that:                             |
//|     row of P which corresponds to "entry" state is zero          |
//|     column of P which corresponds to "exit" state is zero        |
//| Multiplication by such P may decrease sum of vector components.  |
//| Such models arise when:                                          |
//| * there is some population of individuals                        |
//| * individuals can have different states                          |
//| * individuals can transit from one state to another              |
//| * population size is NOT constant                                |
//| * at every moment of time there is some (unpredictable) amount   |
//|   of "new" individuals, which can transit into one of the states |
//|   at the next turn                                               |
//| * some individuals can move (predictably) into "exit" state      |
//|   and leave population at the next turn                          |
//| * you want to model transitions of individuals from one state    |
//|   into another, including transitions from the "entry" state and |
//|   into the "exit" state.                                         |
//| * but you do NOT want to predict amount of "new" individuals     |
//|   because it does not depends on individuals already present     |
//|   (hence system can not transit INTO entry state - it can only   |
//|   transit FROM it).                                              |
//| This model is discussed  in  more  details  in  the ALGLIB User  |
//| Guide (see http://www.alglib.net/dataanalysis/ for more data).   |
//| INPUT PARAMETERS:                                                |
//|     N       -   problem dimension, N>=2                          |
//|     EntryState- index of entry state, in 0..N-1                  |
//|     ExitState-  index of exit state, in 0..N-1                   |
//| OUTPUT PARAMETERS:                                               |
//|     State   -   structure stores algorithm state                 |
//+------------------------------------------------------------------+
static void CMarkovCPD::MCPDCreateEntryExit(const int n,const int entrystate,
                                            const int exitstate,CMCPDState &s)
  {
//--- check
   if(!CAp::Assert(n>=2,__FUNCTION__+": N<2"))
      return;
//--- check
   if(!CAp::Assert(entrystate>=0,__FUNCTION__+": EntryState<0"))
      return;
//--- check
   if(!CAp::Assert(entrystate<n,__FUNCTION__+": EntryState>=N"))
      return;
//--- check
   if(!CAp::Assert(exitstate>=0,__FUNCTION__+": ExitState<0"))
      return;
//--- check
   if(!CAp::Assert(exitstate<n,__FUNCTION__+": ExitState>=N"))
      return;
//--- check
   if(!CAp::Assert(entrystate!=exitstate,__FUNCTION__+": EntryState=ExitState"))
      return;
//--- function call
   MCPDInit(n,entrystate,exitstate,s);
  }
//+------------------------------------------------------------------+
//| This function is used to add a track - sequence of system states |
//| at the different moments of its evolution.                       |
//| You may add one or several tracks to the MCPD solver. In case you|
//| have several tracks, they won't overwrite each other. For        |
//| example, if you pass two tracks, A1-A2-A3 (system at t=A+1, t=A+2|
//| and t=A+3) and B1-B2-B3, then solver will try to model           |
//| transitions from t=A+1 to t=A+2, t=A+2 to t=A+3, t=B+1 to t=B+2, |
//| t=B+2 to t=B+3. But it WONT mix these two tracks - i.e. it wont  |
//| try to model transition from t=A+3 to t=B+1.                     |
//| INPUT PARAMETERS:                                                |
//|     S       -   solver                                           |
//|     XY      -   track, array[K,N]:                               |
//|                 * I-th row is a state at t=I                     |
//|                 * elements of XY must be non-negative (exception |
//|                   will be thrown on negative elements)           |
//|     K       -   number of points in a track                      |
//|                 * if given, only leading K rows of XY are used   |
//|                 * if not given, automatically determined from    |
//|                   size of XY                                     |
//| NOTES:                                                           |
//| 1. Track may contain either proportional or population data:     |
//|    * with proportional data all rows of XY must sum to 1.0, i.e. |
//|      we have proportions instead of absolute population values   |
//|    * with population data rows of XY contain population counts   |
//|      and generally do not sum to 1.0 (although they still must be|
//|      non-negative)                                               |
//+------------------------------------------------------------------+
static void CMarkovCPD::MCPDAddTrack(CMCPDState &s,CMatrixDouble &xy,
                                     const int k)
  {
//--- create variables
   int    i=0;
   int    j=0;
   int    n=0;
   double s0=0;
   double s1=0;
//--- initialization
   n=s.m_n;
//--- check
   if(!CAp::Assert(k>=0,__FUNCTION__+": K<0"))
      return;
//--- check
   if(!CAp::Assert(CAp::Cols(xy)>=n,__FUNCTION__+": Cols(XY)<N"))
      return;
//--- check
   if(!CAp::Assert(CAp::Rows(xy)>=k,__FUNCTION__+": Rows(XY)<K"))
      return;
//--- check
   if(!CAp::Assert(CApServ::IsFiniteMatrix(xy,k,n),__FUNCTION__+": XY contains infinite or NaN elements"))
      return;
   for(i=0;i<=k-1;i++)
     {
      for(j=0;j<=n-1;j++)
        {
         //--- check
         if(!CAp::Assert((double)(xy[i][j])>=0.0,__FUNCTION__+": XY contains negative elements"))
            return;
        }
     }
//--- check
   if(k<2)
      return;
//--- check
   if(CAp::Rows(s.m_data)<s.m_npairs+k-1)
      CApServ::RMatrixResize(s.m_data,MathMax(2*CAp::Rows(s.m_data),s.m_npairs+k-1),2*n);
//--- calculation
   for(i=0;i<=k-2;i++)
     {
      s0=0;
      s1=0;
      for(j=0;j<=n-1;j++)
        {
         //--- check
         if(s.m_states[j]>=0)
            s0=s0+xy[i][j];
         //--- check
         if(s.m_states[j]<=0)
            s1=s1+xy[i+1][j];
        }
      //--- check
      if(s0>0.0 && s1>0.0)
        {
         for(j=0;j<=n-1;j++)
           {
            //--- check
            if(s.m_states[j]>=0)
               s.m_data[s.m_npairs].Set(j,xy[i][j]/s0);
            else
               s.m_data[s.m_npairs].Set(j,0.0);
            //--- check
            if(s.m_states[j]<=0)
               s.m_data[s.m_npairs].Set(n+j,xy[i+1][j]/s1);
            else
               s.m_data[s.m_npairs].Set(n+j,0.0);
           }
         //--- change value
         s.m_npairs=s.m_npairs+1;
        }
     }
  }
//+------------------------------------------------------------------+
//| This function is used to add equality constraints on the elements|
//| of the transition matrix P.                                      |
//| MCPD solver has four types of constraints which can be placed    |
//| on P:                                                            |
//| * user-specified equality constraints (optional)                 |
//| * user-specified bound constraints (optional)                    |
//| * user-specified general linear constraints (optional)           |
//| * basic constraints (always present):                            |
//|   * non-negativity: P[i,j]>=0                                    |
//|   * consistency: every column of P sums to 1.0                   |
//| Final constraints which are passed to the underlying optimizer   |
//| are calculated as intersection of all present constraints. For   |
//| example, you may specify boundary constraint on P[0,0] and       |
//| equality one:                                                    |
//|     0.1<=P[0,0]<=0.9                                             |
//|     P[0,0]=0.5                                                   |
//| Such combination of constraints will be silently reduced to their|
//| intersection, which is P[0,0]=0.5.                               |
//| This function can be used to place equality constraints on       |
//| arbitrary subset of elements of P. Set of constraints is         |
//| specified by EC, which may contain either NAN's or finite numbers|
//| from [0,1]. NAN denotes absence of constraint, finite number     |
//| denotes equality constraint on specific element of P.            |
//| You can also use MCPDAddEC() function which allows to ADD        |
//| equality constraint for one element of P without changing        |
//| constraints for other elements.                                  |
//| These functions (MCPDSetEC and MCPDAddEC) interact as follows:   |
//| * there is internal matrix of equality constraints which is      |
//|   stored in the MCPD solver                                      |
//| * MCPDSetEC() replaces this matrix by another one (SET)          |
//| * MCPDAddEC() modifies one element of this matrix and leaves     |
//|   other ones unchanged (ADD)                                     |
//| * thus MCPDAddEC() call preserves all modifications done by      |
//|   previous calls, while MCPDSetEC() completely discards all      |
//|   changes done to the equality constraints.                      |
//| INPUT PARAMETERS:                                                |
//|     S       -   solver                                           |
//|     EC      -   equality constraints, array[N,N]. Elements of EC |
//|                 can be either NAN's or finite numbers from [0,1].|
//|                 NAN denotes absence of constraints, while finite |
//|                 value denotes equality constraint on the         |
//|                 corresponding element of P.                      |
//| NOTES:                                                           |
//| 1. infinite values of EC will lead to exception being thrown.    |
//| Values less than 0.0 or greater than 1.0 will lead to error code |
//| being returned after call to MCPDSolve().                        |
//+------------------------------------------------------------------+
static void CMarkovCPD::MCPDSetEC(CMCPDState &s,CMatrixDouble &ec)
  {
//--- create variables
   int i=0;
   int j=0;
   int n=0;
//--- initialization
   n=s.m_n;
//--- check
   if(!CAp::Assert(CAp::Cols(ec)>=n,__FUNCTION__+": Cols(EC)<N"))
      return;
//--- check
   if(!CAp::Assert(CAp::Rows(ec)>=n,__FUNCTION__+": Rows(EC)<N"))
      return;
//--- calculation
   for(i=0;i<=n-1;i++)
     {
      for(j=0;j<=n-1;j++)
        {
         //--- check
         if(!CAp::Assert(CMath::IsFinite(ec[i][j]) || CInfOrNaN::IsNaN(ec[i][j]),"MCPDSetEC: EC containts infinite elements"))
            return;
         s.m_ec[i].Set(j,ec[i][j]);
        }
     }
  }
//+------------------------------------------------------------------+
//| This function is used to add equality constraints on the elements|
//| of the transition matrix P.                                      |
//| MCPD solver has four types of constraints which can be placed    |
//| on P:                                                            |
//| * user-specified equality constraints (optional)                 |
//| * user-specified bound constraints (optional)                    |
//| * user-specified general linear constraints (optional)           |
//| * basic constraints (always present):                            |
//|   * non-negativity: P[i,j]>=0                                    |
//|   * consistency: every column of P sums to 1.0                   |
//| Final constraints which are passed to the underlying optimizer   |
//| are calculated as intersection of all present constraints. For   |
//| example, you may specify boundary constraint on P[0,0] and       |
//| equality one:                                                    |
//|     0.1<=P[0,0]<=0.9                                             |
//|     P[0,0]=0.5                                                   |
//| Such combination of constraints will be silently reduced to their|
//| intersection, which is P[0,0]=0.5.                               |
//| This function can be used to ADD equality constraint for one     |
//| element of P without changing constraints for other elements.    |
//| You can also use MCPDSetEC() function which allows you to specify|
//| arbitrary set of equality constraints in one call.               |
//| These functions (MCPDSetEC and MCPDAddEC) interact as follows:   |
//| * there is internal matrix of equality constraints which is      |
//|   stored in the MCPD solver                                      |
//| * MCPDSetEC() replaces this matrix by another one (SET)          |
//| * MCPDAddEC() modifies one element of this matrix and leaves     |
//|   other ones unchanged (ADD)                                     |
//| * thus MCPDAddEC() call preserves all modifications done by      |
//|   previous calls, while MCPDSetEC() completely discards all      |
//|   changes done to the equality constraints.                      |
//| INPUT PARAMETERS:                                                |
//|     S       -   solver                                           |
//|     I       -   row index of element being constrained           |
//|     J       -   column index of element being constrained        |
//|     C       -   value (constraint for P[I,J]). Can be either NAN |
//|                 (no constraint) or finite value from [0,1].      |
//| NOTES:                                                           |
//| 1. infinite values of C will lead to exception being thrown.     |
//| Values less than 0.0 or greater than 1.0 will lead to error code |
//| being returned after call to MCPDSolve().                        |
//+------------------------------------------------------------------+
static void CMarkovCPD::MCPDAddEC(CMCPDState &s,const int i,const int j,
                                  const double c)
  {
//--- check
   if(!CAp::Assert(i>=0,__FUNCTION__+": I<0"))
      return;
//--- check
   if(!CAp::Assert(i<s.m_n,__FUNCTION__+": I>=N"))
      return;
//--- check
   if(!CAp::Assert(j>=0,__FUNCTION__+": J<0"))
      return;
//--- check
   if(!CAp::Assert(j<s.m_n,__FUNCTION__+": J>=N"))
      return;
//--- check
   if(!CAp::Assert(CInfOrNaN::IsNaN(c) || CMath::IsFinite(c),"MCPDAddEC: C is not finite number or NAN"))
      return;
   s.m_ec[i].Set(j,c);
  }
//+------------------------------------------------------------------+
//| This function is used to add bound constraints on the elements   |
//| of the transition matrix P.                                      |
//| MCPD solver has four types of constraints which can be placed    |
//| on P:                                                            |
//| * user-specified equality constraints (optional)                 |
//| * user-specified bound constraints (optional)                    |
//| * user-specified general linear constraints (optional)           |
//| * basic constraints (always present):                            |
//|   * non-negativity: P[i,j]>=0                                    |
//|   * consistency: every column of P sums to 1.0                   |
//| Final constraints which are passed to the underlying optimizer   |
//| are calculated as intersection of all present constraints. For   |
//| example, you may specify boundary constraint on P[0,0] and       |
//| equality one:                                                    |
//|     0.1<=P[0,0]<=0.9                                             |
//|     P[0,0]=0.5                                                   |
//| Such combination of constraints will be silently reduced to their|
//| intersection, which is P[0,0]=0.5.                               |
//| This function can be used to place bound constraints on arbitrary|
//| subset of elements of P. Set of constraints is specified by      |
//| BndL/BndU matrices, which may contain arbitrary combination of   |
//| finite numbers or infinities (like -INF<x<=0.5 or 0.1<=x<+INF).  |
//| You can also use MCPDAddBC() function which allows to ADD bound  |
//| constraint for one element of P without changing constraints for |
//| other elements.                                                  |
//| These functions (MCPDSetBC and MCPDAddBC) interact as follows:   |
//| * there is internal matrix of bound constraints which is stored  |
//|   in the MCPD solver                                             |
//| * MCPDSetBC() replaces this matrix by another one (SET)          |
//| * MCPDAddBC() modifies one element of this matrix and leaves     |
//|   other ones unchanged (ADD)                                     |
//| * thus MCPDAddBC() call preserves all modifications done by      |
//|   previous calls, while MCPDSetBC() completely discards all      |
//|   changes done to the equality constraints.                      |
//| INPUT PARAMETERS:                                                |
//|     S       -   solver                                           |
//|     BndL    -   lower bounds constraints, array[N,N]. Elements of|
//|                 BndL can be finite numbers or -INF.              |
//|     BndU    -   upper bounds constraints, array[N,N]. Elements of|
//|                 BndU can be finite numbers or +INF.              |
//+------------------------------------------------------------------+
static void CMarkovCPD::MCPDSetBC(CMCPDState &s,CMatrixDouble &bndl,
                                  CMatrixDouble &bndu)
  {
//--- create variables
   int i=0;
   int j=0;
   int n=0;
//--- initialization
   n=s.m_n;
//--- check
   if(!CAp::Assert(CAp::Cols(bndl)>=n,__FUNCTION__+": Cols(BndL)<N"))
      return;
//--- check
   if(!CAp::Assert(CAp::Rows(bndl)>=n,__FUNCTION__+": Rows(BndL)<N"))
      return;
//--- check
   if(!CAp::Assert(CAp::Cols(bndu)>=n,__FUNCTION__+": Cols(BndU)<N"))
      return;
//--- check
   if(!CAp::Assert(CAp::Rows(bndu)>=n,__FUNCTION__+": Rows(BndU)<N"))
      return;
//--- calculation
   for(i=0;i<=n-1;i++)
     {
      for(j=0;j<=n-1;j++)
        {
         //--- check
         if(!CAp::Assert(CMath::IsFinite(bndl[i][j]) || CInfOrNaN::IsNegativeInfinity(bndl[i][j]),"MCPDSetBC: BndL containts NAN or +INF"))
            return;
         //--- check
         if(!CAp::Assert(CMath::IsFinite(bndu[i][j]) || CInfOrNaN::IsPositiveInfinity(bndu[i][j]),"MCPDSetBC: BndU containts NAN or -INF"))
            return;
         //--- change values
         s.m_bndl[i].Set(j,bndl[i][j]);
         s.m_bndu[i].Set(j,bndu[i][j]);
        }
     }
  }
//+------------------------------------------------------------------+
//| This function is used to add bound constraints on the elements   |
//| of the transition matrix P.                                      |
//| MCPD solver has four types of constraints which can be placed    |
//| on P:                                                            |
//| * user-specified equality constraints (optional)                 |
//| * user-specified bound constraints (optional)                    |
//| * user-specified general linear constraints (optional)           |
//| * basic constraints (always present):                            |
//|   * non-negativity: P[i,j]>=0                                    |
//|   * consistency: every column of P sums to 1.0                   |
//| Final constraints which are passed to the underlying optimizer   |
//| are calculated as intersection of all present constraints. For   |
//| example, you may specify boundary constraint on P[0,0] and       |
//| equality one:                                                    |
//|     0.1<=P[0,0]<=0.9                                             |
//|     P[0,0]=0.5                                                   |
//| Such combination of constraints will be silently reduced to their|
//| intersection, which is P[0,0]=0.5.                               |
//| This function can be used to ADD bound constraint for one element|
//| of P without changing constraints for other elements.            |
//| You can also use MCPDSetBC() function which allows to place bound|
//| constraints on arbitrary subset of elements of P. Set of         |
//| constraints is specified  by  BndL/BndU matrices, which may      |
//| contain arbitrary combination of finite numbers or infinities    |
//| (like -INF<x<=0.5 or 0.1<=x<+INF).                               |
//| These functions (MCPDSetBC and MCPDAddBC) interact as follows:   |
//| * there is internal matrix of bound constraints which is stored  |
//|   in the MCPD solver                                             |
//| * MCPDSetBC() replaces this matrix by another one (SET)          |
//| * MCPDAddBC() modifies one element of this matrix and leaves     |
//|   other ones unchanged (ADD)                                     |
//| * thus MCPDAddBC() call preserves all modifications done by      |
//|   previous calls, while MCPDSetBC() completely discards all      |
//|   changes done to the equality constraints.                      |
//| INPUT PARAMETERS:                                                |
//|     S       -   solver                                           |
//|     I       -   row index of element being constrained           |
//|     J       -   column index of element being constrained        |
//|     BndL    -   lower bound                                      |
//|     BndU    -   upper bound                                      |
//+------------------------------------------------------------------+
static void CMarkovCPD::MCPDAddBC(CMCPDState &s,const int i,const int j,
                                  double bndl,double bndu)
  {
//--- check
   if(!CAp::Assert(i>=0,__FUNCTION__+": I<0"))
      return;
//--- check
   if(!CAp::Assert(i<s.m_n,__FUNCTION__+": I>=N"))
      return;
//--- check
   if(!CAp::Assert(j>=0,__FUNCTION__+": J<0"))
      return;
//--- check
   if(!CAp::Assert(j<s.m_n,__FUNCTION__+": J>=N"))
      return;
//--- check
   if(!CAp::Assert(CMath::IsFinite(bndl) || CInfOrNaN::IsNegativeInfinity(bndl),"MCPDAddBC: BndL is NAN or +INF"))
      return;
//--- check
   if(!CAp::Assert(CMath::IsFinite(bndu) || CInfOrNaN::IsPositiveInfinity(bndu),"MCPDAddBC: BndU is NAN or -INF"))
      return;
//--- change values
   s.m_bndl[i].Set(j,bndl);
   s.m_bndu[i].Set(j,bndu);
  }
//+------------------------------------------------------------------+
//| This function is used to set linear equality/inequality          |
//| constraints on the elements of the transition matrix P.          |
//| This function can be used to set one or several general linear   |
//| constraints on the elements of P. Two types of constraints are   |
//| supported:                                                       |
//| * equality constraints                                           |
//| * inequality constraints (both less-or-equal and                 |
//|   greater-or-equal)                                              |
//| Coefficients of constraints are specified by matrix C (one of the|
//| parameters). One row of C corresponds to one constraint.         |
//| Because transition matrix P has N*N elements, we need N*N columns|
//| to store all coefficients  (they  are  stored row by row), and   |
//| one more column to store right part - hence C has N*N+1 columns. |
//| Constraint kind is stored in the CT array.                       |
//| Thus, I-th linear constraint is                                  |
//|     P[0,0]*C[I,0] + P[0,1]*C[I,1] + .. + P[0,N-1]*C[I,N-1] +     |
//|         + P[1,0]*C[I,N] + P[1,1]*C[I,N+1] + ... +                |
//|         + P[N-1,N-1]*C[I,N*N-1]  ?=?  C[I,N*N]                   |
//| where ?=? can be either "=" (CT[i]=0), "<=" (CT[i]<0) or ">="    |
//| (CT[i]>0).                                                       |
//| Your constraint may involve only some subset of P (less than N*N |
//| elements).                                                       |
//| For example it can be something like                             |
//|     P[0,0] + P[0,1] = 0.5                                        |
//| In this case you still should pass matrix  with N*N+1 columns,   |
//| but all its elements (except for C[0,0], C[0,1] and C[0,N*N-1])  |
//| will be zero.                                                    |
//| INPUT PARAMETERS:                                                |
//|     S       -   solver                                           |
//|     C       -   array[K,N*N+1] - coefficients of constraints     |
//|                 (see above for complete description)             |
//|     CT      -   array[K] - constraint types                      |
//|                 (see above for complete description)             |
//|     K       -   number of equality/inequality constraints, K>=0: |
//|                 * if given, only leading K elements of C/CT are  |
//|                   used                                           |
//|                 * if not given, automatically determined from    |
//|                   sizes of C/CT                                  |
//+------------------------------------------------------------------+
static void CMarkovCPD::MCPDSetLC(CMCPDState &s,CMatrixDouble &c,int &ct[],
                                  const int k)
  {
//--- create variables
   int i=0;
   int j=0;
   int n=0;
//--- initialization
   n=s.m_n;
//--- check
   if(!CAp::Assert(CAp::Cols(c)>=n*n+1,__FUNCTION__+": Cols(C)<N*N+1"))
      return;
//--- check
   if(!CAp::Assert(CAp::Rows(c)>=k,__FUNCTION__+": Rows(C)<K"))
      return;
//--- check
   if(!CAp::Assert(CAp::Len(ct)>=k,__FUNCTION__+": Len(CT)<K"))
      return;
//--- check
   if(!CAp::Assert(CApServ::IsFiniteMatrix(c,k,n*n+1),__FUNCTION__+": C contains infinite or NaN values!"))
      return;
//--- function call
   CApServ::RMatrixSetLengthAtLeast(s.m_c,k,n*n+1);
//--- function call
   CApServ::IVectorSetLengthAtLeast(s.m_ct,k);
//--- calculation
   for(i=0;i<=k-1;i++)
     {
      for(j=0;j<=n*n;j++)
         s.m_c[i].Set(j,c[i][j]);
      s.m_ct[i]=ct[i];
     }
   s.m_ccnt=k;
  }
//+------------------------------------------------------------------+
//| This function allows to tune amount of Tikhonov regularization   |
//| being applied to your problem.                                   |
//| By default, regularizing term is equal to r*||P-prior_P||^2,     |
//| where r is a small non-zero value,  P is transition matrix,      |
//| prior_P is identity matrix, ||X||^2 is a sum of squared elements |
//| of X.                                                            |
//| This function allows you to change coefficient r. You can also   |
//| change prior values with MCPDSetPrior() function.                |
//| INPUT PARAMETERS:                                                |
//|     S      -   solver                                            |
//|     V      -   regularization  coefficient, finite non-negative  |
//|                value. It is not recommended to specify zero      |
//|                value unless you are pretty sure that you want it.|
//+------------------------------------------------------------------+
static void CMarkovCPD::MCPDSetTikhonovRegularizer(CMCPDState &s,const double v)
  {
//--- check
   if(!CAp::Assert(CMath::IsFinite(v),__FUNCTION__+": V is infinite or NAN"))
      return;
//--- check
   if(!CAp::Assert(v>=0.0,__FUNCTION__+": V is less than zero"))
      return;
//--- change value
   s.m_regterm=v;
  }
//+------------------------------------------------------------------+
//| This function allows to set prior values used for regularization |
//| of your problem.                                                 |
//| By default, regularizing term is equal to r*||P-prior_P||^2,     |
//| where r is a small non-zero value,  P is transition matrix,      |
//| prior_P is identity matrix, ||X||^2 is a sum of squared elements |
//| of X.                                                            |
//| This function allows you to change prior values prior_P. You can |
//| also change r with MCPDSetTikhonovRegularizer() function.        |
//| INPUT PARAMETERS:                                                |
//|     S       -   solver                                           |
//|     PP      -   array[N,N], matrix of prior values:              |
//|                 1. elements must be real numbers from [0,1]      |
//|                 2. columns must sum to 1.0.                      |
//|                 First property is checked (exception is thrown   |
//|                 otherwise), while second one is not              |
//|                 checked/enforced.                                |
//+------------------------------------------------------------------+
static void CMarkovCPD::MCPDSetPrior(CMCPDState &s,CMatrixDouble &cpp)
  {
//--- create variables
   int i=0;
   int j=0;
   int n=0;
//--- create copy of matrices
   CMatrixDouble pp;
   pp=cpp;
//--- initialization
   n=s.m_n;
//--- check
   if(!CAp::Assert(CAp::Cols(pp)>=n,__FUNCTION__+": Cols(PP)<N"))
      return;
//--- check
   if(!CAp::Assert(CAp::Rows(pp)>=n,__FUNCTION__+": Rows(PP)<K"))
      return;
//--- calculation
   for(i=0;i<=n-1;i++)
     {
      for(j=0;j<=n-1;j++)
        {
         //--- check
         if(!CAp::Assert(CMath::IsFinite(pp[i][j]),__FUNCTION__+": PP containts infinite elements"))
            return;
         //--- check
         if(!CAp::Assert(pp[i][j]>=0.0 && pp[i][j]<=1.0,__FUNCTION__+": PP[i][j] is less than 0.0 or greater than 1.0"))
            return;
         //--- change value
         s.m_priorp[i].Set(j,pp[i][j]);
        }
     }
  }
//+------------------------------------------------------------------+
//| This function is used to change prediction weights               |
//| MCPD solver scales prediction errors as follows                  |
//|     Error(P) = ||W*(y-P*x)||^2                                   |
//| where                                                            |
//|     x is a system state at time t                                |
//|     y is a system state at time t+1                              |
//|     P is a transition matrix                                     |
//|     W is a diagonal scaling matrix                               |
//| By default, weights are chosen in order to minimize relative     |
//| prediction error instead of absolute one. For example, if one    |
//| component of state is about 0.5 in magnitude and another one is  |
//| about 0.05, then algorithm will make corresponding weights equal |
//| to 2.0 and 20.0.                                                 |
//| INPUT PARAMETERS:                                                |
//|     S       -   solver                                           |
//|     PW      -   array[N], weights:                               |
//|                 * must be non-negative values (exception will be |
//|                 thrown otherwise)                                |
//|                 * zero values will be replaced by automatically  |
//|                 chosen values                                    |
//+------------------------------------------------------------------+
static void CMarkovCPD::MCPDSetPredictionWeights(CMCPDState &s,double &pw[])
  {
//--- create variables
   int i=0;
   int n=0;
//--- initialization
   n=s.m_n;
//--- check
   if(!CAp::Assert(CAp::Len(pw)>=n,__FUNCTION__+": Length(PW)<N"))
      return;
//--- calculation
   for(i=0;i<=n-1;i++)
     {
      //--- check
      if(!CAp::Assert(CMath::IsFinite(pw[i]),__FUNCTION__+": PW containts infinite or NAN elements"))
         return;
      //--- check
      if(!CAp::Assert(pw[i]>=0.0,__FUNCTION__+": PW containts negative elements"))
         return;
      //--- change value
      s.m_pw[i]=pw[i];
     }
  }
//+------------------------------------------------------------------+
//| This function is used to start solution of the MCPD problem.     |
//| After return from this function, you can use MCPDResults() to get|
//| solution and completion code.                                    |
//+------------------------------------------------------------------+
static void CMarkovCPD::MCPDSolve(CMCPDState &s)
  {
//--- create variables
   int    n=0;
   int    npairs=0;
   int    ccnt=0;
   int    i=0;
   int    j=0;
   int    k=0;
   int    k2=0;
   double v=0;
   double vv=0;
   int    i_=0;
   int    i1_=0;
//--- initialization
   n=s.m_n;
   npairs=s.m_npairs;
//--- init fields of S
   s.m_repterminationtype=0;
   s.m_repinneriterationscount=0;
   s.m_repouteriterationscount=0;
   s.m_repnfev=0;
   for(k=0;k<=n-1;k++)
     {
      for(k2=0;k2<=n-1;k2++)
         s.m_p[k].Set(k2,CInfOrNaN::NaN());
     }
//--- Generate "effective" weights for prediction and calculate preconditioner
   for(i=0;i<=n-1;i++)
     {
      //--- check
      if(s.m_pw[i]==0.0)
        {
         //--- change values
         v=0;
         k=0;
         for(j=0;j<=npairs-1;j++)
           {
            //--- check
            if(s.m_data[j][n+i]!=0.0)
              {
               v=v+s.m_data[j][n+i];
               k=k+1;
              }
           }
         //--- check
         if(k!=0)
            s.m_effectivew[i]=k/v;
         else
            s.m_effectivew[i]=1.0;
        }
      else
         s.m_effectivew[i]=s.m_pw[i];
     }
//--- calculation
   for(i=0;i<=n-1;i++)
     {
      for(j=0;j<=n-1;j++)
         s.m_h[i*n+j]=2*s.m_regterm;
     }
//--- calculation
   for(k=0;k<=npairs-1;k++)
     {
      for(i=0;i<=n-1;i++)
        {
         for(j=0;j<=n-1;j++)
            s.m_h[i*n+j]=s.m_h[i*n+j]+2*CMath::Sqr(s.m_effectivew[i])*CMath::Sqr(s.m_data[k][j]);
        }
     }
//--- calculation
   for(i=0;i<=n-1;i++)
     {
      for(j=0;j<=n-1;j++)
        {
         //--- check
         if(s.m_h[i*n+j]==0.0)
            s.m_h[i*n+j]=1;
        }
     }
//--- Generate "effective" BndL/BndU
   for(i=0;i<=n-1;i++)
     {
      for(j=0;j<=n-1;j++)
        {
         //--- Set default boundary constraints.
         //--- Lower bound is always zero,upper bound is calculated
         //--- with respect to entry/exit states.
         s.m_effectivebndl[i*n+j]=0.0;
         //--- check
         if(s.m_states[i]>0 || s.m_states[j]<0)
            s.m_effectivebndu[i*n+j]=0.0;
         else
            s.m_effectivebndu[i*n+j]=1.0;
         //--- Calculate intersection of the default and user-specified bound constraints.
         //--- This code checks consistency of such combination.
         if(CMath::IsFinite(s.m_bndl[i][j]) && s.m_bndl[i][j]>s.m_effectivebndl[i*n+j])
            s.m_effectivebndl[i*n+j]=s.m_bndl[i][j];
         //--- check
         if(CMath::IsFinite(s.m_bndu[i][j]) && s.m_bndu[i][j]<s.m_effectivebndu[i*n+j])
            s.m_effectivebndu[i*n+j]=s.m_bndu[i][j];
         //--- check
         if(s.m_effectivebndl[i*n+j]>s.m_effectivebndu[i*n+j])
           {
            s.m_repterminationtype=-3;
            return;
           }
         //--- Calculate intersection of the effective bound constraints
         //--- and user-specified equality constraints.
         //--- This code checks consistency of such combination.
         if(CMath::IsFinite(s.m_ec[i][j]))
           {
            //--- check
            if(s.m_ec[i][j]<s.m_effectivebndl[i*n+j] || s.m_ec[i][j]>s.m_effectivebndu[i*n+j])
              {
               s.m_repterminationtype=-3;
               return;
              }
            //--- change values
            s.m_effectivebndl[i*n+j]=s.m_ec[i][j];
            s.m_effectivebndu[i*n+j]=s.m_ec[i][j];
           }
        }
     }
//--- Generate linear constraints:
//--- * "default" sums-to-one constraints (not generated for "exit" states)
   CApServ::RMatrixSetLengthAtLeast(s.m_effectivec,s.m_ccnt+n,n*n+1);
//--- function call
   CApServ::IVectorSetLengthAtLeast(s.m_effectivect,s.m_ccnt+n);
   ccnt=s.m_ccnt;
   for(i=0;i<=s.m_ccnt-1;i++)
     {
      for(j=0;j<=n*n;j++)
         s.m_effectivec[i].Set(j,s.m_c[i][j]);
      s.m_effectivect[i]=s.m_ct[i];
     }
//--- calculation
   for(i=0;i<=n-1;i++)
     {
      //--- check
      if(s.m_states[i]>=0)
        {
         for(k=0;k<=n*n-1;k++)
            s.m_effectivec[ccnt].Set(k,0);
         for(k=0;k<=n-1;k++)
            s.m_effectivec[ccnt].Set(k*n+i,1);
         //--- change values
         s.m_effectivec[ccnt].Set(n*n,1.0);
         s.m_effectivect[ccnt]=0;
         ccnt=ccnt+1;
        }
     }
//--- create optimizer
   for(i=0;i<=n-1;i++)
     {
      for(j=0;j<=n-1;j++)
         s.m_tmpp[i*n+j]=1.0/(double)n;
     }
//--- function calls
   CMinBLEIC::MinBLEICRestartFrom(s.m_bs,s.m_tmpp);
   CMinBLEIC::MinBLEICSetBC(s.m_bs,s.m_effectivebndl,s.m_effectivebndu);
   CMinBLEIC::MinBLEICSetLC(s.m_bs,s.m_effectivec,s.m_effectivect,ccnt);
   CMinBLEIC::MinBLEICSetInnerCond(s.m_bs,0,0,m_xtol);
   CMinBLEIC::MinBLEICSetOuterCond(s.m_bs,m_xtol,1.0E-5);
   CMinBLEIC::MinBLEICSetPrecDiag(s.m_bs,s.m_h);
//--- solve problem
   while(CMinBLEIC::MinBLEICIteration(s.m_bs))
     {
      //--- check
      if(!CAp::Assert(s.m_bs.m_needfg,__FUNCTION__+": internal error"))
         return;
      //--- check
      if(s.m_bs.m_needfg)
        {
         //--- Calculate regularization term
         s.m_bs.m_f=0.0;
         vv=s.m_regterm;
         for(i=0;i<=n-1;i++)
           {
            for(j=0;j<=n-1;j++)
              {
               s.m_bs.m_f=s.m_bs.m_f+vv*CMath::Sqr(s.m_bs.m_x[i*n+j]-s.m_priorp[i][j]);
               s.m_bs.m_g[i*n+j]=2*vv*(s.m_bs.m_x[i*n+j]-s.m_priorp[i][j]);
              }
           }
         //--- calculate prediction error/gradient for K-th pair
         for(k=0;k<=npairs-1;k++)
           {
            for(i=0;i<=n-1;i++)
              {
               i1_=(0)-(i*n);
               v=0.0;
               for(i_=i*n;i_<=i*n+n-1;i_++)
                  v+=s.m_bs.m_x[i_]*s.m_data[k][i_+i1_];
               vv=s.m_effectivew[i];
               s.m_bs.m_f=s.m_bs.m_f+CMath::Sqr(vv*(v-s.m_data[k][n+i]));
               for(j=0;j<=n-1;j++)
                 {
                  s.m_bs.m_g[i*n+j]=s.m_bs.m_g[i*n+j]+2*vv*vv*(v-s.m_data[k][n+i])*s.m_data[k][j];
                 }
              }
           }
         //--- continue
         continue;
        }
     }
//--- function call
   CMinBLEIC::MinBLEICResultsBuf(s.m_bs,s.m_tmpp,s.m_br);
   for(i=0;i<=n-1;i++)
     {
      for(j=0;j<=n-1;j++)
         s.m_p[i].Set(j,s.m_tmpp[i*n+j]);
     }
//--- change values
   s.m_repterminationtype=s.m_br.m_terminationtype;
   s.m_repinneriterationscount=s.m_br.m_inneriterationscount;
   s.m_repouteriterationscount=s.m_br.m_outeriterationscount;
   s.m_repnfev=s.m_br.m_nfev;
  }
//+------------------------------------------------------------------+
//| MCPD results                                                     |
//| INPUT PARAMETERS:                                                |
//|     State   -   algorithm state                                  |
//| OUTPUT PARAMETERS:                                               |
//|     P      -   array[N,N], transition matrix                     |
//|     Rep    -   optimization report. You should check Rep.        |
//|                TerminationType in order to distinguish successful|
//|                termination from unsuccessful one. Speaking short,|
//|                positive values denote success, negative ones are |
//|                failures. More information about fields of this   |
//|                structure  can befound in the comments on         |
//|                MCPDReport datatype.                              |
//+------------------------------------------------------------------+
static void CMarkovCPD::MCPDResults(CMCPDState &s,CMatrixDouble &p,
                                    CMCPDReport &rep)
  {
//--- create variables
   int i=0;
   int j=0;
//--- allocation
   p.Resize(s.m_n,s.m_n);
//--- copy
   for(i=0;i<=s.m_n-1;i++)
     {
      for(j=0;j<=s.m_n-1;j++)
         p[i].Set(j,s.m_p[i][j]);
     }
//--- change values
   rep.m_terminationtype=s.m_repterminationtype;
   rep.m_inneriterationscount=s.m_repinneriterationscount;
   rep.m_outeriterationscount=s.m_repouteriterationscount;
   rep.m_nfev=s.m_repnfev;
  }
//+------------------------------------------------------------------+
//| Internal initialization function                                 |
//+------------------------------------------------------------------+
static void CMarkovCPD::MCPDInit(const int n,const int entrystate,
                                 const int exitstate,CMCPDState &s)
  {
//--- create variables
   int i=0;
   int j=0;
//--- check
   if(!CAp::Assert(n>=1,__FUNCTION__+": N<1"))
      return;
//--- initialization
   s.m_n=n;
//--- allocation
   ArrayResizeAL(s.m_states,n);
   for(i=0;i<=n-1;i++)
      s.m_states[i]=0;
//--- check
   if(entrystate>=0)
      s.m_states[entrystate]=1;
//--- check
   if(exitstate>=0)
      s.m_states[exitstate]=-1;
//--- initialization
   s.m_npairs=0;
   s.m_regterm=1.0E-8;
   s.m_ccnt=0;
//--- allocation
   s.m_p.Resize(n,n);
   s.m_ec.Resize(n,n);
   s.m_bndl.Resize(n,n);
   s.m_bndu.Resize(n,n);
   ArrayResizeAL(s.m_pw,n);
   s.m_priorp.Resize(n,n);
   ArrayResizeAL(s.m_tmpp,n*n);
   ArrayResizeAL(s.m_effectivew,n);
   ArrayResizeAL(s.m_effectivebndl,n*n);
   ArrayResizeAL(s.m_effectivebndu,n*n);
   ArrayResizeAL(s.m_h,n*n);
//--- change values
   for(i=0;i<=n-1;i++)
     {
      for(j=0;j<=n-1;j++)
        {
         s.m_p[i].Set(j,0.0);
         s.m_priorp[i].Set(j,0.0);
         s.m_bndl[i].Set(j,CInfOrNaN::NegativeInfinity());
         s.m_bndu[i].Set(j,CInfOrNaN::PositiveInfinity());
         s.m_ec[i].Set(j,CInfOrNaN::NaN());
        }
      s.m_pw[i]=0.0;
      s.m_priorp[i].Set(i,1.0);
     }
//--- allocation
   s.m_data.Resize(1,2*n);
   for(i=0;i<=2*n-1;i++)
      s.m_data[0].Set(i,0.0);
   for(i=0;i<=n*n-1;i++)
      s.m_tmpp[i]=0.0;
//--- function call
   CMinBLEIC::MinBLEICCreate(n*n,s.m_tmpp,s.m_bs);
  }
//+------------------------------------------------------------------+
//| Training report:                                                 |
//|     * NGrad     - number of gradient calculations                |
//|     * NHess     - number of Hessian calculations                 |
//|     * NCholesky - number of Cholesky decompositions              |
//+------------------------------------------------------------------+
class CMLPReport
  {
public:
   //--- variables
   int               m_ngrad;
   int               m_nhess;
   int               m_ncholesky;
   //--- constructor, destructor
                     CMLPReport(void);
                    ~CMLPReport(void);
   //--- copy
   void              Copy(CMLPReport &obj);
  };
//+------------------------------------------------------------------+
//| Constructor without parameters                                   |
//+------------------------------------------------------------------+
CMLPReport::CMLPReport(void)
  {

  }
//+------------------------------------------------------------------+
//| Destructor                                                       |
//+------------------------------------------------------------------+
CMLPReport::~CMLPReport(void)
  {

  }
//+------------------------------------------------------------------+
//| Copy                                                             |
//+------------------------------------------------------------------+
void CMLPReport::Copy(CMLPReport &obj)
  {
//--- copy variables
   m_ngrad=obj.m_ngrad;
   m_nhess=obj.m_nhess;
   m_ncholesky=obj.m_ncholesky;
  }
//+------------------------------------------------------------------+
//| Training report:                                                 |
//|     * NGrad     - number of gradient calculations                |
//|     * NHess     - number of Hessian calculations                 |
//|     * NCholesky - number of Cholesky decompositions              |
//+------------------------------------------------------------------+
class CMLPReportShell
  {
private:
   CMLPReport        m_innerobj;
public:
   //--- constructors, destructor
                     CMLPReportShell(void);
                     CMLPReportShell(CMLPReport &obj);
                    ~CMLPReportShell(void);
   //--- methods
   int               GetNGrad(void);
   void              SetNGrad(const int i);
   int               GetNHess(void);
   void              SetNHess(const int i);
   int               GetNCholesky(void);
   void              SetNCholesky(const int i);
   CMLPReport       *GetInnerObj(void);
  };
//+------------------------------------------------------------------+
//| Constructor without parameters                                   |
//+------------------------------------------------------------------+
CMLPReportShell::CMLPReportShell(void)
  {

  }
//+------------------------------------------------------------------+
//| Copy constructor                                                 |
//+------------------------------------------------------------------+
CMLPReportShell::CMLPReportShell(CMLPReport &obj)
  {
//--- copy
   m_innerobj.Copy(obj);
  }
//+------------------------------------------------------------------+
//| Destructor                                                       |
//+------------------------------------------------------------------+
CMLPReportShell::~CMLPReportShell(void)
  {

  }
//+------------------------------------------------------------------+
//| Returns the value of the variable ngrad                          |
//+------------------------------------------------------------------+
int CMLPReportShell::GetNGrad(void)
  {
//--- return result
   return(m_innerobj.m_ngrad);
  }
//+------------------------------------------------------------------+
//| Changing the value of the variable ngrad                         |
//+------------------------------------------------------------------+
void CMLPReportShell::SetNGrad(const int i)
  {
//--- change value
   m_innerobj.m_ngrad=i;
  }
//+------------------------------------------------------------------+
//| Returns the value of the variable nhess                          |
//+------------------------------------------------------------------+
int CMLPReportShell::GetNHess(void)
  {
//--- return result
   return(m_innerobj.m_nhess);
  }
//+------------------------------------------------------------------+
//| Changing the value of the variable nhess                         |
//+------------------------------------------------------------------+
void CMLPReportShell::SetNHess(const int i)
  {
//--- change value
   m_innerobj.m_nhess=i;
  }
//+------------------------------------------------------------------+
//| Returns the value of the variable ncholesky                      |
//+------------------------------------------------------------------+
int CMLPReportShell::GetNCholesky(void)
  {
//--- return result
   return(m_innerobj.m_ncholesky);
  }
//+------------------------------------------------------------------+
//| Changing the value of the variable ncholesky                     |
//+------------------------------------------------------------------+
void CMLPReportShell::SetNCholesky(const int i)
  {
//--- change value
   m_innerobj.m_ncholesky=i;
  }
//+------------------------------------------------------------------+
//| Return object of class                                           |
//+------------------------------------------------------------------+
CMLPReport *CMLPReportShell::GetInnerObj(void)
  {
//--- return result
   return(GetPointer(m_innerobj));
  }
//+------------------------------------------------------------------+
//| Cross-validation estimates of generalization error               |
//+------------------------------------------------------------------+
class CMLPCVReport
  {
public:
   //--- variables
   double            m_relclserror;
   double            m_avgce;
   double            m_rmserror;
   double            m_avgerror;
   double            m_avgrelerror;
   //--- constructor, destructor
                     CMLPCVReport(void);
                    ~CMLPCVReport(void);
   //--- copy
   void              Copy(CMLPCVReport &obj);
  };
//+------------------------------------------------------------------+
//| Constructor without parameters                                   |
//+------------------------------------------------------------------+
CMLPCVReport::CMLPCVReport(void)
  {

  }
//+------------------------------------------------------------------+
//| Destructor                                                       |
//+------------------------------------------------------------------+
CMLPCVReport::~CMLPCVReport(void)
  {

  }
//+------------------------------------------------------------------+
//| Copy                                                             |
//+------------------------------------------------------------------+
void CMLPCVReport::Copy(CMLPCVReport &obj)
  {
//--- copy variables
   m_relclserror=obj.m_relclserror;
   m_avgce=obj.m_avgce;
   m_rmserror=obj.m_rmserror;
   m_avgerror=obj.m_avgerror;
   m_avgrelerror=obj.m_avgrelerror;
  }
//+------------------------------------------------------------------+
//| Cross-validation estimates of generalization error               |
//+------------------------------------------------------------------+
class CMLPCVReportShell
  {
private:
   CMLPCVReport      m_innerobj;
public:
   //--- constructors, destructor
                     CMLPCVReportShell(void);
                     CMLPCVReportShell(CMLPCVReport &obj);
                    ~CMLPCVReportShell(void);
   //--- methods
   double            GetRelClsError(void);
   void              SetRelClsError(const double d);
   double            GetAvgCE(void);
   void              SetAvgCE(const double d);
   double            GetRMSError(void);
   void              SetRMSError(const double d);
   double            GetAvgError(void);
   void              SetAvgError(const double d);
   double            GetAvgRelError(void);
   void              SetAvgRelError(const double d);
   CMLPCVReport     *GetInnerObj(void);
  };
//+------------------------------------------------------------------+
//| Constructor without parameters                                   |
//+------------------------------------------------------------------+
CMLPCVReportShell::CMLPCVReportShell(void)
  {

  }
//+------------------------------------------------------------------+
//| Copy constructor                                                 |
//+------------------------------------------------------------------+
CMLPCVReportShell::CMLPCVReportShell(CMLPCVReport &obj)
  {
//--- copy
   m_innerobj.Copy(obj);
  }
//+------------------------------------------------------------------+
//| Destructor                                                       |
//+------------------------------------------------------------------+
CMLPCVReportShell::~CMLPCVReportShell(void)
  {

  }
//+------------------------------------------------------------------+
//| Returns the value of the variable relclserror                    |
//+------------------------------------------------------------------+
double CMLPCVReportShell::GetRelClsError(void)
  {
//--- return result
   return(m_innerobj.m_relclserror);
  }
//+------------------------------------------------------------------+
//| Changing the value of the variable relclserror                   |
//+------------------------------------------------------------------+
void CMLPCVReportShell::SetRelClsError(const double d)
  {
//--- change value
   m_innerobj.m_relclserror=d;
  }
//+------------------------------------------------------------------+
//| Returns the value of the variable avgce                          |
//+------------------------------------------------------------------+
double CMLPCVReportShell::GetAvgCE(void)
  {
//--- return result
   return(m_innerobj.m_avgce);
  }
//+------------------------------------------------------------------+
//| Changing the value of the variable avgce                         |
//+------------------------------------------------------------------+
void CMLPCVReportShell::SetAvgCE(const double d)
  {
//--- change value
   m_innerobj.m_avgce=d;
  }
//+------------------------------------------------------------------+
//| Returns the value of the variable rmserror                       |
//+------------------------------------------------------------------+
double CMLPCVReportShell::GetRMSError(void)
  {
//--- return result
   return(m_innerobj.m_rmserror);
  }
//+------------------------------------------------------------------+
//| Changing the value of the variable rmserror                      |
//+------------------------------------------------------------------+
void CMLPCVReportShell::SetRMSError(const double d)
  {
//--- change value
   m_innerobj.m_rmserror=d;
  }
//+------------------------------------------------------------------+
//| Returns the value of the variable avgerror                       |
//+------------------------------------------------------------------+
double CMLPCVReportShell::GetAvgError(void)
  {
//--- return result
   return(m_innerobj.m_avgerror);
  }
//+------------------------------------------------------------------+
//| Changing the value of the variable avgerror                      |
//+------------------------------------------------------------------+
void CMLPCVReportShell::SetAvgError(const double d)
  {
//--- change value
   m_innerobj.m_avgerror=d;
  }
//+------------------------------------------------------------------+
//| Returns the value of the variable avgrelerror                    |
//+------------------------------------------------------------------+
double CMLPCVReportShell::GetAvgRelError(void)
  {
//--- return result
   return(m_innerobj.m_avgrelerror);
  }
//+------------------------------------------------------------------+
//| Changing the value of the variable avgrelerror                   |
//+------------------------------------------------------------------+
void CMLPCVReportShell::SetAvgRelError(const double d)
  {
//--- change value
   m_innerobj.m_avgrelerror=d;
  }
//+------------------------------------------------------------------+
//| Return object of class                                           |
//+------------------------------------------------------------------+
CMLPCVReport *CMLPCVReportShell::GetInnerObj(void)
  {
//--- return result
   return(GetPointer(m_innerobj));
  }
//+------------------------------------------------------------------+
//| Training neural networks                                         |
//+------------------------------------------------------------------+
class CMLPTrain
  {
private:
   //--- private methods
   static void       MLPKFoldCVGeneral(CMultilayerPerceptron &n,CMatrixDouble &xy,const int npoints,const double decay,const int restarts,const int foldscount,const bool lmalgorithm,const double wstep,const int maxits,int &info,CMLPReport &rep,CMLPCVReport &cvrep);
   static void       MLPKFoldSplit(CMatrixDouble &xy,const int npoints,const int nclasses,const int foldscount,const bool stratifiedsplits,int &folds[]);
public:
   //--- constant
   static const double m_mindecay;
   //--- constructor, destructor
                     CMLPTrain(void);
                    ~CMLPTrain(void);
   //--- public methods
   static void       MLPTrainLM(CMultilayerPerceptron &network,CMatrixDouble &xy,const int npoints,double decay,const int restarts,int &info,CMLPReport &rep);
   static void       MLPTrainLBFGS(CMultilayerPerceptron &network,CMatrixDouble &xy,const int npoints,double decay,const int restarts,const double wstep,int maxits,int &info,CMLPReport &rep);
   static void       MLPTrainES(CMultilayerPerceptron &network,CMatrixDouble &trnxy,const int trnsize,CMatrixDouble &valxy,const int valsize,const double decay,const int restarts,int &info,CMLPReport &rep);
   static void       MLPKFoldCVLBFGS(CMultilayerPerceptron &network,CMatrixDouble &xy,const int npoints,const double decay,const int restarts,const double wstep,const int maxits,const int foldscount,int &info,CMLPReport &rep,CMLPCVReport &cvrep);
   static void       MLPKFoldCVLM(CMultilayerPerceptron &network,CMatrixDouble &xy,const int npoints,const double decay,const int restarts,int foldscount,int &info,CMLPReport &rep,CMLPCVReport &cvrep);
  };
//+------------------------------------------------------------------+
//| Initialize constant                                              |
//+------------------------------------------------------------------+ 
const double CMLPTrain::m_mindecay=0.001;
//+------------------------------------------------------------------+
//| Constructor without parameters                                   |
//+------------------------------------------------------------------+
CMLPTrain::CMLPTrain(void)
  {

  }
//+------------------------------------------------------------------+
//| Destructor                                                       |
//+------------------------------------------------------------------+
CMLPTrain::~CMLPTrain(void)
  {

  }
//+------------------------------------------------------------------+
//| Neural network training using modified Levenberg-Marquardt with  |
//| exact Hessian calculation and regularization. Subroutine trains  |
//| neural network with restarts from random positions. Algorithm is |
//| well suited for small                                            |
//| and medium scale problems (hundreds of weights).                 |
//| INPUT PARAMETERS:                                                |
//|     Network     -   neural network with initialized geometry     |
//|     XY          -   training set                                 |
//|     NPoints     -   training set size                            |
//|     Decay       -   weight decay constant, >=0.001               |
//|                     Decay term 'Decay*||Weights||^2' is added to |
//|                     error function.                              |
//|                     If you don't know what Decay to choose, use  |
//|                     0.001.                                       |
//|     Restarts    -   number of restarts from random position, >0. |
//|                     If you don't know what Restarts to choose,   |
//|                     use 2.                                       |
//| OUTPUT PARAMETERS:                                               |
//|     Network     -   trained neural network.                      |
//|     Info        -   return code:                                 |
//|                     * -9, if internal matrix inverse subroutine  |
//|                           failed                                 |
//|                     * -2, if there is a point with class number  |
//|                           outside of [0..NOut-1].                |
//|                     * -1, if wrong parameters specified          |
//|                           (NPoints<0, Restarts<1).               |
//|                     *  2, if task has been solved.               |
//|     Rep         -   training report                              |
//+------------------------------------------------------------------+
static void CMLPTrain::MLPTrainLM(CMultilayerPerceptron &network,CMatrixDouble &xy,
                                  const int npoints,double decay,const int restarts,
                                  int &info,CMLPReport &rep)
  {
//--- create variables
   int    nin=0;
   int    nout=0;
   int    wcount=0;
   double lmm_ftol=0;
   double lmsteptol=0;
   int    i=0;
   int    k=0;
   double v=0;
   double e=0;
   double enew=0;
   double xnorm2=0;
   double stepnorm=0;
   bool   spd;
   double nu=0;
   double lambdav=0;
   double lambdaup=0;
   double lambdadown=0;
   int    pass=0;
   double ebest=0;
   int    invinfo=0;
   int    solverinfo=0;
   int    i_=0;
//--- creating arrays
   double g[];
   double d[];
   double x[];
   double y[];
   double wbase[];
   double wdir[];
   double wt[];
   double wx[];
   double wbest[];
//--- create matrix
   CMatrixDouble h;
   CMatrixDouble hmod;
   CMatrixDouble z;
//--- objects of classes
   CMinLBFGSReport    internalrep;
   CMinLBFGSState     state;
   CMatInvReport      invrep;
   CDenseSolverReport solverrep;
//--- initialization
   info=0;
//--- function call
   CMLPBase::MLPProperties(network,nin,nout,wcount);
//--- initialization
   lambdaup=10;
   lambdadown=0.3;
   lmm_ftol=0.001;
   lmsteptol=0.001;
//--- Test for inputs
   if(npoints<=0 || restarts<1)
     {
      info=-1;
      return;
     }
//--- check
   if(CMLPBase::MLPIsSoftMax(network))
     {
      for(i=0;i<=npoints-1;i++)
        {
         //--- check
         if((int)MathRound(xy[i][nin])<0 || (int)MathRound(xy[i][nin])>=nout)
           {
            info=-2;
            return;
           }
        }
     }
//--- change values
   decay=MathMax(decay,m_mindecay);
   info=2;
//--- Initialize data
   rep.m_ngrad=0;
   rep.m_nhess=0;
   rep.m_ncholesky=0;
//--- General case.
//--- Prepare task and network. Allocate space.
   CMLPBase::MLPInitPreprocessor(network,xy,npoints);
//--- allocation
   ArrayResizeAL(g,wcount);
   h.Resize(wcount,wcount);
   hmod.Resize(wcount,wcount);
   ArrayResizeAL(wbase,wcount);
   ArrayResizeAL(wdir,wcount);
   ArrayResizeAL(wbest,wcount);
   ArrayResizeAL(wt,wcount);
   ArrayResizeAL(wx,wcount);
//--- initialization
   ebest=CMath::m_maxrealnumber;
//--- Multiple passes
   for(pass=1;pass<=restarts;pass++)
     {
      //--- Initialize weights
      CMLPBase::MLPRandomize(network);
      //--- First stage of the hybrid algorithm: LBFGS
      for(i_=0;i_<=wcount-1;i_++)
         wbase[i_]=network.m_weights[i_];
      //--- function calls
      CMinLBFGS::MinLBFGSCreate(wcount,(int)(MathMin(wcount,5)),wbase,state);
      CMinLBFGS::MinLBFGSSetCond(state,0,0,0,(int)(MathMax(25,wcount)));
      while(CMinLBFGS::MinLBFGSIteration(state))
        {
         //--- gradient
         for(i_=0;i_<=wcount-1;i_++)
            network.m_weights[i_]=state.m_x[i_];
         //--- function call
         CMLPBase::MLPGradBatch(network,xy,npoints,state.m_f,state.m_g);
         //--- weight decay
         v=0.0;
         for(i_=0;i_<=wcount-1;i_++)
            v+=network.m_weights[i_]*network.m_weights[i_];
         state.m_f=state.m_f+0.5*decay*v;
         for(i_=0;i_<=wcount-1;i_++)
            state.m_g[i_]=state.m_g[i_]+decay*network.m_weights[i_];
         //--- next iteration
         rep.m_ngrad=rep.m_ngrad+1;
        }
      //--- function call
      CMinLBFGS::MinLBFGSResults(state,wbase,internalrep);
      for(i_=0;i_<=wcount-1;i_++)
         network.m_weights[i_]=wbase[i_];
      //--- Second stage of the hybrid algorithm: LM
      //--- Initialize H with identity matrix,
      //--- G with gradient,
      //--- E with regularized error.
      CMLPBase::MLPHessianBatch(network,xy,npoints,e,g,h);
      v=0.0;
      for(i_=0;i_<=wcount-1;i_++)
         v+=network.m_weights[i_]*network.m_weights[i_];
      //--- change values
      e=e+0.5*decay*v;
      for(i_=0;i_<=wcount-1;i_++)
         g[i_]=g[i_]+decay*network.m_weights[i_];
      for(k=0;k<=wcount-1;k++)
         h[k].Set(k,h[k][k]+decay);
      //--- change values
      rep.m_nhess=rep.m_nhess+1;
      lambdav=0.001;
      nu=2;
      //--- cycle
      while(true)
        {
         //--- 1. HMod=H+lambda*I
         //--- 2. Try to solve (H+Lambda*I)*dx=-g.
         //---    Increase lambda if left part is not positive definite.
         for(i=0;i<=wcount-1;i++)
           {
            for(i_=0;i_<=wcount-1;i_++)
               hmod[i].Set(i_,h[i][i_]);
            hmod[i].Set(i,hmod[i][i]+lambdav);
           }
         //--- function call
         spd=CTrFac::SPDMatrixCholesky(hmod,wcount,true);
         rep.m_ncholesky=rep.m_ncholesky+1;
         //--- check
         if(!spd)
           {
            lambdav=lambdav*lambdaup*nu;
            nu=nu*2;
            continue;
           }
         //--- function call
         CDenseSolver::SPDMatrixCholeskySolve(hmod,wcount,true,g,solverinfo,solverrep,wdir);
         //--- check
         if(solverinfo<0)
           {
            lambdav=lambdav*lambdaup*nu;
            nu=nu*2;
            continue;
           }
         for(i_=0;i_<=wcount-1;i_++)
            wdir[i_]=-1*wdir[i_];
         //--- Lambda found.
         //--- 1. Save old w in WBase
         //--- 1. Test some stopping criterions
         //--- 2. If error(w+wdir)>error(w),increase lambda
         for(i_=0;i_<=wcount-1;i_++)
            network.m_weights[i_]=network.m_weights[i_]+wdir[i_];
         xnorm2=0.0;
         for(i_=0;i_<=wcount-1;i_++)
            xnorm2+=network.m_weights[i_]*network.m_weights[i_];
         //--- change value
         stepnorm=0.0;
         for(i_=0;i_<=wcount-1;i_++)
            stepnorm+=wdir[i_]*wdir[i_];
         stepnorm=MathSqrt(stepnorm);
         //--- function call
         enew=CMLPBase::MLPError(network,xy,npoints)+0.5*decay*xnorm2;
         //--- check
         if(stepnorm<lmsteptol*(1+MathSqrt(xnorm2)))
            break;
         //--- check
         if(enew>e)
           {
            lambdav=lambdav*lambdaup*nu;
            nu=nu*2;
            continue;
           }
         //--- Optimize using inv(cholesky(H)) as preconditioner
         CMatInv::RMatrixTrInverse(hmod,wcount,true,false,invinfo,invrep);
         //--- check
         if(invinfo<=0)
           {
            //--- if matrix can't be inverted then exit with errors
            //--- TODO: make WCount steps in direction suggested by HMod
            info=-9;
            return;
           }
         //--- calculation
         for(i_=0;i_<=wcount-1;i_++)
            wbase[i_]=network.m_weights[i_];
         for(i=0;i<=wcount-1;i++)
            wt[i]=0;
         //--- function calls
         CMinLBFGS::MinLBFGSCreateX(wcount,wcount,wt,1,0.0,state);
         CMinLBFGS::MinLBFGSSetCond(state,0,0,0,5);
         while(CMinLBFGS::MinLBFGSIteration(state))
           {
            //--- gradient
            for(i=0;i<=wcount-1;i++)
              {
               v=0.0;
               for(i_=i;i_<=wcount-1;i_++)
                  v+=state.m_x[i_]*hmod[i][i_];
               network.m_weights[i]=wbase[i]+v;
              }
            //--- function call
            CMLPBase::MLPGradBatch(network,xy,npoints,state.m_f,g);
            for(i=0;i<=wcount-1;i++)
               state.m_g[i]=0;
            for(i=0;i<=wcount-1;i++)
              {
               v=g[i];
               for(i_=i;i_<=wcount-1;i_++)
                  state.m_g[i_]=state.m_g[i_]+v*hmod[i][i_];
              }
            //--- weight decay
            //--- grad(x'*x)=A'*(x0+A*t)
            v=0.0;
            for(i_=0;i_<=wcount-1;i_++)
               v+=network.m_weights[i_]*network.m_weights[i_];
            state.m_f=state.m_f+0.5*decay*v;
            for(i=0;i<=wcount-1;i++)
              {
               v=decay*network.m_weights[i];
               for(i_=i;i_<=wcount-1;i_++)
                  state.m_g[i_]=state.m_g[i_]+v*hmod[i][i_];
              }
            //--- next iteration
            rep.m_ngrad=rep.m_ngrad+1;
           }
         //--- function call
         CMinLBFGS::MinLBFGSResults(state,wt,internalrep);
         //--- Accept new position.
         //--- Calculate Hessian
         for(i=0;i<=wcount-1;i++)
           {
            v=0.0;
            for(i_=i;i_<=wcount-1;i_++)
               v+=wt[i_]*hmod[i][i_];
            network.m_weights[i]=wbase[i]+v;
           }
         //--- function call
         CMLPBase::MLPHessianBatch(network,xy,npoints,e,g,h);
         v=0.0;
         for(i_=0;i_<=wcount-1;i_++)
            v+=network.m_weights[i_]*network.m_weights[i_];
         //--- change value
         e=e+0.5*decay*v;
         for(i_=0;i_<=wcount-1;i_++)
            g[i_]=g[i_]+decay*network.m_weights[i_];
         for(k=0;k<=wcount-1;k++)
            h[k].Set(k,h[k][k]+decay);
         rep.m_nhess=rep.m_nhess+1;
         //--- Update lambda
         lambdav=lambdav*lambdadown;
         nu=2;
        }
      //--- update WBest
      v=0.0;
      for(i_=0;i_<=wcount-1;i_++)
         v+=network.m_weights[i_]*network.m_weights[i_];
      //--- change value
      e=0.5*decay*v+CMLPBase::MLPError(network,xy,npoints);
      //--- check
      if(e<ebest)
        {
         ebest=e;
         for(i_=0;i_<=wcount-1;i_++)
            wbest[i_]=network.m_weights[i_];
        }
     }
//--- copy WBest to output
   for(i_=0;i_<=wcount-1;i_++)
      network.m_weights[i_]=wbest[i_];
  }
//+------------------------------------------------------------------+
//| Neural network training using L-BFGS algorithm with              |
//| regularization. Subroutine trains neural network with restarts   |
//| from random positions. Algorithm is well suited for problems of  |
//| any dimensionality (memory requirements and step complexity are  |
//| linear by weights number).                                       |
//| INPUT PARAMETERS:                                                |
//|     Network    -   neural network with initialized geometry      |
//|     XY         -   training set                                  |
//|     NPoints    -   training set size                             |
//|     Decay      -   weight decay constant, >=0.001                |
//|                    Decay term 'Decay*||Weights||^2' is added to  |
//|                    error function.                               |
//|                    If you don't know what Decay to choose, use   |
//|                    0.001.                                        |
//|     Restarts   -   number of restarts from random position, >0.  |
//|                    If you don't know what Restarts to choose,    |
//|                    use 2.                                        |
//|     WStep      -   stopping criterion. Algorithm stops if step   |
//|                    size is less than WStep. Recommended          |
//|                    value - 0.01. Zero step size means stopping   |
//|                    after MaxIts iterations.                      |
//|     MaxIts     -   stopping criterion. Algorithm stops after     |
//|                    MaxIts iterations (NOT gradient calculations).|
//|                    Zero MaxIts means stopping when step is       |
//|                    sufficiently small.                           |
//| OUTPUT PARAMETERS:                                               |
//|     Network     -   trained neural network.                      |
//|     Info        -   return code:                                 |
//|                     * -8, if both WStep=0 and MaxIts=0           |
//|                     * -2, if there is a point with class number  |
//|                           outside of [0..NOut-1].                |
//|                     * -1, if wrong parameters specified          |
//|                           (NPoints<0, Restarts<1).               |
//|                     *  2, if task has been solved.               |
//|     Rep         -   training report                              |
//+------------------------------------------------------------------+
static void CMLPTrain::MLPTrainLBFGS(CMultilayerPerceptron &network,
                                     CMatrixDouble &xy,const int npoints,
                                     double decay,const int restarts,
                                     const double wstep,int maxits,
                                     int &info,CMLPReport &rep)
  {
//--- create variables
   int    i=0;
   int    pass=0;
   int    nin=0;
   int    nout=0;
   int    wcount=0;
   double e=0;
   double v=0;
   double ebest=0;
//--- creating arrays
   double w[];
   double wbest[];
//--- create objects of classes
   CMinLBFGSReport internalrep;
   CMinLBFGSState  state;
   int i_=0;
//--- initialization
   info=0;
//--- Test inputs,parse flags,read network geometry
   if(wstep==0.0 && maxits==0)
     {
      info=-8;
      return;
     }
//--- check
   if(((npoints<=0 || restarts<1) || wstep<0.0) || maxits<0)
     {
      info=-1;
      return;
     }
//--- function call
   CMLPBase::MLPProperties(network,nin,nout,wcount);
//--- check
   if(CMLPBase::MLPIsSoftMax(network))
     {
      for(i=0;i<=npoints-1;i++)
        {
         //--- check
         if((int)MathRound(xy[i][nin])<0 || (int)MathRound(xy[i][nin])>=nout)
           {
            info=-2;
            return;
           }
        }
     }
//--- change values
   decay=MathMax(decay,m_mindecay);
   info=2;
//--- Prepare
   CMLPBase::MLPInitPreprocessor(network,xy,npoints);
//--- allocation
   ArrayResizeAL(w,wcount);
   ArrayResizeAL(wbest,wcount);
//--- initialization
   ebest=CMath::m_maxrealnumber;
//--- Multiple starts
   rep.m_ncholesky=0;
   rep.m_nhess=0;
   rep.m_ngrad=0;
   for(pass=1;pass<=restarts;pass++)
     {
      //--- Process
      CMLPBase::MLPRandomize(network);
      for(i_=0;i_<=wcount-1;i_++)
         w[i_]=network.m_weights[i_];
      //--- function calls
      CMinLBFGS::MinLBFGSCreate(wcount,(int)(MathMin(wcount,10)),w,state);
      CMinLBFGS::MinLBFGSSetCond(state,0.0,0.0,wstep,maxits);
      while(CMinLBFGS::MinLBFGSIteration(state))
        {
         for(i_=0;i_<=wcount-1;i_++)
            network.m_weights[i_]=state.m_x[i_];
         //--- function call
         CMLPBase::MLPGradNBatch(network,xy,npoints,state.m_f,state.m_g);
         v=0.0;
         for(i_=0;i_<=wcount-1;i_++)
            v+=network.m_weights[i_]*network.m_weights[i_];
         state.m_f=state.m_f+0.5*decay*v;
         for(i_=0;i_<=wcount-1;i_++)
            state.m_g[i_]=state.m_g[i_]+decay*network.m_weights[i_];
         rep.m_ngrad=rep.m_ngrad+1;
        }
      //--- function call
      CMinLBFGS::MinLBFGSResults(state,w,internalrep);
      for(i_=0;i_<=wcount-1;i_++)
         network.m_weights[i_]=w[i_];
      //--- Compare with best
      v=0.0;
      for(i_=0;i_<=wcount-1;i_++)
         v+=network.m_weights[i_]*network.m_weights[i_];
      //--- change value
      e=CMLPBase::MLPErrorN(network,xy,npoints)+0.5*decay*v;
      //--- check
      if(e<ebest)
        {
         for(i_=0;i_<=wcount-1;i_++)
            wbest[i_]=network.m_weights[i_];
         ebest=e;
        }
     }
//--- The best network
   for(i_=0;i_<=wcount-1;i_++)
      network.m_weights[i_]=wbest[i_];
  }
//+------------------------------------------------------------------+
//| Neural network training using early stopping (base algorithm -   |
//| L-BFGS with regularization).                                     |
//| INPUT PARAMETERS:                                                |
//|     Network     -   neural network with initialized geometry     |
//|     TrnXY       -   training set                                 |
//|     TrnSize     -   training set size                            |
//|     ValXY       -   validation set                               |
//|     ValSize     -   validation set size                          |
//|     Decay       -   weight decay constant, >=0.001               |
//|                     Decay term 'Decay*||Weights||^2' is added to |
//|                     error function.                              |
//|                     If you don't know what Decay to choose, use  |
//|                     0.001.                                       |
//|     Restarts    -   number of restarts from random position, >0. |
//|                     If you don't know what Restarts to choose,   |
//|                     use 2.                                       |
//| OUTPUT PARAMETERS:                                               |
//|     Network     -   trained neural network.                      |
//|     Info        -   return code:                                 |
//|                     * -2, if there is a point with class number  |
//|                           outside of [0..NOut-1].                |
//|                     * -1, if wrong parameters specified          |
//|                           (NPoints<0, Restarts<1, ...).          |
//|                     *  2, task has been solved, stopping         |
//|                           criterion met - sufficiently small     |
//|                           step size. Not expected (we use EARLY  |
//|                           stopping) but possible and not an error|
//|                     *  6, task has been solved, stopping         |
//|                           criterion  met - increasing of         |
//|                           validation set error.                  |
//|     Rep         -   training report                              |
//| NOTE:                                                            |
//| Algorithm stops if validation set error increases for a long     |
//| enough or step size is small enought (there are task where       |
//| validation set may decrease for eternity). In any case solution  |
//| returned corresponds to the minimum of validation set error.     |
//+------------------------------------------------------------------+
static void CMLPTrain::MLPTrainES(CMultilayerPerceptron &network,
                                  CMatrixDouble &trnxy,const int trnsize,
                                  CMatrixDouble &valxy,const int valsize,
                                  const double decay,const int restarts,
                                  int &info,CMLPReport &rep)
  {
//--- create variables
   int    i=0;
   int    pass=0;
   int    nin=0;
   int    nout=0;
   int    wcount=0;
   double e=0;
   double v=0;
   double ebest=0;
   int    itbest=0;
   double wstep=0;
   int    i_=0;
//--- creating arrays
   double w[];
   double wbest[];
   double wfinal[];
   double efinal=0;
//--- objects of classes
   CMinLBFGSReport internalrep;
   CMinLBFGSState state;
//--- initialization
   info=0;
   wstep=0.001;
//--- Test inputs,parse flags,read network geometry
   if(((trnsize<=0 || valsize<=0) || restarts<1) || decay<0.0)
     {
      info=-1;
      return;
     }
//--- function call
   CMLPBase::MLPProperties(network,nin,nout,wcount);
//--- check
   if(CMLPBase::MLPIsSoftMax(network))
     {
      for(i=0;i<=trnsize-1;i++)
        {
         //--- check
         if((int)MathRound(trnxy[i][nin])<0 || (int)MathRound(trnxy[i][nin])>=nout)
           {
            info=-2;
            return;
           }
        }
      for(i=0;i<=valsize-1;i++)
        {
         //--- check
         if((int)MathRound(valxy[i][nin])<0 || (int)MathRound(valxy[i][nin])>=nout)
           {
            info=-2;
            return;
           }
        }
     }
//--- change value
   info=2;
//--- Prepare
   CMLPBase::MLPInitPreprocessor(network,trnxy,trnsize);
//--- allocation
   ArrayResizeAL(w,wcount);
   ArrayResizeAL(wbest,wcount);
   ArrayResizeAL(wfinal,wcount);
//--- initialization
   efinal=CMath::m_maxrealnumber;
   for(i=0;i<=wcount-1;i++)
      wfinal[i]=0;
//--- Multiple starts
   rep.m_ncholesky=0;
   rep.m_nhess=0;
   rep.m_ngrad=0;
//--- calculation
   for(pass=1;pass<=restarts;pass++)
     {
      //--- Process
      CMLPBase::MLPRandomize(network);
      //--- change values
      ebest=CMLPBase::MLPError(network,valxy,valsize);
      for(i_=0;i_<=wcount-1;i_++)
         wbest[i_]=network.m_weights[i_];
      //--- change values
      itbest=0;
      for(i_=0;i_<=wcount-1;i_++)
         w[i_]=network.m_weights[i_];
      //--- function calls
      CMinLBFGS::MinLBFGSCreate(wcount,(int)(MathMin(wcount,10)),w,state);
      CMinLBFGS::MinLBFGSSetCond(state,0.0,0.0,wstep,0);
      CMinLBFGS::MinLBFGSSetXRep(state,true);
      while(CMinLBFGS::MinLBFGSIteration(state))
        {
         //--- Calculate gradient
         for(i_=0;i_<=wcount-1;i_++)
            network.m_weights[i_]=state.m_x[i_];
         //--- function call
         CMLPBase::MLPGradNBatch(network,trnxy,trnsize,state.m_f,state.m_g);
         v=0.0;
         for(i_=0;i_<=wcount-1;i_++)
            v+=network.m_weights[i_]*network.m_weights[i_];
         state.m_f=state.m_f+0.5*decay*v;
         for(i_=0;i_<=wcount-1;i_++)
            state.m_g[i_]=state.m_g[i_]+decay*network.m_weights[i_];
         rep.m_ngrad=rep.m_ngrad+1;
         //--- Validation set
         if(state.m_xupdated)
           {
            for(i_=0;i_<=wcount-1;i_++)
               network.m_weights[i_]=w[i_];
            //--- function call
            e=CMLPBase::MLPError(network,valxy,valsize);
            //--- check
            if(e<ebest)
              {
               ebest=e;
               for(i_=0;i_<=wcount-1;i_++)
                  wbest[i_]=network.m_weights[i_];
               itbest=internalrep.m_iterationscount;
              }
            //--- check
            if(internalrep.m_iterationscount>30 && (double)(internalrep.m_iterationscount)>(double)(1.5*itbest))
              {
               info=6;
               break;
              }
           }
        }
      //--- function call
      CMinLBFGS::MinLBFGSResults(state,w,internalrep);
      //--- Compare with final answer
      if(ebest<efinal)
        {
         for(i_=0;i_<=wcount-1;i_++)
            wfinal[i_]=wbest[i_];
         efinal=ebest;
        }
     }
//--- The best network
   for(i_=0;i_<=wcount-1;i_++)
      network.m_weights[i_]=wfinal[i_];
  }
//+------------------------------------------------------------------+
//| Cross-validation estimate of generalization error.               |
//| Base algorithm - L-BFGS.                                         |
//| INPUT PARAMETERS:                                                |
//|     Network     -   neural network with initialized geometry.    |
//|                     Network is not changed during                |
//|                     cross-validation - it is used only as a      |
//|                     representative of its architecture.          |
//|     XY          -   training set.                                |
//|     SSize       -   training set size                            |
//|     Decay       -   weight  decay, same as in MLPTrainLBFGS      |
//|     Restarts    -   number of restarts, >0.                      |
//|                     restarts are counted for each partition      |
//|                     separately, so total number of restarts will |
//|                     be Restarts*FoldsCount.                      |
//|     WStep       -   stopping criterion, same as in MLPTrainLBFGS |
//|     MaxIts      -   stopping criterion, same as in MLPTrainLBFGS |
//|     FoldsCount  -   number of folds in k-fold cross-validation,  |
//|                     2<=FoldsCount<=SSize.                        |
//|                     recommended value: 10.                       |
//| OUTPUT PARAMETERS:                                               |
//|     Info        -   return code, same as in MLPTrainLBFGS        |
//|     Rep         -   report, same as in MLPTrainLM/MLPTrainLBFGS  |
//|     CVRep       -   generalization error estimates               |
//+------------------------------------------------------------------+
static void CMLPTrain::MLPKFoldCVLBFGS(CMultilayerPerceptron &network,
                                       CMatrixDouble &xy,const int npoints,
                                       const double decay,const int restarts,
                                       const double wstep,const int maxits,
                                       const int foldscount,int &info,
                                       CMLPReport &rep,CMLPCVReport &cvrep)
  {
//--- initialization
   info=0;
//--- function call
   MLPKFoldCVGeneral(network,xy,npoints,decay,restarts,foldscount,false,wstep,maxits,info,rep,cvrep);
  }
//+------------------------------------------------------------------+
//| Cross-validation estimate of generalization error.               |
//| Base algorithm - Levenberg-Marquardt.                            |
//| INPUT PARAMETERS:                                                |
//|     Network     -   neural network with initialized geometry.    |
//|                     Network is not changed during                |
//|                     cross-validation - it is used only as a      |
//|                     representative of its architecture.          |
//|     XY          -   training set.                                |
//|     SSize       -   training set size                            |
//|     Decay       -   weight  decay, same as in MLPTrainLBFGS      |
//|     Restarts    -   number of restarts, >0.                      |
//|                     restarts are counted for each partition      |
//|                     separately, so total number of restarts will |
//|                     be Restarts*FoldsCount.                      |
//|     FoldsCount  -   number of folds in k-fold cross-validation,  |
//|                     2<=FoldsCount<=SSize.                        |
//|                     recommended value: 10.                       |
//| OUTPUT PARAMETERS:                                               |
//|     Info        -   return code, same as in MLPTrainLBFGS        |
//|     Rep         -   report, same as in MLPTrainLM/MLPTrainLBFGS  |
//|     CVRep       -   generalization error estimates               |
//+------------------------------------------------------------------+
static void CMLPTrain::MLPKFoldCVLM(CMultilayerPerceptron &network,
                                    CMatrixDouble &xy,const int npoints,
                                    const double decay,const int restarts,
                                    int foldscount,int &info,CMLPReport &rep,
                                    CMLPCVReport &cvrep)
  {
//--- initialization
   info=0;
//--- function call
   MLPKFoldCVGeneral(network,xy,npoints,decay,restarts,foldscount,true,0.0,0,info,rep,cvrep);
  }
//+------------------------------------------------------------------+
//| Internal cross-validation subroutine                             |
//+------------------------------------------------------------------+
static void CMLPTrain::MLPKFoldCVGeneral(CMultilayerPerceptron &n,
                                         CMatrixDouble &xy,const int npoints,
                                         const double decay,const int restarts,
                                         const int foldscount,const bool lmalgorithm,
                                         const double wstep,const int maxits,
                                         int &info,CMLPReport &rep,
                                         CMLPCVReport &cvrep)
  {
//--- create variables
   int i=0;
   int fold=0;
   int j=0;
   int k=0;
   int nin=0;
   int nout=0;
   int rowlen=0;
   int wcount=0;
   int nclasses=0;
   int tssize=0;
   int cvssize=0;
   int relcnt=0;
   int i_=0;
//--- creating arrays
   int    folds[];
   double x[];
   double y[];
//--- create matrix
   CMatrixDouble cvset;
   CMatrixDouble testset;
//--- creating arrays
   CMultilayerPerceptron network;
   CMLPReport            internalrep;
//--- initialization
   info=0;
//--- Read network geometry,test parameters
   CMLPBase::MLPProperties(n,nin,nout,wcount);
//--- check
   if(CMLPBase::MLPIsSoftMax(n))
     {
      nclasses=nout;
      rowlen=nin+1;
     }
   else
     {
      nclasses=-nout;
      rowlen=nin+nout;
     }
//--- check
   if((npoints<=0 || foldscount<2) || foldscount>npoints)
     {
      info=-1;
      return;
     }
//--- function call
   CMLPBase::MLPCopy(n,network);
//--- K-fold out cross-validation.
//--- First,estimate generalization error
   testset.Resize(npoints,rowlen);
   cvset.Resize(npoints,rowlen);
   ArrayResizeAL(x,nin);
   ArrayResizeAL(y,nout);
//--- function call
   MLPKFoldSplit(xy,npoints,nclasses,foldscount,false,folds);
//--- change values
   cvrep.m_relclserror=0;
   cvrep.m_avgce=0;
   cvrep.m_rmserror=0;
   cvrep.m_avgerror=0;
   cvrep.m_avgrelerror=0;
   rep.m_ngrad=0;
   rep.m_nhess=0;
   rep.m_ncholesky=0;
   relcnt=0;
//--- calculation
   for(fold=0;fold<=foldscount-1;fold++)
     {
      //--- Separate set
      tssize=0;
      cvssize=0;
      for(i=0;i<=npoints-1;i++)
        {
         //--- check
         if(folds[i]==fold)
           {
            for(i_=0;i_<=rowlen-1;i_++)
               testset[tssize].Set(i_,xy[i][i_]);
            tssize=tssize+1;
           }
         else
           {
            for(i_=0;i_<=rowlen-1;i_++)
               cvset[cvssize].Set(i_,xy[i][i_]);
            cvssize=cvssize+1;
           }
        }
      //--- Train on CV training set
      if(lmalgorithm)
         MLPTrainLM(network,cvset,cvssize,decay,restarts,info,internalrep);
      else
         MLPTrainLBFGS(network,cvset,cvssize,decay,restarts,wstep,maxits,info,internalrep);
      //--- check
      if(info<0)
        {
         //--- change values
         cvrep.m_relclserror=0;
         cvrep.m_avgce=0;
         cvrep.m_rmserror=0;
         cvrep.m_avgerror=0;
         cvrep.m_avgrelerror=0;
         //--- exit the function
         return;
        }
      //--- change values
      rep.m_ngrad=rep.m_ngrad+internalrep.m_ngrad;
      rep.m_nhess=rep.m_nhess+internalrep.m_nhess;
      rep.m_ncholesky=rep.m_ncholesky+internalrep.m_ncholesky;
      //--- Estimate error using CV test set
      if(CMLPBase::MLPIsSoftMax(network))
        {
         //--- classification-only code
         cvrep.m_relclserror=cvrep.m_relclserror+CMLPBase::MLPClsError(network,testset,tssize);
         cvrep.m_avgce=cvrep.m_avgce+CMLPBase::MLPErrorN(network,testset,tssize);
        }
      //--- calculation
      for(i=0;i<=tssize-1;i++)
        {
         for(i_=0;i_<=nin-1;i_++)
            x[i_]=testset[i][i_];
         //--- function call
         CMLPBase::MLPProcess(network,x,y);
         //--- check
         if(CMLPBase::MLPIsSoftMax(network))
           {
            //--- Classification-specific code
            k=(int)MathRound(testset[i][nin]);
            for(j=0;j<=nout-1;j++)
              {
               //--- check
               if(j==k)
                 {
                  //--- change values
                  cvrep.m_rmserror=cvrep.m_rmserror+CMath::Sqr(y[j]-1);
                  cvrep.m_avgerror=cvrep.m_avgerror+MathAbs(y[j]-1);
                  cvrep.m_avgrelerror=cvrep.m_avgrelerror+MathAbs(y[j]-1);
                  relcnt=relcnt+1;
                 }
               else
                 {
                  //--- change values
                  cvrep.m_rmserror=cvrep.m_rmserror+CMath::Sqr(y[j]);
                  cvrep.m_avgerror=cvrep.m_avgerror+MathAbs(y[j]);
                 }
              }
           }
         else
           {
            //--- Regression-specific code
            for(j=0;j<=nout-1;j++)
              {
               cvrep.m_rmserror=cvrep.m_rmserror+CMath::Sqr(y[j]-testset[i][nin+j]);
               cvrep.m_avgerror=cvrep.m_avgerror+MathAbs(y[j]-testset[i][nin+j]);
               //--- check
               if(testset[i][nin+j]!=0.0)
                 {
                  cvrep.m_avgrelerror=cvrep.m_avgrelerror+MathAbs((y[j]-testset[i][nin+j])/testset[i][nin+j]);
                  relcnt=relcnt+1;
                 }
              }
           }
        }
     }
//--- check
   if(CMLPBase::MLPIsSoftMax(network))
     {
      cvrep.m_relclserror=cvrep.m_relclserror/npoints;
      cvrep.m_avgce=cvrep.m_avgce/(MathLog(2)*npoints);
     }
//--- change values
   cvrep.m_rmserror=MathSqrt(cvrep.m_rmserror/(npoints*nout));
   cvrep.m_avgerror=cvrep.m_avgerror/(npoints*nout);
   cvrep.m_avgrelerror=cvrep.m_avgrelerror/relcnt;
   info=1;
  }
//+------------------------------------------------------------------+
//| Subroutine prepares K-fold split of the training set.            |
//| NOTES:                                                           |
//|     "NClasses>0" means that we have classification task.         |
//|     "NClasses<0" means regression task with -NClasses real       |
//|     outputs.                                                     |
//+------------------------------------------------------------------+
static void CMLPTrain::MLPKFoldSplit(CMatrixDouble &xy,const int npoints,
                                     const int nclasses,const int foldscount,
                                     const bool stratifiedsplits,int &folds[])
  {
//--- create variables
   int i=0;
   int j=0;
   int k=0;
//--- test parameters
   if(!CAp::Assert(npoints>0,__FUNCTION__+": wrong NPoints!"))
      return;
//--- check
   if(!CAp::Assert(nclasses>1 || nclasses<0,__FUNCTION__+": wrong NClasses!"))
      return;
//--- check
   if(!CAp::Assert(foldscount>=2 && foldscount<=npoints,__FUNCTION__+" wrong FoldsCount!"))
      return;
//--- check
   if(!CAp::Assert(!stratifiedsplits,__FUNCTION__+": stratified splits are not supported!"))
      return;
//--- Folds
   ArrayResizeAL(folds,npoints);
   for(i=0;i<=npoints-1;i++)
      folds[i]=i*foldscount/npoints;
//--- calculation
   for(i=0;i<=npoints-2;i++)
     {
      j=i+CMath::RandomInteger(npoints-i);
      //--- check
      if(j!=i)
        {
         k=folds[i];
         folds[i]=folds[j];
         folds[j]=k;
        }
     }
  }
//+------------------------------------------------------------------+
//| Neural networks ensemble                                         |
//+------------------------------------------------------------------+
class CMLPEnsemble
  {
public:
   //--- variables
   int               m_ensemblesize;
   int               m_nin;
   int               m_nout;
   int               m_wcount;
   bool              m_issoftmax;
   bool              m_postprocessing;
   int               m_serializedlen;
   //--- arrays
   int               m_structinfo[];
   double            m_weights[];
   double            m_columnmeans[];
   double            m_columnsigmas[];
   double            m_serializedmlp[];
   double            m_tmpweights[];
   double            m_tmpmeans[];
   double            m_tmpsigmas[];
   double            m_neurons[];
   double            m_dfdnet[];
   double            m_y[];
   //--- constructor, destructor
                     CMLPEnsemble(void);
                    ~CMLPEnsemble(void);
   //--- copy
   void              Copy(CMLPEnsemble &obj);
  };
//+------------------------------------------------------------------+
//| Constructor without parameters                                   |
//+------------------------------------------------------------------+
CMLPEnsemble::CMLPEnsemble(void)
  {

  }
//+------------------------------------------------------------------+
//| Destructor                                                       |
//+------------------------------------------------------------------+
CMLPEnsemble::~CMLPEnsemble(void)
  {

  }
//+------------------------------------------------------------------+
//| Copy                                                             |
//+------------------------------------------------------------------+
void CMLPEnsemble::Copy(CMLPEnsemble &obj)
  {
//--- copy variables
   m_ensemblesize=obj.m_ensemblesize;
   m_nin=obj.m_nin;
   m_nout=obj.m_nout;
   m_wcount=obj.m_wcount;
   m_issoftmax=obj.m_issoftmax;
   m_postprocessing=obj.m_postprocessing;
   m_serializedlen=obj.m_serializedlen;
//--- copy arrays
   ArrayCopy(m_structinfo,obj.m_structinfo);
   ArrayCopy(m_weights,obj.m_weights);
   ArrayCopy(m_columnmeans,obj.m_columnmeans);
   ArrayCopy(m_columnsigmas,obj.m_columnsigmas);
   ArrayCopy(m_serializedmlp,obj.m_serializedmlp);
   ArrayCopy(m_tmpweights,obj.m_tmpweights);
   ArrayCopy(m_tmpmeans,obj.m_tmpmeans);
   ArrayCopy(m_tmpsigmas,obj.m_tmpsigmas);
   ArrayCopy(m_neurons,obj.m_neurons);
   ArrayCopy(m_dfdnet,obj.m_dfdnet);
   ArrayCopy(m_y,obj.m_y);
  }
//+------------------------------------------------------------------+
//| Neural networks ensemble                                         |
//+------------------------------------------------------------------+
class CMLPEnsembleShell
  {
private:
   CMLPEnsemble      m_innerobj;
public:
   //--- constructors, destructor
                     CMLPEnsembleShell(void);
                     CMLPEnsembleShell(CMLPEnsemble &obj);
                    ~CMLPEnsembleShell(void);
   //--- method
   CMLPEnsemble     *GetInnerObj(void);
  };
//+------------------------------------------------------------------+
//| Constructor without parameters                                   |
//+------------------------------------------------------------------+
CMLPEnsembleShell::CMLPEnsembleShell(void)
  {

  }
//+------------------------------------------------------------------+
//| Copy constructor                                                 |
//+------------------------------------------------------------------+
CMLPEnsembleShell::CMLPEnsembleShell(CMLPEnsemble &obj)
  {
//--- copy
   m_innerobj.Copy(obj);
  }
//+------------------------------------------------------------------+
//| Destructor                                                       |
//+------------------------------------------------------------------+
CMLPEnsembleShell::~CMLPEnsembleShell(void)
  {

  }
//+------------------------------------------------------------------+
//| Return object of class                                           |
//+------------------------------------------------------------------+
CMLPEnsemble *CMLPEnsembleShell::GetInnerObj(void)
  {
//--- return result
   return(GetPointer(m_innerobj));
  }
//+------------------------------------------------------------------+
//| Neural networks ensemble                                         |
//+------------------------------------------------------------------+
class CMLPE
  {
private:
   //--- private methods
   static void       MLPEAllErrors(CMLPEnsemble &ensemble,CMatrixDouble &xy,const int npoints,double &relcls,double &avgce,double &rms,double &avg,double &avgrel);
   static void       MLPEBaggingInternal(CMLPEnsemble &ensemble,CMatrixDouble &xy,const int npoints,const double decay,const int restarts,const double wstep,const int maxits,const bool lmalgorithm,int &info,CMLPReport &rep,CMLPCVReport &ooberrors);
public:
   //--- class constants
   static const int  m_mlpntotaloffset;
   static const int  m_mlpevnum;
   //--- constructor, destructor
                     CMLPE(void);
                    ~CMLPE(void);
   //--- public methods
   static void       MLPECreate0(const int nin,const int nout,const int ensemblesize,CMLPEnsemble &ensemble);
   static void       MLPECreate1(const int nin,const int nhid,const int nout,const int ensemblesize,CMLPEnsemble &ensemble);
   static void       MLPECreate2(const int nin,const int nhid1,const int nhid2,const int nout,const int ensemblesize,CMLPEnsemble &ensemble);
   static void       MLPECreateB0(const int nin,const int nout,const double b,const double d,const int ensemblesize,CMLPEnsemble &ensemble);
   static void       MLPECreateB1(const int nin,const int nhid,const int nout,const double b,const double d,const int ensemblesize,CMLPEnsemble &ensemble);
   static void       MLPECreateB2(const int nin,const int nhid1,const int nhid2,const int nout,const double b,const double d,const int ensemblesize,CMLPEnsemble &ensemble);
   static void       MLPECreateR0(const int nin,const int nout,const double a,const double b,const int ensemblesize,CMLPEnsemble &ensemble);
   static void       MLPECreateR1(const int nin,const int nhid,const int nout,const double a,const double b,const int ensemblesize,CMLPEnsemble &ensemble);
   static void       MLPECreateR2(const int nin,const int nhid1,const int nhid2,const int nout,const double a,const double b,const int ensemblesize,CMLPEnsemble &ensemble);
   static void       MLPECreateC0(const int nin,const int nout,const int ensemblesize,CMLPEnsemble &ensemble);
   static void       MLPECreateC1(const int nin,const int nhid,const int nout,const int ensemblesize,CMLPEnsemble &ensemble);
   static void       MLPECreateC2(const int nin,const int nhid1,const int nhid2,const int nout,const int ensemblesize,CMLPEnsemble &ensemble);
   static void       MLPECreateFromNetwork(CMultilayerPerceptron &network,const int ensemblesize,CMLPEnsemble &ensemble);
   static void       MLPECopy(CMLPEnsemble &ensemble1,CMLPEnsemble &ensemble2);
   static void       MLPESerialize(CMLPEnsemble &ensemble,double &ra[],int &rlen);
   static void       MLPEUnserialize(double &ra[],CMLPEnsemble &ensemble);
   static void       MLPERandomize(CMLPEnsemble &ensemble);
   static void       MLPEProperties(CMLPEnsemble &ensemble,int &nin,int &nout);
   static bool       MLPEIsSoftMax(CMLPEnsemble &ensemble);
   static void       MLPEProcess(CMLPEnsemble &ensemble,double &x[],double &y[]);
   static void       MLPEProcessI(CMLPEnsemble &ensemble,double &x[],double &y[]);
   static double     MLPERelClsError(CMLPEnsemble &ensemble,CMatrixDouble &xy,const int npoints);
   static double     MLPEAvgCE(CMLPEnsemble &ensemble,CMatrixDouble &xy,const int npoints);
   static double     MLPERMSError(CMLPEnsemble &ensemble,CMatrixDouble &xy,const int npoints);
   static double     MLPEAvgError(CMLPEnsemble &ensemble,CMatrixDouble &xy,const int npoints);
   static double     MLPEAvgRelError(CMLPEnsemble &ensemble,CMatrixDouble &xy,const int npoints);
   static void       MLPEBaggingLM(CMLPEnsemble &ensemble,CMatrixDouble &xy,const int npoints,const double decay,const int restarts,int &info,CMLPReport &rep,CMLPCVReport &ooberrors);
   static void       MLPEBaggingLBFGS(CMLPEnsemble &ensemble,CMatrixDouble &xy,const int npoints,const double decay,const int restarts,const double wstep,const int maxits,int &info,CMLPReport &rep,CMLPCVReport &ooberrors);
   static void       MLPETrainES(CMLPEnsemble &ensemble,CMatrixDouble &xy,const int npoints,const double decay,const int restarts,int &info,CMLPReport &rep);
  };
//+------------------------------------------------------------------+
//| Initialize constants                                             |
//+------------------------------------------------------------------+
const int CMLPE::m_mlpntotaloffset=3;
const int CMLPE::m_mlpevnum=9;
//+------------------------------------------------------------------+
//| Constructor without parameters                                   |
//+------------------------------------------------------------------+
CMLPE::CMLPE(void)
  {

  }
//+------------------------------------------------------------------+
//| Destructor                                                       |
//+------------------------------------------------------------------+
CMLPE::~CMLPE(void)
  {

  }
//+------------------------------------------------------------------+
//| Like MLPCreate0, but for ensembles.                              |
//+------------------------------------------------------------------+
static void CMLPE::MLPECreate0(const int nin,const int nout,const int ensemblesize,
                               CMLPEnsemble &ensemble)
  {
//--- object of class
   CMultilayerPerceptron net;
//--- function call
   CMLPBase::MLPCreate0(nin,nout,net);
//--- function call
   MLPECreateFromNetwork(net,ensemblesize,ensemble);
  }
//+------------------------------------------------------------------+
//| Like MLPCreate1, but for ensembles.                              |
//+------------------------------------------------------------------+
static void CMLPE::MLPECreate1(const int nin,const int nhid,const int nout,
                               const int ensemblesize,CMLPEnsemble &ensemble)
  {
//--- object of class
   CMultilayerPerceptron net;
//--- function call
   CMLPBase::MLPCreate1(nin,nhid,nout,net);
//--- function call
   MLPECreateFromNetwork(net,ensemblesize,ensemble);
  }
//+------------------------------------------------------------------+
//| Like MLPCreate2, but for ensembles.                              |
//+------------------------------------------------------------------+
static void CMLPE::MLPECreate2(const int nin,const int nhid1,const int nhid2,
                               const int nout,const int ensemblesize,
                               CMLPEnsemble &ensemble)
  {
//--- object of class
   CMultilayerPerceptron net;
//--- function call
   CMLPBase::MLPCreate2(nin,nhid1,nhid2,nout,net);
//--- function call
   MLPECreateFromNetwork(net,ensemblesize,ensemble);
  }
//+------------------------------------------------------------------+
//| Like MLPCreateB0, but for ensembles.                             |
//+------------------------------------------------------------------+
static void CMLPE::MLPECreateB0(const int nin,const int nout,const double b,
                                const double d,const int ensemblesize,
                                CMLPEnsemble &ensemble)
  {
//--- object of class
   CMultilayerPerceptron net;
//--- function call
   CMLPBase::MLPCreateB0(nin,nout,b,d,net);
//--- function call
   MLPECreateFromNetwork(net,ensemblesize,ensemble);
  }
//+------------------------------------------------------------------+
//| Like MLPCreateB1, but for ensembles.                             |
//+------------------------------------------------------------------+
static void CMLPE::MLPECreateB1(const int nin,const int nhid,const int nout,
                                const double b,const double d,const int ensemblesize,
                                CMLPEnsemble &ensemble)
  {
//--- object of class
   CMultilayerPerceptron net;
//--- function call
   CMLPBase::MLPCreateB1(nin,nhid,nout,b,d,net);
//--- function call
   MLPECreateFromNetwork(net,ensemblesize,ensemble);
  }
//+------------------------------------------------------------------+
//| Like MLPCreateB2, but for ensembles.                             |
//+------------------------------------------------------------------+
static void CMLPE::MLPECreateB2(const int nin,const int nhid1,const int nhid2,
                                const int nout,const double b,const double d,
                                const int ensemblesize,CMLPEnsemble &ensemble)
  {
//--- object of class
   CMultilayerPerceptron net;
//--- function call
   CMLPBase::MLPCreateB2(nin,nhid1,nhid2,nout,b,d,net);
//--- function call
   MLPECreateFromNetwork(net,ensemblesize,ensemble);
  }
//+------------------------------------------------------------------+
//| Like MLPCreateR0, but for ensembles.                             |
//+------------------------------------------------------------------+
static void CMLPE::MLPECreateR0(const int nin,const int nout,const double a,
                                const double b,const int ensemblesize,
                                CMLPEnsemble &ensemble)
  {
//--- object of class
   CMultilayerPerceptron net;
//--- function call
   CMLPBase::MLPCreateR0(nin,nout,a,b,net);
//--- function call
   MLPECreateFromNetwork(net,ensemblesize,ensemble);
  }
//+------------------------------------------------------------------+
//| Like MLPCreateR1, but for ensembles.                             |
//+------------------------------------------------------------------+
static void CMLPE::MLPECreateR1(const int nin,const int nhid,const int nout,
                                const double a,const double b,
                                const int ensemblesize,CMLPEnsemble &ensemble)
  {
//--- object of class
   CMultilayerPerceptron net;
//--- function call
   CMLPBase::MLPCreateR1(nin,nhid,nout,a,b,net);
//--- function call
   MLPECreateFromNetwork(net,ensemblesize,ensemble);
  }
//+------------------------------------------------------------------+
//| Like MLPCreateR2, but for ensembles.                             |
//+------------------------------------------------------------------+
static void CMLPE::MLPECreateR2(const int nin,const int nhid1,const int nhid2,
                                const int nout,const double a,const double b,
                                const int ensemblesize,CMLPEnsemble &ensemble)
  {
//--- object of class
   CMultilayerPerceptron net;
//--- function call
   CMLPBase::MLPCreateR2(nin,nhid1,nhid2,nout,a,b,net);
//--- function call
   MLPECreateFromNetwork(net,ensemblesize,ensemble);
  }
//+------------------------------------------------------------------+
//| Like MLPCreateC0, but for ensembles.                             |
//+------------------------------------------------------------------+
static void CMLPE::MLPECreateC0(const int nin,const int nout,const int ensemblesize,
                                CMLPEnsemble &ensemble)
  {
//--- object of class
   CMultilayerPerceptron net;
//--- function call
   CMLPBase::MLPCreateC0(nin,nout,net);
//--- function call
   MLPECreateFromNetwork(net,ensemblesize,ensemble);
  }
//+------------------------------------------------------------------+
//| Like MLPCreateC1, but for ensembles.                             |
//+------------------------------------------------------------------+
static void CMLPE::MLPECreateC1(const int nin,const int nhid,const int nout,
                                const int ensemblesize,CMLPEnsemble &ensemble)
  {
//--- object of class
   CMultilayerPerceptron net;
//--- function call
   CMLPBase::MLPCreateC1(nin,nhid,nout,net);
//--- function call
   MLPECreateFromNetwork(net,ensemblesize,ensemble);
  }
//+------------------------------------------------------------------+
//| Like MLPCreateC2, but for ensembles.                             |
//+------------------------------------------------------------------+
static void CMLPE::MLPECreateC2(const int nin,const int nhid1,const int nhid2,
                                const int nout,const int ensemblesize,
                                CMLPEnsemble &ensemble)
  {
//--- object of class
   CMultilayerPerceptron net;
//--- function call
   CMLPBase::MLPCreateC2(nin,nhid1,nhid2,nout,net);
//--- function call
   MLPECreateFromNetwork(net,ensemblesize,ensemble);
  }
//+------------------------------------------------------------------+
//| Creates ensemble from network. Only network geometry is copied.  |
//+------------------------------------------------------------------+
static void CMLPE::MLPECreateFromNetwork(CMultilayerPerceptron &network,
                                         const int ensemblesize,
                                         CMLPEnsemble &ensemble)
  {
//--- create variables
   int i=0;
   int ccount=0;
   int i_=0;
   int i1_=0;
//--- check
   if(!CAp::Assert(ensemblesize>0,__FUNCTION__+": incorrect ensemble size!"))
      return;
//--- network properties
   CMLPBase::MLPProperties(network,ensemble.m_nin,ensemble.m_nout,ensemble.m_wcount);
//--- check
   if(CMLPBase::MLPIsSoftMax(network))
      ccount=ensemble.m_nin;
   else
      ccount=ensemble.m_nin+ensemble.m_nout;
//--- change values
   ensemble.m_postprocessing=false;
   ensemble.m_issoftmax=CMLPBase::MLPIsSoftMax(network);
   ensemble.m_ensemblesize=ensemblesize;
//--- structure information
   ArrayResizeAL(ensemble.m_structinfo,network.m_structinfo[0]);
//--- copy
   for(i=0;i<=network.m_structinfo[0]-1;i++)
      ensemble.m_structinfo[i]=network.m_structinfo[i];
//--- weights,means,sigmas
   ArrayResizeAL(ensemble.m_weights,ensemblesize*ensemble.m_wcount);
   ArrayResizeAL(ensemble.m_columnmeans,ensemblesize*ccount);
   ArrayResizeAL(ensemble.m_columnsigmas,ensemblesize*ccount);
//--- calculation
   for(i=0;i<=ensemblesize*ensemble.m_wcount-1;i++)
      ensemble.m_weights[i]=CMath::RandomReal()-0.5;
//--- calculation
   for(i=0;i<=ensemblesize-1;i++)
     {
      i1_=-(i*ccount);
      for(i_=i*ccount;i_<=(i+1)*ccount-1;i_++)
         ensemble.m_columnmeans[i_]=network.m_columnmeans[i_+i1_];
      i1_=-(i*ccount);
      for(i_=i*ccount;i_<=(i+1)*ccount-1;i_++)
         ensemble.m_columnsigmas[i_]=network.m_columnsigmas[i_+i1_];
     }
//--- serialized part
   CMLPBase::MLPSerializeOld(network,ensemble.m_serializedmlp,ensemble.m_serializedlen);
//--- temporaries,internal buffers
   ArrayResizeAL(ensemble.m_tmpweights,ensemble.m_wcount);
   ArrayResizeAL(ensemble.m_tmpmeans,ccount);
   ArrayResizeAL(ensemble.m_tmpsigmas,ccount);
   ArrayResizeAL(ensemble.m_neurons,ensemble.m_structinfo[m_mlpntotaloffset]);
   ArrayResizeAL(ensemble.m_dfdnet,ensemble.m_structinfo[m_mlpntotaloffset]);
   ArrayResizeAL(ensemble.m_y,ensemble.m_nout);
  }
//+------------------------------------------------------------------+
//| Copying of MLPEnsemble strucure                                  |
//| INPUT PARAMETERS:                                                |
//|     Ensemble1 -   original                                       |
//| OUTPUT PARAMETERS:                                               |
//|     Ensemble2 -   copy                                           |
//+------------------------------------------------------------------+
static void CMLPE::MLPECopy(CMLPEnsemble &ensemble1,CMLPEnsemble &ensemble2)
  {
//--- create variables
   int i=0;
   int ssize=0;
   int ccount=0;
   int ntotal=0;
   int i_=0;
//--- Unload info
   ssize=ensemble1.m_structinfo[0];
//--- check
   if(ensemble1.m_issoftmax)
      ccount=ensemble1.m_nin;
   else
      ccount=ensemble1.m_nin+ensemble1.m_nout;
//--- change value
   ntotal=ensemble1.m_structinfo[m_mlpntotaloffset];
//--- Allocate space
   ArrayResizeAL(ensemble2.m_structinfo,ssize);
   ArrayResizeAL(ensemble2.m_weights,ensemble1.m_ensemblesize*ensemble1.m_wcount);
   ArrayResizeAL(ensemble2.m_columnmeans,ensemble1.m_ensemblesize*ccount);
   ArrayResizeAL(ensemble2.m_columnsigmas,ensemble1.m_ensemblesize*ccount);
   ArrayResizeAL(ensemble2.m_tmpweights,ensemble1.m_wcount);
   ArrayResizeAL(ensemble2.m_tmpmeans,ccount);
   ArrayResizeAL(ensemble2.m_tmpsigmas,ccount);
   ArrayResizeAL(ensemble2.m_serializedmlp,ensemble1.m_serializedlen);
   ArrayResizeAL(ensemble2.m_neurons,ntotal);
   ArrayResizeAL(ensemble2.m_dfdnet,ntotal);
   ArrayResizeAL(ensemble2.m_y,ensemble1.m_nout);
//--- Copy
   ensemble2.m_nin=ensemble1.m_nin;
   ensemble2.m_nout=ensemble1.m_nout;
   ensemble2.m_wcount=ensemble1.m_wcount;
   ensemble2.m_ensemblesize=ensemble1.m_ensemblesize;
   ensemble2.m_issoftmax=ensemble1.m_issoftmax;
   ensemble2.m_postprocessing=ensemble1.m_postprocessing;
   ensemble2.m_serializedlen=ensemble1.m_serializedlen;
//--- copy
   for(i=0;i<=ssize-1;i++)
      ensemble2.m_structinfo[i]=ensemble1.m_structinfo[i];
   for(i_=0;i_<=ensemble1.m_ensemblesize*ensemble1.m_wcount-1;i_++)
      ensemble2.m_weights[i_]=ensemble1.m_weights[i_];
   for(i_=0;i_<=ensemble1.m_ensemblesize*ccount-1;i_++)
      ensemble2.m_columnmeans[i_]=ensemble1.m_columnmeans[i_];
   for(i_=0;i_<=ensemble1.m_ensemblesize*ccount-1;i_++)
      ensemble2.m_columnsigmas[i_]=ensemble1.m_columnsigmas[i_];
   for(i_=0;i_<=ensemble1.m_serializedlen-1;i_++)
      ensemble2.m_serializedmlp[i_]=ensemble1.m_serializedmlp[i_];
  }
//+------------------------------------------------------------------+
//| Serialization of MLPEnsemble strucure                            |
//| INPUT PARAMETERS:                                                |
//|     Ensemble-   original                                         |
//| OUTPUT PARAMETERS:                                               |
//|     RA      -   array of real numbers which stores ensemble,     |
//|                 array[0..RLen-1]                                 |
//|     RLen    -   RA lenght                                        |
//+------------------------------------------------------------------+
static void CMLPE::MLPESerialize(CMLPEnsemble &ensemble,double &ra[],int &rlen)
  {
//--- create variables
   int i=0;
   int ssize=0;
   int ntotal=0;
   int ccount=0;
   int hsize=0;
   int offs=0;
   int i_=0;
   int i1_=0;
//--- initialization
   rlen=0;
   hsize=13;
   ssize=ensemble.m_structinfo[0];
//--- check
   if(ensemble.m_issoftmax)
      ccount=ensemble.m_nin;
   else
      ccount=ensemble.m_nin+ensemble.m_nout;
//--- change values
   ntotal=ensemble.m_structinfo[m_mlpntotaloffset];
   rlen=hsize+ssize+ensemble.m_ensemblesize*ensemble.m_wcount+2*ccount*ensemble.m_ensemblesize+ensemble.m_serializedlen;
//---  RA format:
//---  [0]     RLen
//---  [1]     Version (MLPEVNum)
//---  [2]     EnsembleSize
//---  [3]     NIn
//---  [4]     NOut
//---  [5]     WCount
//---  [6]     IsSoftmax 0/1
//---  [7]     PostProcessing 0/1
//---  [8]     sizeof(StructInfo)
//---  [9]     NTotal (sizeof(Neurons),sizeof(DFDNET))
//---  [10]    CCount (sizeof(ColumnMeans),sizeof(ColumnSigmas))
//---  [11]    data offset
//---  [12]    SerializedLen
//---  [..]    StructInfo
//---  [..]    Weights
//---  [..]    ColumnMeans
//---  [..]    ColumnSigmas
   ArrayResizeAL(ra,rlen);
//--- change values
   ra[0]=rlen;
   ra[1]=m_mlpevnum;
   ra[2]=ensemble.m_ensemblesize;
   ra[3]=ensemble.m_nin;
   ra[4]=ensemble.m_nout;
   ra[5]=ensemble.m_wcount;
//--- check
   if(ensemble.m_issoftmax)
      ra[6]=1;
   else
      ra[6]=0;
//--- check
   if(ensemble.m_postprocessing)
      ra[7]=1;
   else
      ra[7]=9;
//--- change values
   ra[8]=ssize;
   ra[9]=ntotal;
   ra[10]=ccount;
   ra[11]=hsize;
   ra[12]=ensemble.m_serializedlen;
//--- copy
   offs=hsize;
   for(i=offs;i<=offs+ssize-1;i++)
      ra[i]=ensemble.m_structinfo[i-offs];
//--- copy
   offs=offs+ssize;
   i1_=-offs;
   for(i_=offs;i_<=offs+ensemble.m_ensemblesize*ensemble.m_wcount-1;i_++)
      ra[i_]=ensemble.m_weights[i_+i1_];
//--- copy
   offs=offs+ensemble.m_ensemblesize*ensemble.m_wcount;
   i1_=-offs;
   for(i_=offs;i_<=offs+ensemble.m_ensemblesize*ccount-1;i_++)
      ra[i_]=ensemble.m_columnmeans[i_+i1_];
//--- copy
   offs=offs+ensemble.m_ensemblesize*ccount;
   i1_=-offs;
   for(i_=offs;i_<=offs+ensemble.m_ensemblesize*ccount-1;i_++)
      ra[i_]=ensemble.m_columnsigmas[i_+i1_];
//--- copy
   offs=offs+ensemble.m_ensemblesize*ccount;
   i1_=-offs;
   for(i_=offs;i_<=offs+ensemble.m_serializedlen-1;i_++)
      ra[i_]=ensemble.m_serializedmlp[i_+i1_];
   offs=offs+ensemble.m_serializedlen;
  }
//+------------------------------------------------------------------+
//| Unserialization of MLPEnsemble strucure                          |
//| INPUT PARAMETERS:                                                |
//|     RA      -   real array which stores ensemble                 |
//| OUTPUT PARAMETERS:                                               |
//|     Ensemble-   restored structure                               |
//+------------------------------------------------------------------+
static void CMLPE::MLPEUnserialize(double &ra[],CMLPEnsemble &ensemble)
  {
//--- create variables
   int i=0;
   int ssize=0;
   int ntotal=0;
   int ccount=0;
   int hsize=0;
   int offs=0;
   int i_=0;
   int i1_=0;
//--- check
   if(!CAp::Assert((int)MathRound(ra[1])==m_mlpevnum,__FUNCTION__+": incorrect array!"))
      return;
//--- load info
   hsize=13;
   ensemble.m_ensemblesize=(int)MathRound(ra[2]);
   ensemble.m_nin=(int)MathRound(ra[3]);
   ensemble.m_nout=(int)MathRound(ra[4]);
   ensemble.m_wcount=(int)MathRound(ra[5]);
   ensemble.m_issoftmax=(int)MathRound(ra[6])==1;
   ensemble.m_postprocessing=(int)MathRound(ra[7])==1;
   ssize=(int)MathRound(ra[8]);
   ntotal=(int)MathRound(ra[9]);
   ccount=(int)MathRound(ra[10]);
   offs=(int)MathRound(ra[11]);
   ensemble.m_serializedlen=(int)MathRound(ra[12]);
//---  Allocate arrays
   ArrayResizeAL(ensemble.m_structinfo,ssize);
   ArrayResizeAL(ensemble.m_weights,ensemble.m_ensemblesize*ensemble.m_wcount);
   ArrayResizeAL(ensemble.m_columnmeans,ensemble.m_ensemblesize*ccount);
   ArrayResizeAL(ensemble.m_columnsigmas,ensemble.m_ensemblesize*ccount);
   ArrayResizeAL(ensemble.m_tmpweights,ensemble.m_wcount);
   ArrayResizeAL(ensemble.m_tmpmeans,ccount);
   ArrayResizeAL(ensemble.m_tmpsigmas,ccount);
   ArrayResizeAL(ensemble.m_neurons,ntotal);
   ArrayResizeAL(ensemble.m_dfdnet,ntotal);
   ArrayResizeAL(ensemble.m_serializedmlp,ensemble.m_serializedlen);
   ArrayResizeAL(ensemble.m_y,ensemble.m_nout);
//--- load data
   for(i=offs;i<=offs+ssize-1;i++)
      ensemble.m_structinfo[i-offs]=(int)MathRound(ra[i]);
//--- copy
   offs=offs+ssize;
   i1_=offs;
   for(i_=0;i_<=ensemble.m_ensemblesize*ensemble.m_wcount-1;i_++)
      ensemble.m_weights[i_]=ra[i_+i1_];
//--- copy
   offs=offs+ensemble.m_ensemblesize*ensemble.m_wcount;
   i1_=offs;
   for(i_=0;i_<=ensemble.m_ensemblesize*ccount-1;i_++)
      ensemble.m_columnmeans[i_]=ra[i_+i1_];
//--- copy
   offs=offs+ensemble.m_ensemblesize*ccount;
   i1_=offs;
   for(i_=0;i_<=ensemble.m_ensemblesize*ccount-1;i_++)
      ensemble.m_columnsigmas[i_]=ra[i_+i1_];
//--- copy
   offs=offs+ensemble.m_ensemblesize*ccount;
   i1_=offs;
   for(i_=0;i_<=ensemble.m_serializedlen-1;i_++)
      ensemble.m_serializedmlp[i_]=ra[i_+i1_];
   offs=offs+ensemble.m_serializedlen;
  }
//+------------------------------------------------------------------+
//| Randomization of MLP ensemble                                    |
//+------------------------------------------------------------------+
static void CMLPE::MLPERandomize(CMLPEnsemble &ensemble)
  {
//--- create a variable
   int i=0;
//--- calculation
   for(i=0;i<=ensemble.m_ensemblesize*ensemble.m_wcount-1;i++)
      ensemble.m_weights[i]=CMath::RandomReal()-0.5;
  }
//+------------------------------------------------------------------+
//| Return ensemble properties (number of inputs and outputs).       |
//+------------------------------------------------------------------+
static void CMLPE::MLPEProperties(CMLPEnsemble &ensemble,int &nin,int &nout)
  {
//--- change values
   nin=ensemble.m_nin;
   nout=ensemble.m_nout;
  }
//+------------------------------------------------------------------+
//| Return normalization type (whether ensemble is SOFTMAX-normalized|
//| or not).                                                         |
//+------------------------------------------------------------------+
static bool CMLPE::MLPEIsSoftMax(CMLPEnsemble &ensemble)
  {
//--- return result
   return(ensemble.m_issoftmax);
  }
//+------------------------------------------------------------------+
//| Procesing                                                        |
//| INPUT PARAMETERS:                                                |
//|     Ensemble-   neural networks ensemble                         |
//|     X       -   input vector,  array[0..NIn-1].                  |
//|     Y       -   (possibly) preallocated buffer; if size of Y is  |
//|                 less than NOut, it will be reallocated. If it is |
//|                 large enough, it is NOT reallocated, so we can   |
//|                 save some time on reallocation.                  |
//| OUTPUT PARAMETERS:                                               |
//|     Y       -   result. Regression estimate when solving         |
//|                 regression task, vector of posterior             |
//|                 probabilities for classification task.           |
//+------------------------------------------------------------------+
static void CMLPE::MLPEProcess(CMLPEnsemble &ensemble,double &x[],double &y[])
  {
//--- create variables
   int    i=0;
   int    es=0;
   int    wc=0;
   int    cc=0;
   double v=0;
   int    i_=0;
   int    i1_=0;
//--- check
   if(CAp::Len(y)<ensemble.m_nout)
      ArrayResizeAL(y,ensemble.m_nout);
//--- initialization
   es=ensemble.m_ensemblesize;
   wc=ensemble.m_wcount;
//--- check
   if(ensemble.m_issoftmax)
      cc=ensemble.m_nin;
   else
      cc=ensemble.m_nin+ensemble.m_nout;
//--- initialization
   v=1.0/(double)es;
   for(i=0;i<=ensemble.m_nout-1;i++)
      y[i]=0;
//--- calculation
   for(i=0;i<=es-1;i++)
     {
      i1_=i*wc;
      for(i_=0;i_<=wc-1;i_++)
         ensemble.m_tmpweights[i_]=ensemble.m_weights[i_+i1_];
      i1_=i*cc;
      for(i_=0;i_<=cc-1;i_++)
         ensemble.m_tmpmeans[i_]=ensemble.m_columnmeans[i_+i1_];
      i1_=i*cc;
      for(i_=0;i_<=cc-1;i_++)
         ensemble.m_tmpsigmas[i_]=ensemble.m_columnsigmas[i_+i1_];
      //--- function call
      CMLPBase::MLPInternalProcessVector(ensemble.m_structinfo,ensemble.m_tmpweights,ensemble.m_tmpmeans,ensemble.m_tmpsigmas,ensemble.m_neurons,ensemble.m_dfdnet,x,ensemble.m_y);
      //--- change values
      for(i_=0;i_<=ensemble.m_nout-1;i_++)
         y[i_]=y[i_]+v*ensemble.m_y[i_];
     }
  }
//+------------------------------------------------------------------+
//| 'interactive' variant of MLPEProcess for languages like Python   |
//| which support constructs like "Y = MLPEProcess(LM,X)" and        |
//| interactive mode of the interpreter                              |
//| This function allocates new array on each call, so it is         |
//| significantly slower than its 'non-interactive' counterpart, but |
//| it is more convenient when you call it from command line.        |
//+------------------------------------------------------------------+
static void CMLPE::MLPEProcessI(CMLPEnsemble &ensemble,double &x[],double &y[])
  {
//--- function call
   MLPEProcess(ensemble,x,y);
  }
//+------------------------------------------------------------------+
//| Relative classification error on the test set                    |
//| INPUT PARAMETERS:                                                |
//|     Ensemble-   ensemble                                         |
//|     XY      -   test set                                         |
//|     NPoints -   test set size                                    |
//| RESULT:                                                          |
//|     percent of incorrectly classified cases.                     |
//|     Works both for classifier betwork and for regression networks|
//|     which are used as classifiers.                               |
//+------------------------------------------------------------------+
static double CMLPE::MLPERelClsError(CMLPEnsemble &ensemble,CMatrixDouble &xy,
                                     const int npoints)
  {
//--- create variables
   double relcls=0;
   double avgce=0;
   double rms=0;
   double avg=0;
   double avgrel=0;
//--- function call
   MLPEAllErrors(ensemble,xy,npoints,relcls,avgce,rms,avg,avgrel);
//--- return result
   return(relcls);
  }
//+------------------------------------------------------------------+
//| Average cross-entropy (in bits per element) on the test set      |
//| INPUT PARAMETERS:                                                |
//|     Ensemble-   ensemble                                         |
//|     XY      -   test set                                         |
//|     NPoints -   test set size                                    |
//| RESULT:                                                          |
//|     CrossEntropy/(NPoints*LN(2)).                                |
//|     Zero if ensemble solves regression task.                     |
//+------------------------------------------------------------------+
static double CMLPE::MLPEAvgCE(CMLPEnsemble &ensemble,CMatrixDouble &xy,
                               const int npoints)
  {
//--- create variables
   double relcls=0;
   double avgce=0;
   double rms=0;
   double avg=0;
   double avgrel=0;
//--- function call
   MLPEAllErrors(ensemble,xy,npoints,relcls,avgce,rms,avg,avgrel);
//--- return result
   return(avgce);
  }
//+------------------------------------------------------------------+
//| RMS error on the test set                                        |
//| INPUT PARAMETERS:                                                |
//|     Ensemble-   ensemble                                         |
//|     XY      -   test set                                         |
//|     NPoints -   test set size                                    |
//| RESULT:                                                          |
//|     root mean square error.                                      |
//|     Its meaning for regression task is obvious. As for           |
//|     classification task RMS error means error when estimating    |
//|     posterior probabilities.                                     |
//+------------------------------------------------------------------+
static double CMLPE::MLPERMSError(CMLPEnsemble &ensemble,CMatrixDouble &xy,
                                  const int npoints)
  {
//--- create variables
   double relcls=0;
   double avgce=0;
   double rms=0;
   double avg=0;
   double avgrel=0;
//--- function call
   MLPEAllErrors(ensemble,xy,npoints,relcls,avgce,rms,avg,avgrel);
//--- return result
   return(rms);
  }
//+------------------------------------------------------------------+
//| Average error on the test set                                    |
//| INPUT PARAMETERS:                                                |
//|     Ensemble-   ensemble                                         |
//|     XY      -   test set                                         |
//|     NPoints -   test set size                                    |
//| RESULT:                                                          |
//|     Its meaning for regression task is obvious. As for           |
//|     classification task it means average error when estimating   |
//|     posterior probabilities.                                     |
//+------------------------------------------------------------------+
static double CMLPE::MLPEAvgError(CMLPEnsemble &ensemble,CMatrixDouble &xy,
                                  const int npoints)
  {
//--- create variables
   double result=0;
   double relcls=0;
   double avgce=0;
   double rms=0;
   double avg=0;
   double avgrel=0;
//--- function call
   MLPEAllErrors(ensemble,xy,npoints,relcls,avgce,rms,avg,avgrel);
//--- return result
   return(avg);
  }
//+------------------------------------------------------------------+
//| Average relative error on the test set                           |
//| INPUT PARAMETERS:                                                |
//|     Ensemble-   ensemble                                         |
//|     XY      -   test set                                         |
//|     NPoints -   test set size                                    |
//| RESULT:                                                          |
//|     Its meaning for regression task is obvious. As for           |
//|     classification task it means average relative error when     |
//|     estimating posterior probabilities.                          |
//+------------------------------------------------------------------+
static double CMLPE::MLPEAvgRelError(CMLPEnsemble &ensemble,CMatrixDouble &xy,
                                     const int npoints)
  {
//--- create variables
   double result=0;
   double relcls=0;
   double avgce=0;
   double rms=0;
   double avg=0;
   double avgrel=0;
//--- function call
   MLPEAllErrors(ensemble,xy,npoints,relcls,avgce,rms,avg,avgrel);
//--- return result
   return(avgrel);
  }
//+------------------------------------------------------------------+
//| Training neural networks ensemble using  bootstrap  aggregating  |
//| (bagging).                                                       |
//| Modified Levenberg-Marquardt algorithm is used as base training  |
//| method.                                                          |
//| INPUT PARAMETERS:                                                |
//|     Ensemble    -   model with initialized geometry              |
//|     XY          -   training set                                 |
//|     NPoints     -   training set size                            |
//|     Decay       -   weight decay coefficient, >=0.001            |
//|     Restarts    -   restarts, >0.                                |
//| OUTPUT PARAMETERS:                                               |
//|     Ensemble    -   trained model                                |
//|     Info        -   return code:                                 |
//|                     * -2, if there is a point with class number  |
//|                           outside of [0..NClasses-1].            |
//|                     * -1, if incorrect parameters was passed     |
//|                           (NPoints<0, Restarts<1).               |
//|                     *  2, if task has been solved.               |
//|     Rep         -   training report.                             |
//|     OOBErrors   -   out-of-bag generalization error estimate     |
//+------------------------------------------------------------------+
static void CMLPE::MLPEBaggingLM(CMLPEnsemble &ensemble,CMatrixDouble &xy,
                                 const int npoints,const double decay,
                                 const int restarts,int &info,CMLPReport &rep,
                                 CMLPCVReport &ooberrors)
  {
//--- initialization
   info=0;
//--- function call
   MLPEBaggingInternal(ensemble,xy,npoints,decay,restarts,0.0,0,true,info,rep,ooberrors);
  }
//+------------------------------------------------------------------+
//| Training neural networks ensemble using  bootstrap aggregating   |
//| (bagging). L-BFGS algorithm is used as base training method.     |
//| INPUT PARAMETERS:                                                |
//|     Ensemble    -   model with initialized geometry              |
//|     XY          -   training set                                 |
//|     NPoints     -   training set size                            |
//|     Decay       -   weight decay coefficient, >=0.001            |
//|     Restarts    -   restarts, >0.                                |
//|     WStep       -   stopping criterion, same as in MLPTrainLBFGS |
//|     MaxIts      -   stopping criterion, same as in MLPTrainLBFGS |
//| OUTPUT PARAMETERS:                                               |
//|     Ensemble    -   trained model                                |
//|     Info        -   return code:                                 |
//|                     * -8, if both WStep=0 and MaxIts=0           |
//|                     * -2, if there is a point with class number  |
//|                           outside of [0..NClasses-1].            |
//|                     * -1, if incorrect parameters was passed     |
//|                           (NPoints<0, Restarts<1).               |
//|                     *  2, if task has been solved.               |
//|     Rep         -   training report.                             |
//|     OOBErrors   -   out-of-bag generalization error estimate     |
//+------------------------------------------------------------------+
static void CMLPE::MLPEBaggingLBFGS(CMLPEnsemble &ensemble,CMatrixDouble &xy,
                                    const int npoints,const double decay,
                                    const int restarts,const double wstep,
                                    const int maxits,int &info,CMLPReport &rep,
                                    CMLPCVReport &ooberrors)
  {
//--- initialization
   info=0;
//--- function call
   MLPEBaggingInternal(ensemble,xy,npoints,decay,restarts,wstep,maxits,false,info,rep,ooberrors);
  }
//+------------------------------------------------------------------+
//| Training neural networks ensemble using early stopping.          |
//| INPUT PARAMETERS:                                                |
//|     Ensemble    -   model with initialized geometry              |
//|     XY          -   training set                                 |
//|     NPoints     -   training set size                            |
//|     Decay       -   weight decay coefficient, >=0.001            |
//|     Restarts    -   restarts, >0.                                |
//| OUTPUT PARAMETERS:                                               |
//|     Ensemble    -   trained model                                |
//|     Info        -   return code:                                 |
//|                     * -2, if there is a point with class number  |
//|                           outside of [0..NClasses-1].            |
//|                     * -1, if incorrect parameters was passed     |
//|                           (NPoints<0, Restarts<1).               |
//|                     *  6, if task has been solved.               |
//|     Rep         -   training report.                             |
//|     OOBErrors   -   out-of-bag generalization error estimate     |
//+------------------------------------------------------------------+
static void CMLPE::MLPETrainES(CMLPEnsemble &ensemble,CMatrixDouble &xy,
                               const int npoints,const double decay,
                               const int restarts,int &info,CMLPReport &rep)
  {
//--- create variables
   int i=0;
   int k=0;
   int ccount=0;
   int pcount=0;
   int trnsize=0;
   int valsize=0;
   int tmpinfo=0;
   int i_=0;
   int i1_=0;
//--- create matrix
   CMatrixDouble trnxy;
   CMatrixDouble valxy;
//--- objects of classes
   CMultilayerPerceptron network;
   CMLPReport            tmprep;
//--- initialization
   info=0;
//--- check
   if((npoints<2 || restarts<1) || decay<0.0)
     {
      info=-1;
      return;
     }
//--- check
   if(ensemble.m_issoftmax)
     {
      for(i=0;i<=npoints-1;i++)
        {
         //--- check
         if((int)MathRound(xy[i][ensemble.m_nin])<0 || (int)MathRound(xy[i][ensemble.m_nin])>=ensemble.m_nout)
           {
            info=-2;
            return;
           }
        }
     }
//--- change value
   info=6;
//--- allocate
   if(ensemble.m_issoftmax)
     {
      ccount=ensemble.m_nin+1;
      pcount=ensemble.m_nin;
     }
   else
     {
      ccount=ensemble.m_nin+ensemble.m_nout;
      pcount=ensemble.m_nin+ensemble.m_nout;
     }
//--- allocation
   trnxy.Resize(npoints,ccount);
   valxy.Resize(npoints,ccount);
//--- function call
   CMLPBase::MLPUnserializeOld(ensemble.m_serializedmlp,network);
//--- change values
   rep.m_ngrad=0;
   rep.m_nhess=0;
   rep.m_ncholesky=0;
//--- train networks
   for(k=0;k<=ensemble.m_ensemblesize-1;k++)
     {
      //--- Split set
      do
        {
         trnsize=0;
         valsize=0;
         for(i=0;i<=npoints-1;i++)
           {
            //--- check
            if(CMath::RandomReal()<0.66)
              {
               //--- Assign sample to training set
               for(i_=0;i_<=ccount-1;i_++)
                  trnxy[trnsize].Set(i_,xy[i][i_]);
               trnsize=trnsize+1;
              }
            else
              {
               //--- Assign sample to validation set
               for(i_=0;i_<=ccount-1;i_++)
                  valxy[valsize].Set(i_,xy[i][i_]);
               valsize=valsize+1;
              }
           }
        }
      while(!(trnsize!=0 && valsize!=0));
      //--- Train
      CMLPTrain::MLPTrainES(network,trnxy,trnsize,valxy,valsize,decay,restarts,tmpinfo,tmprep);
      //--- check
      if(tmpinfo<0)
        {
         info=tmpinfo;
         return;
        }
      //--- save results
      i1_=-(k*ensemble.m_wcount);
      for(i_=k*ensemble.m_wcount;i_<=(k+1)*ensemble.m_wcount-1;i_++)
         ensemble.m_weights[i_]=network.m_weights[i_+i1_];
      i1_=-(k*pcount);
      for(i_=k*pcount;i_<=(k+1)*pcount-1;i_++)
         ensemble.m_columnmeans[i_]=network.m_columnmeans[i_+i1_];
      i1_=-(k*pcount);
      for(i_=k*pcount;i_<=(k+1)*pcount-1;i_++)
         ensemble.m_columnsigmas[i_]=network.m_columnsigmas[i_+i1_];
      //--- change values
      rep.m_ngrad=rep.m_ngrad+tmprep.m_ngrad;
      rep.m_nhess=rep.m_nhess+tmprep.m_nhess;
      rep.m_ncholesky=rep.m_ncholesky+tmprep.m_ncholesky;
     }
  }
//+------------------------------------------------------------------+
//| Calculation of all types of errors                               |
//+------------------------------------------------------------------+
static void CMLPE::MLPEAllErrors(CMLPEnsemble &ensemble,CMatrixDouble &xy,
                                 const int npoints,double &relcls,
                                 double &avgce,double &rms,
                                 double &avg,double &avgrel)
  {
//--- create variables
   int i=0;
   int i_=0;
   int i1_=0;
//--- creating arrays
   double buf[];
   double workx[];
   double y[];
   double dy[];
//--- initialization
   relcls=0;
   avgce=0;
   rms=0;
   avg=0;
   avgrel=0;
//--- allocation
   ArrayResizeAL(workx,ensemble.m_nin);
   ArrayResizeAL(y,ensemble.m_nout);
//--- check
   if(ensemble.m_issoftmax)
     {
      //--- allocation
      ArrayResizeAL(dy,1);
      //--- function call
      CBdSS::DSErrAllocate(ensemble.m_nout,buf);
     }
   else
     {
      //--- allocation
      ArrayResizeAL(dy,ensemble.m_nout);
      //--- function call
      CBdSS::DSErrAllocate(-ensemble.m_nout,buf);
     }
//--- calculation
   for(i=0;i<=npoints-1;i++)
     {
      for(i_=0;i_<=ensemble.m_nin-1;i_++)
         workx[i_]=xy[i][i_];
      //--- function call
      MLPEProcess(ensemble,workx,y);
      //--- check
      if(ensemble.m_issoftmax)
         dy[0]=xy[i][ensemble.m_nin];
      else
        {
         i1_=ensemble.m_nin;
         for(i_=0;i_<=ensemble.m_nout-1;i_++)
            dy[i_]=xy[i][i_+i1_];
        }
      //--- function call
      CBdSS::DSErrAccumulate(buf,y,dy);
     }
//--- function call
   CBdSS::DSErrFinish(buf);
//--- change values
   relcls=buf[0];
   avgce=buf[1];
   rms=buf[2];
   avg=buf[3];
   avgrel=buf[4];
  }
//+------------------------------------------------------------------+
//| Internal bagging subroutine.                                     |
//+------------------------------------------------------------------+
static void CMLPE::MLPEBaggingInternal(CMLPEnsemble &ensemble,CMatrixDouble &xy,
                                       const int npoints,const double decay,
                                       const int restarts,const double wstep,
                                       const int maxits,const bool lmalgorithm,
                                       int &info,CMLPReport &rep,CMLPCVReport &ooberrors)
  {
//--- create variables
   int nin=0;
   int nout=0;
   int ccnt=0;
   int pcnt=0;
   int i=0;
   int j=0;
   int k=0;
   double v=0;
   int i_=0;
   int i1_=0;
//--- creating arrays
   bool   s[];
   int    oobcntbuf[];
   double x[];
   double y[];
   double dy[];
   double dsbuf[];
//--- create matrix
   CMatrixDouble xys;
   CMatrixDouble oobbuf;
//--- objects of classes
   CMLPReport            tmprep;
   CMultilayerPerceptron network;
//--- initialization
   info=0;
//--- Test for inputs
   if((!lmalgorithm && wstep==0.0) && maxits==0)
     {
      info=-8;
      return;
     }
//--- check
   if(((npoints<=0 || restarts<1) || wstep<0.0) || maxits<0)
     {
      info=-1;
      return;
     }
//--- check
   if(ensemble.m_issoftmax)
     {
      for(i=0;i<=npoints-1;i++)
        {
         //--- check
         if((int)MathRound(xy[i][ensemble.m_nin])<0 || (int)MathRound(xy[i][ensemble.m_nin])>=ensemble.m_nout)
           {
            info=-2;
            return;
           }
        }
     }
//--- allocate temporaries
   info=2;
   rep.m_ngrad=0;
   rep.m_nhess=0;
   rep.m_ncholesky=0;
   ooberrors.m_relclserror=0;
   ooberrors.m_avgce=0;
   ooberrors.m_rmserror=0;
   ooberrors.m_avgerror=0;
   ooberrors.m_avgrelerror=0;
   nin=ensemble.m_nin;
   nout=ensemble.m_nout;
//--- check
   if(ensemble.m_issoftmax)
     {
      ccnt=nin+1;
      pcnt=nin;
     }
   else
     {
      ccnt=nin+nout;
      pcnt=nin+nout;
     }
//--- allocation
   xys.Resize(npoints,ccnt);
   ArrayResizeAL(s,npoints);
   oobbuf.Resize(npoints,nout);
   ArrayResizeAL(oobcntbuf,npoints);
   ArrayResizeAL(x,nin);
   ArrayResizeAL(y,nout);
//--- check
   if(ensemble.m_issoftmax)
      ArrayResizeAL(dy,1);
   else
      ArrayResizeAL(dy,nout);
//--- initialization
   for(i=0;i<=npoints-1;i++)
     {
      for(j=0;j<=nout-1;j++)
         oobbuf[i].Set(j,0);
     }
   for(i=0;i<=npoints-1;i++)
      oobcntbuf[i]=0;
//--- function call
   CMLPBase::MLPUnserializeOld(ensemble.m_serializedmlp,network);
//--- main bagging cycle
   for(k=0;k<=ensemble.m_ensemblesize-1;k++)
     {
      //--- prepare dataset
      for(i=0;i<=npoints-1;i++)
         s[i]=false;
      for(i=0;i<=npoints-1;i++)
        {
         j=CMath::RandomInteger(npoints);
         s[j]=true;
         for(i_=0;i_<=ccnt-1;i_++)
            xys[i].Set(i_,xy[j][i_]);
        }
      //--- train
      if(lmalgorithm)
         CMLPTrain::MLPTrainLM(network,xys,npoints,decay,restarts,info,tmprep);
      else
         CMLPTrain::MLPTrainLBFGS(network,xys,npoints,decay,restarts,wstep,maxits,info,tmprep);
      //--- check
      if(info<0)
         return;
      //--- save results
      rep.m_ngrad=rep.m_ngrad+tmprep.m_ngrad;
      rep.m_nhess=rep.m_nhess+tmprep.m_nhess;
      rep.m_ncholesky=rep.m_ncholesky+tmprep.m_ncholesky;
      //--- copy
      i1_=-(k*ensemble.m_wcount);
      for(i_=k*ensemble.m_wcount;i_<=(k+1)*ensemble.m_wcount-1;i_++)
         ensemble.m_weights[i_]=network.m_weights[i_+i1_];
      //--- copy
      i1_=-(k*pcnt);
      for(i_=k*pcnt;i_<=(k+1)*pcnt-1;i_++)
         ensemble.m_columnmeans[i_]=network.m_columnmeans[i_+i1_];
      //--- copy
      i1_=-(k*pcnt);
      for(i_=k*pcnt;i_<=(k+1)*pcnt-1;i_++)
         ensemble.m_columnsigmas[i_]=network.m_columnsigmas[i_+i1_];
      //--- OOB estimates
      for(i=0;i<=npoints-1;i++)
        {
         //--- check
         if(!s[i])
           {
            for(i_=0;i_<=nin-1;i_++)
               x[i_]=xy[i][i_];
            //--- function call
            CMLPBase::MLPProcess(network,x,y);
            //--- change value
            for(i_=0;i_<=nout-1;i_++)
               oobbuf[i].Set(i_,oobbuf[i][i_]+y[i_]);
            oobcntbuf[i]=oobcntbuf[i]+1;
           }
        }
     }
//--- OOB estimates
   if(ensemble.m_issoftmax)
     {
      //--- function call
      CBdSS::DSErrAllocate(nout,dsbuf);
     }
   else
     {
      //--- function call
      CBdSS::DSErrAllocate(-nout,dsbuf);
     }
   for(i=0;i<=npoints-1;i++)
     {
      //--- check
      if(oobcntbuf[i]!=0)
        {
         v=1.0/(double)oobcntbuf[i];
         for(i_=0;i_<=nout-1;i_++)
            y[i_]=v*oobbuf[i][i_];
         //--- check
         if(ensemble.m_issoftmax)
            dy[0]=xy[i][nin];
         else
           {
            i1_=nin;
            for(i_=0;i_<=nout-1;i_++)
               dy[i_]=v*xy[i][i_+i1_];
           }
         //--- function call
         CBdSS::DSErrAccumulate(dsbuf,y,dy);
        }
     }
//--- function call
   CBdSS::DSErrFinish(dsbuf);
//--- change values
   ooberrors.m_relclserror=dsbuf[0];
   ooberrors.m_avgce=dsbuf[1];
   ooberrors.m_rmserror=dsbuf[2];
   ooberrors.m_avgerror=dsbuf[3];
   ooberrors.m_avgrelerror=dsbuf[4];
  }
//+------------------------------------------------------------------+
//| Principal components analysis                                    |
//+------------------------------------------------------------------+
class CPCAnalysis
  {
public:
   //--- constructor, destructor
                     CPCAnalysis(void);
                    ~CPCAnalysis(void);
   //--- method
   static void       PCABuildBasis(CMatrixDouble &x,const int npoints,const int nvars,int &info,double &s2[],CMatrixDouble &v);
  };
//+------------------------------------------------------------------+
//| Constructor without parameters                                   |
//+------------------------------------------------------------------+
CPCAnalysis::CPCAnalysis(void)
  {

  }
//+------------------------------------------------------------------+
//| Destructor                                                       |
//+------------------------------------------------------------------+
CPCAnalysis::~CPCAnalysis(void)
  {

  }
//+------------------------------------------------------------------+
//| Principal components analysis                                    |
//| Subroutine builds orthogonal basis where first axis corresponds  | 
//| to direction with maximum variance, second axis maximizes        |
//| variance in subspace orthogonal to first axis and so on.         |
//| It should be noted that, unlike LDA, PCA does not use class      |
//| labels.                                                          |
//| INPUT PARAMETERS:                                                |
//|     X           -   dataset, array[0..NPoints-1,0..NVars-1].     |
//|                     matrix contains ONLY INDEPENDENT VARIABLES.  |
//|     NPoints     -   dataset size, NPoints>=0                     |
//|     NVars       -   number of independent variables, NVars>=1    |
//| OUTPUT PARAMETERS:                                               |
//|     Info        -   return code:                                 |
//|                     * -4, if SVD subroutine haven't converged    |
//|                     * -1, if wrong parameters has been passed    |
//|                           (NPoints<0, NVars<1)                   |
//|                     *  1, if task is solved                      |
//|     S2          -   array[0..NVars-1]. variance values           |
//|                     corresponding to basis vectors.              |
//|     V           -   array[0..NVars-1,0..NVars-1]                 |
//|                     matrix, whose columns store basis vectors.   |
//+------------------------------------------------------------------+
static void CPCAnalysis::PCABuildBasis(CMatrixDouble &x,const int npoints,
                                       const int nvars,int &info,double &s2[],
                                       CMatrixDouble &v)
  {
//--- create variables
   int    i=0;
   int    j=0;
   double mean=0;
   double variance=0;
   double skewness=0;
   double kurtosis=0;
   int    i_=0;
//--- creating arrays
   double m[];
   double t[];
//--- create matrix
   CMatrixDouble a;
   CMatrixDouble u;
   CMatrixDouble vt;
//--- initialization
   info=0;
//--- Check input data
   if(npoints<0 || nvars<1)
     {
      info=-1;
      return;
     }
//--- change value
   info=1;
//--- Special case: NPoints=0
   if(npoints==0)
     {
      //--- allocation
      ArrayResizeAL(s2,nvars);
      v.Resize(nvars,nvars);
      //--- initialization
      for(i=0;i<=nvars-1;i++)
         s2[i]=0;
      for(i=0;i<=nvars-1;i++)
        {
         for(j=0;j<=nvars-1;j++)
           {
            //--- check
            if(i==j)
               v[i].Set(j,1);
            else
               v[i].Set(j,0);
           }
        }
      //--- exit the function
      return;
     }
//--- Calculate means
   ArrayResizeAL(m,nvars);
   ArrayResizeAL(t,npoints);
   for(j=0;j<=nvars-1;j++)
     {
      for(i_=0;i_<=npoints-1;i_++)
         t[i_]=x[i_][j];
      //--- function call
      CBaseStat::SampleMoments(t,npoints,mean,variance,skewness,kurtosis);
      m[j]=mean;
     }
//--- Center,apply SVD,prepare output
   a.Resize(MathMax(npoints,nvars),nvars);
//--- calculation
   for(i=0;i<=npoints-1;i++)
     {
      for(i_=0;i_<=nvars-1;i_++)
         a[i].Set(i_,x[i][i_]);
      for(i_=0;i_<=nvars-1;i_++)
         a[i].Set(i_,a[i][i_]-m[i_]);
     }
   for(i=npoints;i<=nvars-1;i++)
     {
      for(j=0;j<=nvars-1;j++)
         a[i].Set(j,0);
     }
//--- check
   if(!CSingValueDecompose::RMatrixSVD(a,MathMax(npoints,nvars),nvars,0,1,2,s2,u,vt))
     {
      info=-4;
      return;
     }
//--- check
   if(npoints!=1)
     {
      for(i=0;i<=nvars-1;i++)
         s2[i]=CMath::Sqr(s2[i])/(npoints-1);
     }
//--- allocation
   v.Resize(nvars,nvars);
//--- function call
   CBlas::CopyAndTranspose(vt,0,nvars-1,0,nvars-1,v,0,nvars-1,0,nvars-1);
  }
//+------------------------------------------------------------------+
