Author: Copyright � 2008, MetaQuotes Software Corp.
Matrix2

This script is designed to perform automated data standardization, a common preprocessing step used in machine learning and statistical analysis. Its primary purpose is to take a collection of raw numbers (organized into a grid or "matrix") and convert them into a uniform format so that different types of data can be compared fairly on the same scale.

Core Logic Overview

The script operates through three main phases:

1. Data Generation and Organization The program begins by creating a small grid of data points. In this specific version, it populates a 5-line by 3-column matrix with random numbers. This represents the "raw input" that needs to be processed.

2. Statistical Analysis To normalize data effectively, the script analyzes the characteristics of the columns within the grid. It contains specialized logic to calculate several key statistics for each column:

  • Mean: The average value of all numbers in a column.
  • Standard Deviation: A measure of how much the numbers spread out or vary from the average.
  • Range: The distance between the highest and lowest numbers in a column.
  • Midrange: The exact middle point between the maximum and minimum values.

3. Normalization (Scaling) The script provides four distinct methods to "level the playing field" for these numbers. Depending on which mode is selected, it applies different mathematical formulas to adjust the raw data:

  • Standard Deviation Scaling: It subtracts the average from each number and then divides by the standard deviation. This is used to ensure that all data points have a similar variance.
  • Range/Midrange Scaling: It centers the data around its midpoint and scales it based on its total spread (range).

Practical Use Cases

The script distinguishes between "Training" data (the data used to build a model) and "Test" data (new data being checked against that model). This is a sophisticated distinction because:

  • It can normalize both sets of data independently.
  • Or, it can use the rules learned from the training data to format the test data, ensuring that the new information is processed in the exact same way the original data was handled.

In summary, this script is a utility for converting messy, raw numerical data into organized, standardized "features" that are ready for advanced mathematical modeling or analysis.

4 Views
0 Downloads
0 Favorites
Matrix2
//+------------------------------------------------------------------+
//|                                                      Matrix1.mq4 |
//|                      Copyright © 2008, MetaQuotes Software Corp. |
//|                                        http://www.metaquotes.net |
//+------------------------------------------------------------------+
#property copyright "Copyright © 2008, MetaQuotes Software Corp."
#property link      "http://www.metaquotes.net"
#define DBL_MAX 99999999999999999999
#define LIN 5
#define COL 3

double M[LIN][COL];   

string m1;

int init()
  {
    m1="";
    MathSrand(TimeLocal());
    return(0);
  }                
//+------------------------------------------------------------------+
//| script program start function                                    |
//+------------------------------------------------------------------+
int start()
  {
//----
  for(int i=0;i<LIN;i++)
    for(int j=0;j<COL;j++)
      M[i][j]=MathRand();   

  PrintMatrixToStr(M);  
  svm_normalize_sample(M,1);
  PrintMatrixToStr(M);  

   Comment(m1);	  
//----
   return(0);
  }
//+------------------------------------------------------------------+

void PrintMatrixToStr(double Matrix[][])
{
	int Line = ArrayRange(Matrix,0);
	int Col = ArrayRange(Matrix,1);

	for (int i=0;i<Line;i++)
	{
	  for (int j=0;j<Col;j++)
	   m1=m1+Matrix[i][j]+",";
	  
	  m1=m1+"\n"; 
	}   
	m1=m1+"\n";
}	

//+------------------------------------------------------------------+
//|                                                                  
//+------------------------------------------------------------------+
double MAX(double x, double y)
{
  if(x>y)
    return(x);
  else
    return(y);  
}

//+------------------------------------------------------------------+
//|                                                                  
//+------------------------------------------------------------------+
double MIN(double x, double y)
{
  if(x<y)
    return(x);
  else
    return(y);  
}

//+------------------------------------------------------------------+
//| Normalize matrix 'fmx' using precomputed 'xmean', 'std'. The                                                                  
//| normalization is: for each column, subtract 'xmean' and, if the
//| column 'std' is non-zero, divide by 'std'.
//| d=nr_of_attrib
//| nv=nr_of_sample_lines 
//| double **x --> double **svm_samples_attrib_scaled_dyn_
//| double *xmean = fmx_mean(x2, nv, d);
//| double *std = fmx_std(x2, nv, d);
//| fmx_prenorm(x, nv, d, xmean, std);
//|       X  - mean
//|        i
//|   S = ---------
//|    i     std
//+------------------------------------------------------------------+
void fmx_prenorm(
                 double &fmx[][], 
                 double xmean[], //  double *xmean = fmx_mean(x2, nv, d);
                 double std[]   //  double *std = fmx_std(x2, nv, d);
                 )
{
  int    i;
  int    j;
  double  f;
  int rows = ArrayRange(fmx,0);
  int columns = ArrayRange(fmx,1);

    for (i = 0; i < columns; i++)
      for (j = 0; j < rows; j++)
	  {
	    f = fmx[j][i];
	      if (std[i] != 0)
		     fmx[j][i] = (f-xmean[i])/std[i];
	      else
		     fmx[j][i] = f-xmean[i];
	 }
	 

	 
}

//+------------------------------------------------------------------+
//|              max X  +  min X
//|               i   i     i   i
//|   midrange = ----------------
//|                     2
//|   range = max X  -  min X
//|            i   i     i   i
//|    
//|       X  - midrange
//|        i
//|   S = -------------
//|    i     range / 2                                                                     
//+------------------------------------------------------------------+
void fmx_prenorm_range(
                       double &fmx[][], 
                       double xrange[], 
                       double xmidrange[] 
                       )
{
  int    i;
  int    j;
  double  f;
  int rows = ArrayRange(fmx,0);
  int columns = ArrayRange(fmx,1);

  for (i = 0; i < columns; i++)
    for (j = 0; j < rows; j++)
    {
	    f = fmx[j][i];
	      if (xrange[i] != 0)
		    fmx[j][i] = (f-xmidrange[i])/(xrange[i]/2);
	      else
		    fmx[j][i] = f-xmidrange[i];

	  }
}

//+------------------------------------------------------------------+
//|                                                                  
//+------------------------------------------------------------------+
void fmx_midrange(double  &xmidrange[],double fmx[][])
{
  int    i;
  int    j;
  double smax = -DBL_MAX;
  double smin = DBL_MAX;
  int rows = ArrayRange(fmx,0);
  int columns = ArrayRange(fmx,1);
  ArrayResize(xmidrange,columns);
  for (i = 0; i < columns; i++)
  {
     smax = -DBL_MAX;
     smin = DBL_MAX;
     for (j = 0; j < rows; j++)
	  {
	      smax=MAX(fmx[j][i],smax);
         smin=MIN(fmx[j][i],smin);
     }  
	  xmidrange[i] = (smax+smin)/2;
  }
}
//+------------------------------------------------------------------+
//|                                                                  
//+------------------------------------------------------------------+
void fmx_range(double  &xrange[], double fmx[][])
{
  int    i;
  int    j;
  double smax = -DBL_MAX;
  double smin = DBL_MAX;
  int rows = ArrayRange(fmx,0);
  int columns = ArrayRange(fmx,1);
  ArrayResize(xrange,columns);
  for (i = 0; i < columns; i++)
  {
     smax = -DBL_MAX;
     smin = DBL_MAX;
     for (j = 0; j < rows; j++)
	  {
	     smax=MAX(fmx[j][i],smax);
        smin=MIN(fmx[j][i],smin);
     }  
	  xrange[i] = smax-smin;
   }

}


//+------------------------------------------------------------------+
//|                                                                  
//+------------------------------------------------------------------+
void fmx_max(double &xrange[],double fmx[][])
{
  int    i;
  int    j;
  double smax = -DBL_MAX;
  int rows = ArrayRange(fmx,0);
  int columns = ArrayRange(fmx,1);

  ArrayResize(xrange,columns);
  for (i = 0; i < columns; i++)
  {
    smax = -DBL_MAX;
	 for (j = 0; j < rows; j++)
	 {
	   smax=MAX(fmx[j][i],smax);
    }  
    xrange[i] = smax;
  }
}

//+------------------------------------------------------------------+
//|                                                                  
//+------------------------------------------------------------------+
void fmx_min(double &xrange[],double fmx[][])
{
  int    i;
  int    j;
  double smin = DBL_MAX;
  int rows = ArrayRange(fmx,0);
  int columns = ArrayRange(fmx,1);

  ArrayResize(xrange,columns);
  for (i = 0; i < columns; i++)
  {
    smin = DBL_MAX;
    for (j = 0; j < rows; j++)
	 {
      smin=MIN(fmx[j][i],smin);
    }  
    xrange[i] = smin;
  }
}

//+------------------------------------------------------------------+
//| Return mean values of columns in 'fmx'.                                                                  
//+------------------------------------------------------------------+
void fmx_mean(double &xmean[], double fmx[][])
{
  int    i;
  int    j;
  double s;
  int rows = ArrayRange(fmx,0);
  int columns = ArrayRange(fmx,1);

  ArrayResize(xmean,columns);
  if (rows > 1)
  {
    for (i = 0; i < columns; i++)
	 {
	   s = 0.0;
	   for (j = 0; j < rows; j++)
	     s += fmx[j][i];
		  xmean[i] = s/rows;
	 }
  }
  else
  {
    for (i = 0; i < columns; i++)
	   xmean[i] = fmx[0][i];
  }
}

//+------------------------------------------------------------------+
//| Set elements of 'vec' to 'value'.                                                                  
//+------------------------------------------------------------------+
void fvec_set(double &vec[], int len, double value)
{
  int i;
  ArrayResize(vec,len);  
  for (i = 0; i < len; i++)
    vec[i] = value;
}

//+------------------------------------------------------------------+
//|  Return standard deviations of columns in 'fmx'.
//+------------------------------------------------------------------+
void fmx_std(double  &std[], double fmx[][])
{
  int    i;
  int    j;
  double  xmean;
  double s;
  int rows = ArrayRange(fmx,0);
  int columns = ArrayRange(fmx,1);
  
  ArrayResize(std,columns);
  if (rows > 1)
  {  
    for (i = 0; i < columns; i++)
    {
	      s = 0.0;
	      for (j = 0; j < rows; j++)
		s += fmx[j][i];
	      xmean = s/rows;
	      s = 0.0;
	      for (j = 0; j < rows; j++)
		s += (fmx[j][i]-xmean)*(fmx[j][i]-xmean);
	      std[i] = MathSqrt(s/(rows-1));
    }
  }  
  else
   fvec_set(std, columns, 0.0);
}

//+------------------------------------------------------------------+
//| 1-normalize TRAIN lines from std and mean calculated from TRAIN lines
//| 1-normalize TEST lines from std and mean calculated from TEST lines
//| ..................................................................
//| 2-normalize TRAIN lines from std and mean calculated from TRAIN lines  
//| 2-normalize TEST lines from std and mean calculated from TRAIN lines    
//| ..................................................................
//| 3-normalize TRAIN lines from range and midrange calculated from TRAIN lines   
//| 3-normalize TEST lines from range and midrange calculated from TEST lines 
//| ..................................................................
//| 4-normalize TEST lines from range and midrange calculated from TRAIN lines                                                          
//+------------------------------------------------------------------+
void svm_normalize_sample(
								double &svm_samples_attrib[][],							
								int normalize
							 )
{

  double xmean[];   
  double std[];  
  double xrange[];  
  double xmidrange[];  

  
  if(normalize==1 || normalize==2)
  { //normalize TRAIN lines from std and mean calculated from TRAIN lines
    fmx_mean(xmean,svm_samples_attrib);
    fmx_std(std,svm_samples_attrib);
    fmx_prenorm(svm_samples_attrib,xmean, std);
  } 
  else if(normalize==3)
  { //normalize TRAIN lines from range and midrange calculated from TRAIN lines
    fmx_range(xrange,svm_samples_attrib);
    fmx_midrange(xmidrange,svm_samples_attrib);       
    fmx_prenorm_range(svm_samples_attrib,xrange, xmidrange);
  }

}

//+------------------------------------------------------------------+
//| 1-normalize TEST lines from std and mean calculated from TEST lines
//| 2-normalize TEST lines from std and mean calculated from TRAIN lines     
//| 3-normalize TEST lines from range and midrange calculated from TEST lines
//| 4-normalize TEST lines from range and midrange calculated from TRAIN lines                                                             
//+------------------------------------------------------------------+
void svm_normalize_test(
	                     double svm_samples_attrib[][],
								double &svm_test_attrib[][],								
								int normalize
							)		
							
{

  if(normalize==1)
  {  //normalize TEST lines from std and mean calculated from TEST lines
    double xmean2[]; 
    fmx_mean(xmean2,svm_test_attrib);
    double std2[];
    fmx_std(std2,svm_test_attrib);
    fmx_prenorm(svm_test_attrib,xmean2, std2);
  }                 
  else if(normalize==2)
  { //normalize TEST lines from std and mean calculated from TRAIN lines
    double xmean1[];
    fmx_mean(xmean1,svm_samples_attrib);
    double std1[];
    fmx_std(std1,svm_samples_attrib);
    fmx_prenorm(svm_test_attrib, xmean1, std1);
  }
  else if(normalize==3)
  { //normalize TEST lines from range and midrange calculated from TEST lines
    double xrange2[];
    fmx_range(xrange2,svm_test_attrib);
    double xmidrange2[];
    fmx_midrange(xmidrange2,svm_test_attrib);   
    fmx_prenorm_range(svm_test_attrib, xrange2, xmidrange2);
  }
  else if(normalize==4)
  { //normalize TEST lines from range and midrange calculated from TRAIN lines
    double xrange1[];
    fmx_range(xrange1,svm_samples_attrib);
    double xmidrange1[];
    fmx_midrange(xmidrange1,svm_samples_attrib);                                       
    fmx_prenorm_range(svm_test_attrib, xrange1, xmidrange1);
  }
  
}        							


Comments