add feature to correct historical tick data

some brokers (markets.com) deliver tick incorrect historical tick data. this commit attempts to automatically correct the tick price data

Indicators/JsonAPIIndicator.mq5

@@ -21,24 +21,20 @@ Context context("MQL5 JSON API");
 Socket chartSubscriptionSocket(context,ZMQ_SUB);
 
 //--- input parameters
-#property indicator_buffers 21
-#property indicator_plots   20
-#property indicator_label1  "JsonAPI"
-#property indicator_type1   DRAW_NONE
-#property indicator_type2   DRAW_NONE
-#property indicator_type3   DRAW_NONE
-//#property indicator_color3  CLR_NONE
-#property indicator_type4   DRAW_NONE
-#property indicator_type5  DRAW_NONE
+#property indicator_buffers 31
+#property indicator_plots   30
 
 input string            IndicatorId="";
-input string            ShortName="JsonAPI";
+input string            ShortName="JsonAPIIndicator";
 
 //--- indicator settings
-double                  B0[], B1[], B2[], B3[], B4[], B5[], B6[], B7[], B8[], B9[], B10[], B11[], B12[], B13[], B14[], B15[], B16[], B17[], B18[], B19[], alive[];
-bool                    debug = true;
-bool                    first = false;
+double                  B0[], B1[], B2[], B3[], B4[], B5[], B6[], B7[], B8[], B9[], B10[];
+double                  B11[], B12[], B13[], B14[], B15[], B16[], B17[], B18[], B19[], B20[];
+double                  B21[], B22[], B23[], B24[], B25[], B26[], B27[], B28[], B29[];
+bool                    debug = false;
 int                     activeBufferCount = 0;
+long                    mtChartId = 0;
+bool                    setFormingCandleBlank = true;
 
 //+------------------------------------------------------------------+
 //| Custom indicator initialization function                         |
@@ -46,12 +42,7 @@ int                     activeBufferCount = 0;
 int OnInit()
 
   {
-// TODO subscribe only to own IndicatorId topic
-// Subscribe to all topics
-   chartSubscriptionSocket.setSubscribe("");
-   chartSubscriptionSocket.setLinger(1000);
-// Number of messages to buffer in RAM.
-   chartSubscriptionSocket.setReceiveHighWaterMark(1000); // TODO confirm settings
+
    bool result = chartSubscriptionSocket.connect(StringFormat("tcp://%s:%d", HOST, CHART_SUB_PORT));
    if(result == false)
      {
@@ -59,10 +50,17 @@ int OnInit()
      }
    else
      {
-      Print("Accepting Chart Indicator data on port ", CHART_SUB_PORT);
+      if(debug)
+         Print("Accepting Chart Indicator data on port ", CHART_SUB_PORT);
+      // TODO subscribe only to own IndicatorId topic
+      // Subscribe to all topics
+      chartSubscriptionSocket.setSubscribe("");
+      //chartSubscriptionSocket.setLinger(1000);
+      chartSubscriptionSocket.setLinger(10000);
+      // Number of messages to buffer in RAM.
+      chartSubscriptionSocket.setReceiveHighWaterMark(5); // TODO confirm settings
      }
 
-
 //--- indicator buffers mapping;
    ArraySetAsSeries(B0,true);
    ArraySetAsSeries(B1,true);
@@ -84,30 +82,47 @@ int OnInit()
    ArraySetAsSeries(B17,true);
    ArraySetAsSeries(B18,true);
    ArraySetAsSeries(B19,true);
-   ArraySetAsSeries(alive,true);
-
-   SetIndexBuffer(0,B0,INDICATOR_DATA);
-   SetIndexBuffer(1,B1,INDICATOR_DATA);
-   SetIndexBuffer(2,B2,INDICATOR_DATA);
-   SetIndexBuffer(3,B3,INDICATOR_DATA);
-   SetIndexBuffer(4,B4,INDICATOR_DATA);
-   SetIndexBuffer(5,B5,INDICATOR_DATA);
-   SetIndexBuffer(6,B6,INDICATOR_DATA);
-   SetIndexBuffer(7,B7,INDICATOR_DATA);
-   SetIndexBuffer(8,B8,INDICATOR_DATA);
-   SetIndexBuffer(9,B9,INDICATOR_DATA);
-   SetIndexBuffer(10,B10,INDICATOR_DATA);
-   SetIndexBuffer(11,B11,INDICATOR_DATA);
-   SetIndexBuffer(12,B12,INDICATOR_DATA);
-   SetIndexBuffer(13,B13,INDICATOR_DATA);
-   SetIndexBuffer(14,B14,INDICATOR_DATA);
-   SetIndexBuffer(15,B15,INDICATOR_DATA);
-   SetIndexBuffer(16,B16,INDICATOR_DATA);
-   SetIndexBuffer(17,B17,INDICATOR_DATA);
-   SetIndexBuffer(18,B18,INDICATOR_DATA);
-   SetIndexBuffer(19,B19,INDICATOR_DATA);
-   SetIndexBuffer(20,alive,INDICATOR_CALCULATIONS); // If the buffer index changes, the line starting with "CopyBuffer(chartWindowIndicators[i].indicatorHandle," in JsonAPI.mq5 has to be updated
+   ArraySetAsSeries(B20,true);
+   ArraySetAsSeries(B21,true);
+   ArraySetAsSeries(B22,true);
+   ArraySetAsSeries(B23,true);
+   ArraySetAsSeries(B24,true);
+   ArraySetAsSeries(B25,true);
+   ArraySetAsSeries(B26,true);
+   ArraySetAsSeries(B27,true);
+   ArraySetAsSeries(B28,true);
+   ArraySetAsSeries(B29,true);
 
+   SetIndexBuffer(0,B0,INDICATOR_CALCULATIONS);
+   SetIndexBuffer(1,B1,INDICATOR_CALCULATIONS);
+   SetIndexBuffer(2,B2,INDICATOR_CALCULATIONS);
+   SetIndexBuffer(3,B3,INDICATOR_CALCULATIONS);
+   SetIndexBuffer(4,B4,INDICATOR_CALCULATIONS);
+   SetIndexBuffer(5,B5,INDICATOR_CALCULATIONS);
+   SetIndexBuffer(6,B6,INDICATOR_CALCULATIONS);
+   SetIndexBuffer(7,B7,INDICATOR_CALCULATIONS);
+   SetIndexBuffer(8,B8,INDICATOR_CALCULATIONS);
+   SetIndexBuffer(9,B9,INDICATOR_CALCULATIONS);
+   SetIndexBuffer(10,B10,INDICATOR_CALCULATIONS);
+   SetIndexBuffer(11,B11,INDICATOR_CALCULATIONS);
+   SetIndexBuffer(12,B12,INDICATOR_CALCULATIONS);
+   SetIndexBuffer(13,B13,INDICATOR_CALCULATIONS);
+   SetIndexBuffer(14,B14,INDICATOR_CALCULATIONS);
+   SetIndexBuffer(15,B15,INDICATOR_CALCULATIONS);
+   SetIndexBuffer(16,B16,INDICATOR_CALCULATIONS);
+   SetIndexBuffer(17,B17,INDICATOR_CALCULATIONS);
+   SetIndexBuffer(18,B18,INDICATOR_CALCULATIONS);
+   SetIndexBuffer(19,B19,INDICATOR_CALCULATIONS);
+   SetIndexBuffer(20,B20,INDICATOR_CALCULATIONS);
+   SetIndexBuffer(21,B21,INDICATOR_CALCULATIONS);
+   SetIndexBuffer(22,B22,INDICATOR_CALCULATIONS);
+   SetIndexBuffer(23,B23,INDICATOR_CALCULATIONS);
+   SetIndexBuffer(24,B24,INDICATOR_CALCULATIONS);
+   SetIndexBuffer(25,B25,INDICATOR_CALCULATIONS);
+   SetIndexBuffer(26,B26,INDICATOR_CALCULATIONS);
+   SetIndexBuffer(27,B27,INDICATOR_CALCULATIONS);
+   SetIndexBuffer(28,B28,INDICATOR_CALCULATIONS);
+   SetIndexBuffer(29,B29,INDICATOR_CALCULATIONS);
 
 //---
    IndicatorSetString(INDICATOR_SHORTNAME,ShortName);
@@ -115,6 +130,14 @@ int OnInit()
    return(INIT_SUCCEEDED);
   }
 
+//+------------------------------------------------------------------+
+//|                                                                  |
+//+------------------------------------------------------------------+
+void OnDeinit(const int reason)
+  {
+// Print("INDI DEINIT ",reason);
+  }
+
 
 //+------------------------------------------------------------------+
 //|                                                                  |
@@ -142,9 +165,11 @@ int OnCalculate(const int rates_total,
                 const long &volume[],
                 const int &spread[])
   {
+
 // While a new candle is forming, set the current value to be empty
 
-   if(rates_total>prev_calculated)
+
+   if(rates_total>prev_calculated && setFormingCandleBlank)
      {
       B0[0] = EMPTY_VALUE;
       B1[0] = EMPTY_VALUE;
@@ -166,10 +191,17 @@ int OnCalculate(const int rates_total,
       B17[0] = EMPTY_VALUE;
       B18[0] = EMPTY_VALUE;
       B19[0] = EMPTY_VALUE;
+      B20[0] = EMPTY_VALUE;
+      B21[0] = EMPTY_VALUE;
+      B22[0] = EMPTY_VALUE;
+      B23[0] = EMPTY_VALUE;
+      B24[0] = EMPTY_VALUE;
+      B25[0] = EMPTY_VALUE;
+      B26[0] = EMPTY_VALUE;
+      B27[0] = EMPTY_VALUE;
+      B28[0] = EMPTY_VALUE;
+      B29[0] = EMPTY_VALUE;
      }
-   if(first==false)
-      alive[0] = 1;
-// ChartRedraw(0);
 
 //--- return value of prev_calculated for next call
    return(rates_total);
@@ -191,52 +223,163 @@ void SubscriptionHandler(ZmqMsg &chartMsg)
       Alert("Deserialization Error");
       ExpertRemove();
      }
-   if(message["indicatorChartId"]==IndicatorId)
+   if(message["chartIndicatorId"]==IndicatorId)
      {
 
       if(message["action"]=="PLOT" && message["actionType"]=="DATA")
         {
          int bufferIdx = message["indicatorBufferId"].ToInt();
          if(bufferIdx == 0)
+           {
             WriteToBuffer(message, B0);
+            SetIndexBuffer(0,B0,INDICATOR_DATA);
+           }
          if(bufferIdx == 1)
+           {
             WriteToBuffer(message, B1);
+            SetIndexBuffer(1,B1,INDICATOR_DATA);
+           }
          if(bufferIdx == 2)
+           {
             WriteToBuffer(message, B2);
+            SetIndexBuffer(2,B2,INDICATOR_DATA);
+           }
          if(bufferIdx == 3)
+           {
             WriteToBuffer(message, B3);
+            SetIndexBuffer(3,B3,INDICATOR_DATA);
+           }
          if(bufferIdx == 4)
+           {
             WriteToBuffer(message, B4);
+            SetIndexBuffer(4,B4,INDICATOR_DATA);
+           }
          if(bufferIdx == 5)
+           {
             WriteToBuffer(message, B5);
+            SetIndexBuffer(5,B5,INDICATOR_DATA);
+           }
          if(bufferIdx == 6)
+           {
             WriteToBuffer(message, B6);
+            SetIndexBuffer(6,B6,INDICATOR_DATA);
+           }
          if(bufferIdx == 7)
+           {
             WriteToBuffer(message, B7);
+            SetIndexBuffer(7,B7,INDICATOR_DATA);
+           }
          if(bufferIdx == 8)
+           {
             WriteToBuffer(message, B8);
+            SetIndexBuffer(8,B8,INDICATOR_DATA);
+           }
          if(bufferIdx == 9)
+           {
             WriteToBuffer(message, B9);
+            SetIndexBuffer(9,B9,INDICATOR_DATA);
+           }
          if(bufferIdx == 10)
+           {
             WriteToBuffer(message, B10);
+            SetIndexBuffer(10,B10,INDICATOR_DATA);
+           }
          if(bufferIdx == 11)
+           {
             WriteToBuffer(message, B11);
+            SetIndexBuffer(11,B11,INDICATOR_DATA);
+           }
          if(bufferIdx == 12)
+           {
             WriteToBuffer(message, B12);
+            SetIndexBuffer(12,B12,INDICATOR_DATA);
+           }
          if(bufferIdx == 13)
+           {
             WriteToBuffer(message, B13);
+            SetIndexBuffer(13,B13,INDICATOR_DATA);
+           }
          if(bufferIdx == 14)
+           {
             WriteToBuffer(message, B14);
+            SetIndexBuffer(14,B14,INDICATOR_DATA);
+           }
          if(bufferIdx == 15)
+           {
             WriteToBuffer(message, B15);
+            SetIndexBuffer(15,B15,INDICATOR_DATA);
+           }
          if(bufferIdx == 16)
+           {
             WriteToBuffer(message, B16);
+            SetIndexBuffer(16,B16,INDICATOR_DATA);
+           }
          if(bufferIdx == 17)
+           {
             WriteToBuffer(message, B17);
+            SetIndexBuffer(17,B17,INDICATOR_DATA);
+           }
          if(bufferIdx == 18)
+           {
             WriteToBuffer(message, B18);
+            SetIndexBuffer(18,B18,INDICATOR_DATA);
+           }
          if(bufferIdx == 19)
+           {
             WriteToBuffer(message, B19);
+            SetIndexBuffer(19,B19,INDICATOR_DATA);
+           }
+         if(bufferIdx == 20)
+           {
+            WriteToBuffer(message, B20);
+            SetIndexBuffer(20,B20,INDICATOR_DATA);
+           }
+         if(bufferIdx == 21)
+           {
+            WriteToBuffer(message, B21);
+            SetIndexBuffer(21,B21,INDICATOR_DATA);
+           }
+         if(bufferIdx == 22)
+           {
+            WriteToBuffer(message, B22);
+            SetIndexBuffer(22,B22,INDICATOR_DATA);
+           }
+         if(bufferIdx == 23)
+           {
+            WriteToBuffer(message, B23);
+            SetIndexBuffer(23,B23,INDICATOR_DATA);
+           }
+         if(bufferIdx == 24)
+           {
+            WriteToBuffer(message, B24);
+            SetIndexBuffer(24,B24,INDICATOR_DATA);
+           }
+         if(bufferIdx == 25)
+           {
+            WriteToBuffer(message, B25);
+            SetIndexBuffer(25,B25,INDICATOR_DATA);
+           }
+         if(bufferIdx == 26)
+           {
+            WriteToBuffer(message, B26);
+            SetIndexBuffer(26,B26,INDICATOR_DATA);
+           }
+         if(bufferIdx == 27)
+           {
+            WriteToBuffer(message, B27);
+            SetIndexBuffer(27,B27,INDICATOR_DATA);
+           }
+         if(bufferIdx == 28)
+           {
+            WriteToBuffer(message, B28);
+            SetIndexBuffer(28,B28,INDICATOR_DATA);
+           }
+         if(bufferIdx == 29)
+           {
+            WriteToBuffer(message, B29);
+            SetIndexBuffer(29,B29,INDICATOR_DATA);
+           }
+         ChartRedraw(mtChartId);
         }
       else
          if(message["action"]=="PLOT" && message["actionType"]=="ADDBUFFER")
@@ -246,30 +389,165 @@ void SubscriptionHandler(ZmqMsg &chartMsg)
             string linetypeStr = message["style"]["linetype"].ToStr();
             string linestyleStr = message["style"]["linestyle"].ToStr();
             int linewidth = message["style"]["linewidth"].ToInt();
+            setFormingCandleBlank = message["style"]["blankforming"].ToBool();
 
             color colorstyle = StringToColor(colorstyleStr);
             int linetype = StringToEnumInt(linetypeStr);
             int linestyle = StringToEnumInt(linestyleStr);
 
-            /*
-            //if (aa == false) {
-            Print("SETBUFF ActCount  ",activeBufferCount);
-            if (activeBufferCount == 0) {SetIndexBuffer(0,B1,INDICATOR_DATA);} // Two semicolons ar required! No idea why. Seems to be a timing problem, better to keep it in init()
-
-            if (activeBufferCount == 1) {SetIndexBuffer(1,B2,INDICATOR_DATA);;}
-            if (activeBufferCount == 2) {SetIndexBuffer(2,B3,INDICATOR_DATA);;}
-            if (activeBufferCount == 3) {SetIndexBuffer(3,B4,INDICATOR_DATA);;}
-            if (activeBufferCount == 4) {SetIndexBuffer(4,B5,INDICATOR_DATA);;}
-
-            //aa = true;}
-            */
-
             SetStyle(activeBufferCount, linelabel, colorstyle, linetype, linestyle, linewidth);
             activeBufferCount = activeBufferCount + 1;
+
+            ClearBuffer(activeBufferCount-1);
            }
      }
   }
 
+//+------------------------------------------------------------------+
+//|                                                                  |
+//+------------------------------------------------------------------+
+void Clear(double &buffer[])
+  {
+   int bufferSize = ArraySize(buffer);
+   for(int i=0; i<bufferSize; i++)
+     {
+      buffer[i] = EMPTY_VALUE;
+     }
+  }
+
+//+------------------------------------------------------------------+
+//|                                                                  |
+//+------------------------------------------------------------------+
+void ClearBuffer(int bufferIdx)
+  {
+   switch(bufferIdx)
+     {
+      case  0:
+        {
+         Clear(B0);
+        }
+      case  1:
+        {
+         Clear(B1);
+        }
+      case  2:
+        {
+         Clear(B2);
+        }
+      case  3:
+        {
+         Clear(B3);
+        }
+      case  4:
+        {
+         Clear(B4);
+        }
+      case  5:
+        {
+         Clear(B5);
+        }
+      case  6:
+        {
+         Clear(B6);
+        }
+      case  7:
+        {
+         Clear(B7);
+        }
+      case  8:
+        {
+         Clear(B8);
+        }
+      case  9:
+        {
+         Clear(B9);
+        }
+      case  10:
+        {
+         Clear(B10);
+        }
+      case  11:
+        {
+         Clear(B11);
+        }
+      case  12:
+        {
+         Clear(B12);
+        }
+      case  13:
+        {
+         Clear(B13);
+        }
+      case  14:
+        {
+         Clear(B14);
+        }
+      case  15:
+        {
+         Clear(B15);
+        }
+      case  16:
+        {
+         Clear(B16);
+        }
+      case  17:
+        {
+         Clear(B17);
+        }
+      case  18:
+        {
+         Clear(B18);
+        }
+      case  19:
+        {
+         Clear(B19);
+        }
+      case  20:
+        {
+         Clear(B20);
+        }
+      case  21:
+        {
+         Clear(B21);
+        }
+      case  22:
+        {
+         Clear(B22);
+        }
+      case  23:
+        {
+         Clear(B23);
+        }
+      case  24:
+        {
+         Clear(B24);
+        }
+      case  25:
+        {
+         Clear(B25);
+        }
+      case  26:
+        {
+         Clear(B26);
+        }
+      case  27:
+        {
+         Clear(B27);
+        }
+      case  28:
+        {
+         Clear(B28);
+        }
+      case  29:
+        {
+         Clear(B29);
+        }
+      break;
+      default:
+        {} break;
+     }
+  }
+
 //+------------------------------------------------------------------+
 //| Update indicator buffer function                                 |
 //+------------------------------------------------------------------+
@@ -277,32 +555,32 @@ void WriteToBuffer(CJAVal &message, double &buffer[])
   {
 
    int bufferSize = ArraySize(buffer);
-
    int messageDataSize = message["data"].Size();
-// TODO check if this is working as expected. Seems to
-   if(first==false)
-     {
-      for(int i=0; i<activeBufferCount; i++)
-        {
-         //Print("BUFF ",bufferSize-messageDataSize, " ",ArraySize(B2)," ", ArraySize(B3), " ",messageDataSize);
-         PlotIndexSetInteger(i,PLOT_DRAW_BEGIN,bufferSize-messageDataSize);
-        }
-      first = true;
-     }
 
+// calculate the buffer offset
+   MqlRates r[];
+   mtChartId =(datetime)message["mtChartId"].ToInt();
+   datetime fromDate=(datetime)message["fromDate"].ToInt();
+   datetime toDate=TimeCurrent();
+   ENUM_TIMEFRAMES period = ChartPeriod(mtChartId);
+   string symbol = ChartSymbol(mtChartId);
+   int rateCount;
+
+   rateCount = CopyRates(symbol, period, fromDate, toDate, r);
+   int offset = rateCount - 1;
+
+// write to buffer
    for(int i=0; i<messageDataSize; i++)
      {
-      // don't add more elements than the automatically sized buffer array can hold
-      if(i+1<bufferSize)
+      // don't add more elements than the automatically sized buffer array can
+      if(i+offset<bufferSize)
         {
-         // the first element is the current unformed candle, so we start at index 1
-         // we reverse the order of the incoming values, which are expected to be ascending
-         //buffer[i+1] = message["data"][messageDataSize-1-i].ToDbl();
-         buffer[i+1] = message["data"][messageDataSize-1-i].ToDbl();
+         double val = message["data"][i].ToDbl();
+         if(val >= EMPTY_VALUE)
+            val = EMPTY_VALUE;
+         buffer[i+offset] = val;
         }
      }
-// Set the most recent plotted value to nothing, as we do not have any data for yet unformed candles
-   buffer[0] = EMPTY_VALUE;
   }