EVZReader

ÿþ//+------------------------------------------------------------------+

//|                                                    EVZReader.mq5 |

//|                    Copyright 2021, Manuel Alejandro Cercós Pérez |

//|                         https://www.mql5.com/en/users/alexcercos |

//+------------------------------------------------------------------+

#property copyright "Copyright 2021, Manuel Alejandro Cercós Pérez"

#property link      "https://www.mql5.com/en/users/alexcercos"

#property version   "1.00"

#property service

#property strict





#define TIME_LIMIT 1217635200

#define MAX_TIME   2147483647

#define CUSTOM_FILE_NAME "EVZ_Data"

#define ROW_BYTE_SIZE 40



enum UpdateTime

   {

    NO_UPDATE, //Update once and end program

    MINUTE,  //Update each minute

    MINUTES_5,  //Update each 5 minutes

    MINUTES_10  //Update each 10 minutes

   };



input UpdateTime Update_Values = NO_UPDATE; //Update EVZ values





//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

void OnStart()

   {

    int timeUpdate = OnInitFunction();



    if(timeUpdate>0)

       {

        datetime lastTickUpdate = 0;

        while(!IsStopped())

           {

            if(lastTickUpdate + timeUpdate*1000 < GetTickCount())

               {

                TimerFunction();

                lastTickUpdate = GetTickCount();

               }

           }

       }





    DeInitFunction();

   }



//---

//---





//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

int OnInitFunction()

   {

    scraper = new YahooScraper("^EVZ");



    scraper.SaveFile(CUSTOM_FILE_NAME+".txt", TIME_LIMIT, MAX_TIME);



    switch(Update_Values)

       {

        case NO_UPDATE:

            return 0;

        case MINUTE:

            return 60;

        case MINUTES_5:

            return 300;

        case MINUTES_10:

            return 600;

       }

    return 0;

   }



//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

void DeInitFunction()

   {

    delete scraper;

   }



//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

void TimerFunction()

   {

    scraper.UpdateFile(CUSTOM_FILE_NAME+".txt", MAX_TIME);

   }



//+------------------------------------------------------------------+

//|        CLASS TO RETRIEVE DATA FROM YAHOO FINANCE                 |

//+------------------------------------------------------------------+

class YahooScraper

   {

    string           symbol_read;



public:

                     YahooScraper(string symbolToRead);



    void             GetWebPage(int initTime, int endTime, string &resultStr);



    void             SaveFile(string filename, int initTime, int endTime);



    void             UpdateFile(string filename, int endTime);



    string           ProcessRow(string row);

    string           nZeros(int num);

   };

YahooScraper* scraper;



//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

YahooScraper::YahooScraper(string symbolToRead)

   {

    symbol_read = symbolToRead;

   }





//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

void YahooScraper::GetWebPage(int initTime,int endTime,string &resultStr)

   {

    resultStr = "";



    string cookie=NULL,headers;

    char   post[],result[];

    string domain = "https://query1.finance.yahoo.com";

    string url= domain + "/v7/finance/download/" + symbol_read + "?period1=" + IntegerToString(initTime) + "&period2=" + IntegerToString(endTime) + "&interval=1d&events=history";



//Print(url);



    ResetLastError();



    int res=WebRequest("GET",url,cookie,NULL,500,post,0,result,headers);

    if(res==-1)

       {

        Print("WebRequest Error. Error code  =",GetLastError());



        MessageBox("It is necessary to add the address '"+domain+"' to the list of allowed URLs in the 'Advisors' tab","Error",MB_ICONINFORMATION);



       }

    else

        if(res==200)

           {

            resultStr = CharArrayToString(result,0,WHOLE_ARRAY,CP_UTF8);

           }

        else

           {

            PrintFormat("Load page error '%s', code %d",url,res);



           }

   }



//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

string YahooScraper::ProcessRow(string row)

   {

    string items[];

    StringSplit(row, ',', items);





    string date = items[0];



    StringReplace(date, "-", ".");



    string openS, highS, lowS, closeS;



    if(items[1]=="null")

       {

        openS = DoubleToString(0.0, 2);

        openS = nZeros(6-StringLen(openS)) + openS;



        highS = DoubleToString(0.0, 2);

        highS = nZeros(6-StringLen(highS)) + highS;



        lowS = DoubleToString(0.0, 2);

        lowS = nZeros(6-StringLen(lowS)) + lowS;



        closeS = DoubleToString(0.0, 2);

        closeS = nZeros(6-StringLen(closeS)) + closeS;

       }

    else

       {

        openS = DoubleToString(StringToDouble(items[1]), 2);

        openS = nZeros(6-StringLen(openS)) + openS;



        highS = DoubleToString(StringToDouble(items[2]), 2);

        highS = nZeros(6-StringLen(highS)) + highS;



        lowS = DoubleToString(StringToDouble(items[3]), 2);

        lowS = nZeros(6-StringLen(lowS)) + lowS;



        closeS = DoubleToString(StringToDouble(items[4]), 2);

        closeS = nZeros(6-StringLen(closeS)) + closeS;



       }



    return date + "," + openS + "," + lowS + "," + highS + ","+closeS;

   }



//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

void YahooScraper::SaveFile(string filename, int initTime, int endTime)

   {

    string tmpStr;

    GetWebPage(initTime, endTime, tmpStr);



    if(tmpStr == "")

        return;



    string queryRows[];

    StringSplit(tmpStr, '\n', queryRows);



    int requestSize = ArraySize(queryRows);



    int filehandle=FileOpen(filename,FILE_COMMON|FILE_SHARE_WRITE|FILE_SHARE_READ|FILE_WRITE|FILE_TXT|FILE_ANSI);





    if(filehandle!=INVALID_HANDLE)

       {

        FileWrite(filehandle, IntegerToString(requestSize-1));

        for(int i = 1; i<requestSize; i++)

           {

            string toWrite = ProcessRow(queryRows[i]);



            FileWrite(filehandle, toWrite);

           }

       }

    FileFlush(filehandle);

    FileClose(filehandle);



   }



//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

string YahooScraper::nZeros(int num)

   {

    string zeros = "";



    for(int z=0; z<num; z++)

       {

        zeros = zeros + "0";

       }

    return zeros;

   }



//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

void YahooScraper::UpdateFile(string filename,int endTime)

   {

//Abrir archivo anterior, initTime es el datetime de la ultima fila

    if(!FileIsExist(filename, FILE_COMMON))

       {

        SaveFile(filename, TIME_LIMIT, MAX_TIME);

        return;

       }



    int filehandle=FileOpen(filename,FILE_COMMON|FILE_SHARE_WRITE|FILE_SHARE_READ|FILE_READ|FILE_WRITE|FILE_TXT|FILE_ANSI);





    if(filehandle!=INVALID_HANDLE)

       {

        FileSeek(filehandle, -ROW_BYTE_SIZE, SEEK_END);





        int str_size=FileReadInteger(filehandle,INT_VALUE);

        string str = FileReadString(filehandle,str_size);



        string items[];

        StringSplit(str, ',', items);



        int initTime = (int)StringToTime(items[0]);



        string tmpStr;



        //Pagina web

        GetWebPage(initTime, endTime, tmpStr);



        string queryRows[];

        StringSplit(tmpStr, '\n', queryRows);



        int requestSize = ArraySize(queryRows);



        if(requestSize>=2 && queryRows[requestSize-1]==queryRows[requestSize-2])  //Prevent repeated values from weekends

           {

            requestSize--;

           }



        //Reescribir inicio

        FileSeek(filehandle, 0, SEEK_SET);



        str_size=FileReadInteger(filehandle,INT_VALUE);

        str = FileReadString(filehandle,str_size);



        int lastRows = (int)StringToInteger(str);



        FileSeek(filehandle, 0, SEEK_SET);





        FileWriteString(filehandle, IntegerToString(lastRows + requestSize - 2)+" ");



        //Reescribir filas

        FileSeek(filehandle, -ROW_BYTE_SIZE, SEEK_END);

        for(int i = 1; i<requestSize; i++)

           {

            string toWrite = ProcessRow(queryRows[i]);



            FileWrite(filehandle, toWrite);

           }

       }

    FileFlush(filehandle);

    FileClose(filehandle);

   }

//+------------------------------------------------------------------+