CME_FUTURES_DOWNLOAD

//+------------------------------------------------------------------+
//|                                       FuturesHistoryDownload.mq4 |
//|                     Copyright © 2012–2013, Financial Arsenal LTD |
//|                                     http://www.arsenal-forex.ru/ |
//+------------------------------------------------------------------+
#property copyright "Copyright © 2012–2013, Financial Arsenal LTD"
#property link      "http://www.arsenal-forex.ru/"
#property show_inputs
//+------------------------------------------------------------------+
//| Defines and Imports and Includes                                 |
//+------------------------------------------------------------------+
#import "wininet.dll"
	int InternetAttemptConnect (int x);
	int InternetOpenA	 			(string sAgent, int lAccessType, string sProxyName, string sProxyBypass, int lFlags );
	int InternetOpenUrlA			(int hInternetSession, string sUrl, string sHeaders, int lHeadersLength, int lFlags, int lContext );
	int InternetReadFile			(int hFile, int& lpvBuffer[], int lNumBytesToRead, int& lNumberOfBytesRead[] );
	int InternetCloseHandle		(int hInet );
#import
//+------------------------------------------------------------------+
//|	Defines                                                        |
//+------------------------------------------------------------------+
#define HTTP_QUERY_CONTENT_LENGTH 0x00000005
#define HTTP_QUERY_FLAG_NUMBER    0x20000000
//----
#define INTERNET_OPEN_TYPE_DIRECT       0
#define INTERNET_OPEN_TYPE_PRECONFIG    0x00000000   // use registry configuration
#define INTERNET_FLAG_RELOAD            0x80000000
#define INTERNET_FLAG_NO_CACHE_WRITE    0x04000000
#define INTERNET_FLAG_PRAGMA_NOCACHE    0x00000100
//----
#define AGENT "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Q312461)"
#define LINK  "http://www.arsenal-forex.ru/"
//+------------------------------------------------------------------+
//| Extern and Global variables                                      |
//+---externs--------------------------------------------------------+
extern string Symbols = "fAUDUSD,fCADUSD,fCHFUSD,fEURUSD,fEURGBP,fEURJPY,fGBPUSD,fNZDUSD,fUSDJPY,fUSDCAD,fUSDCHF";//,f6EM3,f6EU3,f6EZ3";
extern bool   DownloadAllFiles                                = false;
extern bool   DownloadDelayedData                             = false;
extern bool   CheckNews                                       = true ;
//+---globals--------------------------------------------------------+
string url = "http://history.arsenal-fx.com/";
string ifn = "date.min";
string ext = ".vol";
string nfm = "last.new";
//+------------------------------------------------------------------+
//| script program start function                                    |
//+------------------------------------------------------------------+
void start(){int i,ii,sz,bt,hd,tm,br,pos,err,in[1000000];string res="",fn,sys[50];
   //----÷èòàåì èíôî ôàéë / read remote ini file
   if(InetToArray(url+ifn,in)>0){return;}
   sz  = ArraySize(in); 
   if(sz<4){Print("WebServer access error, please try later.");return;}
   //----ïîäêà÷èâàåì íåäåëüíûå òô / find and download weekly bars
   for(i=0;i<10;i++){
      iBars(Symbol(),PERIOD_D1);
      if(iBarShift(Symbol(),PERIOD_W1,in[0],true)<0){
         Comment(iBars(Symbol(),PERIOD_W1)+" Download Weekly period, Please waite.");
         Sleep(1000);
         continue;
      }
      Comment(iBars(Symbol(),PERIOD_W1)+" Bars downloaded");
      break;
   }
   if(iBarShift(Symbol(),PERIOD_W1,in[0],true)<0){Alert("Please check Weekly timeframe and try again");return;}
   //----çàãîíÿåì ñòðîêó ñèìâîëîâ â ìàññèâ / create Symbols array
   for(i=0;i<50;i++){
      pos = StringFind(Symbols,",");
      if(pos<0){
         if(StringLen(Symbols)<3){break;}
         sys[i] = Symbols;
         i++;
         break;
      }
      sys[i] = StringSubstr(Symbols,0,pos);
      Symbols = StringSubstr(Symbols,pos+1);
   }
   ArrayResize(sys,i);
   br = iBarShift(Symbol(),PERIOD_W1,in[0],true);
   sz = ArraySize(sys);
   if(ii<0){Alert("Please check Weekly timeframe and try again");return;}
   //---- çàãðóæàåì ôàéëè èñòîðèè ñ âåáñåðâåðà / download futures history files
   for(i=0;i<=br;i++){
      res = TimeToStr(iTime(Symbol(),PERIOD_W1,i),TIME_DATE);
      res = StringReplace(res,".","_");
      res = StringReplace(res,".","_")+"_";
      for(ii=0;ii<sz;ii++){
         fn = res+sys[ii]+ext;
         if(DownloadDelayedData){fn = res+"d"+sys[ii]+ext;}
         bool havef = true;
         if(i>0){
            if(!DownloadAllFiles){continue;}
            havef = CheckHistFile(fn);
            if(havef){Print("Delete file "+TerminalPath()+"\experts\files\\"+fn+" for recall");continue;}
         }
         Comment("Try download "+url+fn+" file from webserver");
         Sleep(500);
         if(InetToArray(url+fn,in)>0){return;}
         if(ArraySize(in)<4){Print(fn+" not have in WebServer.");continue;}
         Print(fn+" : "+ArraySize(in)+" Bytes downloaded");
         bt+=ArraySize(in);
         if(!havef){Print("Now Create file :"+fn);}
         if(!WriteHistFile(fn,i,in)){return;}
      }
   }
   if(!CheckNews){
      Comment(StringSubstr(LINK,0,StringFind(LINK,"/d")));
   }else{
      CheckNewsFile();
   }
   Print(Int2Str(bt)+" Bytes downloaded"); 
//----
return;}
//+------------------------------------------------------------------+
//|                                  Function : CheckNewsFile()      |
//|                                 Copyright © 2013, XrustSolution. |
//+------------------------------------------------------------------+
void CheckNewsFile(){int tm,ltm;string txt="";
   if(InetToString(url+nfm,txt)>0){
      Comment(StringSubstr(LINK,0,StringFind(LINK,"/d")));
      return;
   }
   if(StringLen(txt)<10){Comment(StringSubstr(LINK,0,StringFind(LINK,"/d")));return;}
   tm = StrToInteger(StringSubstr(txt,0,10));
   txt = StringSubstr(txt,10);
   if(tm == 0){Comment(StringSubstr(LINK,0,StringFind(LINK,"/d")));return;}
   if(!GlobalVariableCheck(nfm)){
      GlobalVariableSet(nfm,tm);
      Comment(StringSubstr(LINK,0,StringFind(LINK,"/d"))+"\n"+txt);
   }else{
      if(GlobalVariableGet(nfm)>=tm){return;}
      GlobalVariableSet(nfm,tm);
      Comment(StringSubstr(LINK,0,StringFind(LINK,"/d"))+"\n"+txt);
   }
   return;
}
//+------------------------------------------------------------------+
//|                              Function : CheckHistFile(string fn) |
//|                                 Copyright © 2013, XrustSolution. |
//+------------------------------------------------------------------+
bool CheckHistFile(string fn){int hd;
   hd = FileOpen(fn,FILE_BIN|FILE_READ);
   if(hd<1){return(false);}
   FileClose(hd);
   return(true);
}
//+------------------------------------------------------------------+
//|                     Function : WriteHistFile(string fn,int arr[])|
//|                                 Copyright © 2013, XrustSolution. |
//+------------------------------------------------------------------+
bool WriteHistFile(string fn,int co,int &arr[]){bool have = false; int i,ii,x,y,tm,hd,sz,buff[100000],out[100000];
   ArrayResize(arr,ArraySize(arr)/4);
   if(co==0){
      hd = FileOpen(fn,FILE_BIN|FILE_READ);
      if(hd>0){
         sz = FileReadArray(hd,buff,0,FileSize(hd));
         FileClose(hd); 
         ArrayResize(buff,sz);
         tm = iTime(Symbol(),PERIOD_D1,0);
         for(i=0;i<ArraySize(buff);i++){
            if(buff[i]>=tm){have = true;break;}
         }
         if(have){
            sz = ArraySize(buff)-i;
            ii = ArraySize(arr);   
            ArrayResize(arr,sz+ii);
            ArrayCopy(arr,buff,ii,i,sz);
         }
      }
   }
   hd = FileOpen(fn,FILE_BIN|FILE_WRITE);
   if(hd<1){
      Alert("File history create error :"+GetLastError()+"\n Please check your UAC settings \n System will be Stopped.");
      return(false);
   }
   if(FileWriteArray(hd,arr,0,ArraySize(arr))<0){
      Alert("File history write error :"+GetLastError()+"\n Please check your UAC settings \n System will be Stopped.");
      FileClose(hd);
      return(false);
   }
   FileClose(hd);
   return(true);
}
//+------------------------------------------------------------------+
//|                          Function  : StringReplace(in,find,repl) |
//|                          Copyright © 2010, XrustSolution.        |
//+------------------------------------------------------------------+
string StringReplace(string in,string find,string repl=""){int x;string out="";
	x  = StringFind(in,find,0);
	if(x<0){return(in);}
	if(x==0){
		out = StringConcatenate(repl,StringSubstr(in,x+StringLen(find)));
	}else{
		out = StringConcatenate(StringSubstr(in,0,x),repl,StringSubstr(in,x+StringLen(find)));
	}
	return(out);
}
//+------------------------------------------------------------------+
//|                                 Function  : Int2Str(int)         |
//|                                 Copyright © 2013, XrustSolution. |
//+------------------------------------------------------------------+
string Int2Str(string in){int i,ii=1;string out="";
   for(i=StringLen(in)-1;i>=0;i--){
      if(ii%3==0){
         out = out+StringSubstr(in,i,1)+" ";
      }else{
         out = out+StringSubstr(in,i,1);
      }
      ii++;
   }
   in="";
   for(i=StringLen(out)-1;i>=0;i--){in=in+StringSubstr(out,i,1);}
   return(in);
}
//+------------------------------------------------------------------+
//|                           Function  : InetToArray(Url, &array[]) |
//|                                 Copyright © 2013, XrustSolution. |
//+------------------------------------------------------------------+
//|   ×èòàåò áèíàðíûé ôàéë ñ óäàëåííîãî âåáñåðâåðà,                  |
//| ïèøåò åãî â ïåðåäàâàåìûé îäíîìåðíûé ìàññèâ,âîçâðàùàåò êîä îøèáêè |
//+------------------------------------------------------------------+
int InetToArray(string fUrl, int &out[]){int res=0;
   ArrayResize(out,1000000);
   ArrayInitialize(out,0);
   if(!IsDllsAllowed()){
     Alert("Must be configured to allow the use of DLL");
     res=4020;
     return(res);
   }
   int rv = InternetAttemptConnect(0);
   if(rv != 0){
     Print("Error when calling InternetAttemptConnect ()");
     res=1002;
     return(res);
   }
   int hSession = InternetOpenA(AGENT, INTERNET_OPEN_TYPE_DIRECT, "0", "0", 0);
   if(hSession <= 0){
     Print("Error when calling InternetOpenA()");
     res=1003;
     return(res);         
   } 	
	int hReq = InternetOpenUrlA(hSession, fUrl, "0", 0,
     									INTERNET_FLAG_NO_CACHE_WRITE |
     									INTERNET_FLAG_PRAGMA_NOCACHE |
     									INTERNET_FLAG_RELOAD, 0
     									);
   if(hReq <= 0){
     Print("Error when calling InternetOpenUrlA()");
     InternetCloseHandle(hSession);
     res=1004;
     return(res);
   }
   int cBuffer[256];
   ArrayInitialize(cBuffer,0);
   int dwBytesRead[1]; 
   ArrayInitialize(dwBytesRead,0);
   string TXT = "";
   int i=0,ii=0,len=0;
   while(!IsStopped()){
    	if(!InternetReadFile(hReq, cBuffer, 256, dwBytesRead)){
    	 Print("File not found");
    	 res=404;
    	 break;
    	}
    	if(dwBytesRead[0] == 0){break;}
    	ArrayCopy(out,cBuffer,ii,0,dwBytesRead[0]);
    	ii+=(dwBytesRead[0]/4);
    	i+=dwBytesRead[0];
    	Comment("Download : "+i+" bytes");
    	Sleep(1);
  	}
  	InternetCloseHandle(hReq);
  	InternetCloseHandle(hSession);  
  	ArrayResize(out,i); 									
  	return(0);         
}
//+------------------------------------------------------------------+
//|                                 Function  : InetToString(url,out)|
//|                                 Copyright © 2010, XrustSolution. |
//|                                           mail: xrustx@gmail.com |
//+------------------------------------------------------------------+
//|  ×èòàåò òåêñòîâûé ôàéë ñ óäàëåííîãî âåáñåðâåðà,                  |
//| ïèøåò åãî â ïåðåäàâàåìûé îäíîìåðíûé ìàññèâ,âîçâðàùàåò êîä îøèáêè |
//+------------------------------------------------------------------+
int InetToString(string fUrl, string &out){
   if(!IsDllsAllowed()){
     Alert("Must be configured to allow the use of DLL");
     return(1001);
   }
   int rv = InternetAttemptConnect(0);
   if(rv != 0){
     Print("Error when calling InternetAttemptConnect ()");
     return(1002);
   }
   int hSession = InternetOpenA(AGENT, INTERNET_OPEN_TYPE_DIRECT, "0", "0", 0);
   if(hSession <= 0){
     Print("Error when calling InternetOpenA()");
     return(1003);         
   } 	
	int hReq = InternetOpenUrlA(hSession, fUrl, "0", 0,
     									INTERNET_FLAG_NO_CACHE_WRITE |
     									INTERNET_FLAG_PRAGMA_NOCACHE |
     									INTERNET_FLAG_RELOAD, 0
     									);
   if(hReq <= 0){
     Print("Error when calling InternetOpenUrlA()");
     InternetCloseHandle(hSession);
     return(1004);
   }
   int cBuffer[256];
   ArrayInitialize(cBuffer,0);
   int dwBytesRead[1]; 
   ArrayInitialize(dwBytesRead,0);
   string TXT = "";
   while(!IsStopped()){
       	bool bResult = InternetReadFile(hReq, cBuffer, 1024, dwBytesRead);
       	if(dwBytesRead[0] == 0)break;
       	string text = "";   
       	for(int i = 0; i < 256; i++){
           	text = text + CharToStr(cBuffer[i] & 0x000000FF);
        	  	if(StringLen(text) == dwBytesRead[0])break;
        	  	text = text + CharToStr(cBuffer[i] >> 8 & 0x000000FF);
        	  	if(StringLen(text) == dwBytesRead[0])break;
           	text = text + CharToStr(cBuffer[i] >> 16 & 0x000000FF);
           	if(StringLen(text) == dwBytesRead[0])break;
           	text = text + CharToStr(cBuffer[i] >> 24 & 0x000000FF);
           	if(StringLen(text) == dwBytesRead[0])break;
         	}
       	TXT = TXT + text;
       	Sleep(1);
     	}
   if(StringFind(TXT,"<html>",0)>=0||StringFind(TXT,"<HTML>",0)>=0||StringFind(TXT,"<title>",0)>=0||StringFind(TXT,"<head>",0)>=0){
   	if(StringFind(TXT,">404",0)>=0){return(404);}
   }  
   out = TXT;  
  	InternetCloseHandle(hReq);
  	InternetCloseHandle(hSession);    									
  	return(0);         
}
//+------------------------------------------------------------------+-+