grabweb1
Download script
This script is designed to automatically gather economic news data from an external website and save it into a structured file format that can be easily read by other applications or used for analysis.
How the Logic Works:
1. Connecting to the Source The script begins by establishing a connection to a specific web address (the Forex Factory calendar). It mimics a standard web browser to ensure the website allows the connection to proceed smoothly.
2. Retrieving the Data Once connected, the script "reads" the raw code of the webpage. Think of this as downloading the entire blueprint of the website's calendar page. Because large webpages can be heavy, the script captures this data in small chunks and pieces them together into one continuous stream of information.
3. Saving a Temporary Copy Before doing any complex processing, the script saves a complete copy of this raw webpage data as an HTML file on your computer. This serves as a backup of the information captured at that specific moment.
4. Extracting Specific Information (Parsing) The core logic then moves to "filtering" the mess of code. The script looks for specific markers within the HTML to find the actual news entries. It specifically hunts for:
- The Date and Time of the event
- The Country involved
- The Title of the news event
- The Importance level (visual icons)
- Economic figures like the "Actual," "Forecast," and "Previous" values
5. Organizing into a Spreadsheet Format As the script identifies each piece of information, it cleans up any extra spaces and organizes them into a single line. Finally, it writes this organized data into a CSV file (a format compatible with Excel or other spreadsheet software). It ensures that every new piece of news is placed on its own new line, creating a clean, structured list of economic events.
Summary of the Workflow:
- Step 1: Reach out to the internet to find the news.
- Step 2: Download the raw webpage content.
- Step 3: Save a copy of that content locally.
- Step 4: Scour the code to find only the specific economic details you care about.
- Step 5: Export those details into a neat, organized file on your hard drive.
/*-----------------------------+
| |
| Shared by www.Aptrafx.com |
| |
+------------------------------*/
//+------------------------------------------------------------------+
//| grabweb.mq4 |
//| Copyright © 2006, Abhi |
//| http://www.megadelfi.com/experts/ |
//| E-mail: grabwebexpert{Q)megadelfi.com |
//| fix my e-mail address before mailing me ;) |
//+------------------------------------------------------------------+
#property copyright "Copyright © 2006, Abhi"
#property link "http://www.megadelfi.com/experts/"
extern string sUrl = "http://www.forexfactory.com/index.php?s=&page=calendar&timezoneoffset=t+3";
int hSession_IEType;
int hSession_Direct;
int Internet_Open_Type_Preconfig = 0;
int Internet_Open_Type_Direct = 1;
int Internet_Open_Type_Proxy = 3;
int Buffer_LEN = 13;
#import "wininet.dll"
int InternetOpenA(
string sAgent,
int lAccessType,
string sProxyName="",
string sProxyBypass="",
int lFlags=0
);
int InternetOpenUrlA(
int hInternetSession,
string sUrl,
string sHeaders="",
int lHeadersLength=0,
int lFlags=0,
int lContext=0
);
int InternetReadFile(
int hFile,
string sBuffer,
int lNumBytesToRead,
int& lNumberOfBytesRead[]
);
int InternetCloseHandle(
int hInet
);
int hSession(bool Direct)
{
string InternetAgent;
if (hSession_IEType == 0)
{
InternetAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Q312461)";
hSession_IEType = InternetOpenA(InternetAgent, Internet_Open_Type_Preconfig, "0", "0", 0);
hSession_Direct = InternetOpenA(InternetAgent, Internet_Open_Type_Direct, "0", "0", 0);
}
if (Direct)
{ return(hSession_Direct); }
else
{ return(hSession_IEType); }
}
int start()
{
// Print("fake start");
}
int init()
{
int hInternet;
string iResult;
int lReturn[]={1};
string sBuffer="x";
string sData, csvoutput;
int handle;
int bytes;
int beginning, finalend,end,i;
string aju, text, newsdate, newstime, country, news, level, actual, forecast, previous;
hInternet = InternetOpenUrlA(hSession(FALSE), sUrl, "0", 0, 67108864, 0);
Print("hInternet: " + hInternet);
iResult = InternetReadFile(hInternet, sBuffer, Buffer_LEN, lReturn);
Print("iResult: " + iResult);
Print("lReturn: " + lReturn[0]);
//iResult = InternetCloseHandle(hInternet);
Print("iResult: " + iResult);
Print("sBuffer: " + sBuffer);
Comment("sBuffer: " + sBuffer);
bytes = lReturn[0];
sData = StringSubstr(sBuffer, 0, lReturn[0]);
//if there's more data then keep reading it into the buffer
while (lReturn[0] != 0)
{
iResult = InternetReadFile(hInternet, sBuffer, Buffer_LEN, lReturn);
if (lReturn[0]==0) break;
bytes = bytes + lReturn[0];
sData = sData + StringSubstr(sBuffer, 0, lReturn[0]);
}
Print("sData: " + sData);
iResult = InternetCloseHandle(hInternet);
Print("iResult: " + iResult);
handle=FileOpen(Day()+ "-"+ Month() +"-"+ Year() +"-"+ "grabweb.htm",FILE_BIN|FILE_READ|FILE_WRITE);
if(handle<1)
{
Print("Can\'t open htm file, the last error is ", GetLastError());
return(false);
}
FileWriteString(handle,sData,StringLen(sData));
FileClose(handle);
Print("bytes: "+bytes);
// htm file is written, let's parse it now and write to csv file
handle=FileOpen(Day()+ "-"+ Month() +"-"+ Year() +"-"+ "grabweb.csv",FILE_BIN|FILE_READ|FILE_WRITE);
if(handle<1)
{
Print("Can\'t open csv file, the last error is ", GetLastError());
return(false);
}
beginning = 1;
beginning = StringFind(sData,"<span class=\"smallfont\">Info.</span>",beginning)+36;
finalend = StringFind(sData,"</table>",beginning)-1235;
while (beginning < finalend)
{
for (i=0; i < 8; i++)
{
beginning = StringFind(sData,"<span class=\"smallfont\"",beginning)+23;
beginning = StringFind(sData,">",beginning)+1;
end = StringFind(sData,"</span>",beginning)-0;
if (end != beginning)
{
text = StringSubstr(sData, beginning, end-beginning);
}
else
{
text = "";
}
if (i == 0)
{
if (text != "")
{
newsdate = text;
}
else
{
text = newsdate;
}
}
// date and time conversion here if needed
if (i == 1)
{
text = StringTrimLeft(text);
newstime = StringTrimRight(text);
}
if (i == 2)
{
country = text;
}
if (i == 3)
{
news = text;
beginning = StringFind(sData,"src=\"http://www.forexfactory.com/forexforum/images/misc/",beginning)+59;
end = StringFind(sData,".gif",beginning);
text = StringSubstr(sData,beginning,end-beginning);
level = text;
}
if (i == 4)
{
actual = text;
}
if (i == 5)
{
forecast = text;
}
if (i == 6)
{
previous = text;
}
}
csvoutput = newsdate + ", " + newstime + ", " + country + ", " + news + ", " +level + ", " + actual + ", " + forecast + ", " + previous;
Print("newslist: "+ csvoutput);
FileWriteString(handle,csvoutput,StringLen(csvoutput));
FileWriteString(handle,"\x0D\x0A",2); // force new line
}
FileClose(handle);
return(0);
}
//+------------------------------------------------------------------+
Comments