Auto Download files

[Dante_t]

Hi guys,

I have autoit script which i use to download files from a certain site. the script logs in and then download the files. my problem is, sometimes downloading of maybe 1 or 2 files out of 4 that has to be downloaded fails, and that is a problem because i need all 4 files for further processing. How can i prevent download failures or atleast make sure that all files are downloaded as required? The whole process is automated. below is my script. Your help will be highly appreciated. Thank you in advance.

expandcollapse popup

#include "WinHttp.au3"
#include "Date.au3"
#include <FileConstants.au3>

$sServerAddress = "https://rda.ucar.edu"
$sGeneratorLocation = "/cgi-bin/login"

$sEmail = "xxxxx@xxx.co.za"
$sPassword = "xxxxxxx"

$hOpen = _WinHttpOpen()

; collect access cookie first
$hConnect = _WinHttpConnect($hOpen, $sServerAddress)
_WinHttpSimpleSSLRequest($hConnect)
_WinHttpCloseHandle($hConnect)

; build and fill the login form
$sForm = _
        '<form action="' & $sServerAddress & $sGeneratorLocation & '" method="post">' & _
        '   <input name="email" />' & _
        '   <input name="passwd" />' & _
        '   <input name="remember" />' & _
        '   <input name="do" />' & _
        '   <input name="url" />' & _
        '</form>'
$hConnect = $sForm

$sReturned = _WinHttpSimpleFormFill($hConnect, $hOpen, _
        Default, _
        "name:email", $sEmail, _
        "name:passwd", $sPassword, _
        "name:remember", "on", _
        "name:do", "login", _
        "name:url", "/")
If @error Then
    MsgBox(4096, "Error", @error)
Else


;=========================setup file naming according to date========================================

Local $Today = _Date_Time_GetSystemTime()
Local $CalDate = _Date_Time_SystemTimeToDateStr($Today)
Local $Yesterday = _DateTimeFormat(_DateAdd('d', -1, _NowCalcDate()), 2)
 ; Split DTG
 $aYesterday = StringSplit($Yesterday, "/")
 ; Reassemble in desired order
 $sYear = $aYesterday[1]
 $sMonth = _DateToMonth($aYesterday[2])
 $sYearMonth = $aYesterday[1] & "." & $aYesterday[2]
 $sYesterdayF = $aYesterday[1] & $aYesterday[2] & $aYesterday[3]
 $sFile1 = "fnl_" & $sYesterdayF & "_00_00.grib2"
 $sFile2 = "fnl_" & $sYesterdayF & "_06_00.grib2"
 $sFile3 = "fnl_" & $sYesterdayF & "_12_00.grib2"
 $sFile4 = "fnl_" & $sYesterdayF & "_18_00.grib2"

 $sTarget1 = "data/ds083.2/grib2/" & $sYear & "/" & $sYearMonth & "/" & $sFile1
 $sTarget2 = "data/ds083.2/grib2/" & $sYear & "/" & $sYearMonth & "/" & $sFile2
 $sTarget3 = "data/ds083.2/grib2/" & $sYear & "/" & $sYearMonth & "/" & $sFile3
 $sTarget4 = "data/ds083.2/grib2/" & $sYear & "/" & $sYearMonth & "/" & $sFile4

;create a folder where
 $sFldr = "C:\data\"& $sMonth & " " & $sYear &"\"
 DirCreate($sFldr)

;====================================================================================================
;========================================download file 1=============================================
  _WinHttpCloseHandle($hConnect)
  $hConnect = _WinHttpConnect($hOpen, $sServerAddress)
  $hRequest = _WinHttpSimpleSendSSLRequest($hConnect, "GET", $sTarget1)
  If _WinHttpQueryDataAvailable($hRequest) Then

    $headers = _WinHttpQueryHeaders($hRequest)
    $Length = StringRegExpReplace($headers, '(?s).*Content-Length:\h*(\d+).*', "$1")
    Local $bChunk, $bData = Binary("")

    ProgressOn($sFile1, "Downloading", "0 %")
    While 1
         $bChunk = _WinHttpReadData($hRequest, 2, 8192) ; binary
         If @error Then ExitLoop
         $bData &= $bChunk
         $percent = Int((BinaryLen($bData)/$Length)*100)
         ProgressSet($percent, "Downloading", $percent & " %")
     WEnd
     ProgressSet(100, "Done !", "100 %")

     FileWrite($sFldr & $sFile1, $bData )
     Sleep(1000)
     ProgressOff()
  Else
      MsgBox(48, "", "connection error")
  EndIf
  _WinHttpCloseHandle($hRequest)
  sleep(500)
;====================================================================================================
;========================================download file 2=============================================
 $hRequest = _WinHttpSimpleSendSSLRequest($hConnect, "GET", $sTarget2)
  If _WinHttpQueryDataAvailable($hRequest) Then

    $headers = _WinHttpQueryHeaders($hRequest)
    $Length = StringRegExpReplace($headers, '(?s).*Content-Length:\h*(\d+).*', "$1")
    Local $bChunk, $bData = Binary("")

    ProgressOn($sFile2, "Downloading", "0 %")
    While 1
         $bChunk = _WinHttpReadData($hRequest, 2, 8192) ; binary
         If @error Then ExitLoop
         $bData &= $bChunk
         $percent = Int((BinaryLen($bData)/$Length)*100)
         ProgressSet($percent, "Downloading", $percent & " %")
     WEnd
     ProgressSet(100, "Done !", "100 %")
     FileWrite($sFldr & $sFile2, $bData)
     Sleep(1000)
     ProgressOff()
  Else; retry connecting

    $headers = _WinHttpQueryHeaders($hRequest)
    $Length = StringRegExpReplace($headers, '(?s).*Content-Length:\h*(\d+).*', "$1")
    Local $bChunk, $bData = Binary("")

    ProgressOn($sFile2, "Downloading", "0 %")
    While 1
         $bChunk = _WinHttpReadData($hRequest, 2, 8192) ; binary
         If @error Then ExitLoop
         $bData &= $bChunk
         $percent = Int((BinaryLen($bData)/$Length)*100)
         ProgressSet($percent, "Downloading", $percent & " %")
     WEnd
     ProgressSet(100, "Done !", "100 %")
     FileWrite($sFldr & $sFile2, $bData)
     Sleep(1000)
     ProgressOff()
  EndIf
  _WinHttpCloseHandle($hRequest)
sleep(500)
;====================================================================================================
;========================================download file 3=============================================
  $hRequest = _WinHttpSimpleSendSSLRequest($hConnect, "GET", $sTarget3)
  If _WinHttpQueryDataAvailable($hRequest) Then

    $headers = _WinHttpQueryHeaders($hRequest)
    $Length = StringRegExpReplace($headers, '(?s).*Content-Length:\h*(\d+).*', "$1")
    Local $bChunk, $bData = Binary("")

    ProgressOn($sFile3, "Downloading", "0 %")
    While 1
         $bChunk = _WinHttpReadData($hRequest, 2, 8192) ; binary
         If @error Then ExitLoop
         $bData &= $bChunk
         $percent = Int((BinaryLen($bData)/$Length)*100)
         ProgressSet($percent, "Downloading", $percent & " %")
     WEnd
     ProgressSet(100, "Done !", "100 %")
     FileWrite($sFldr & $sFile3, $bData)
     Sleep(1000)
     ProgressOff()
  Else
      MsgBox(48, "", "connection error")
  EndIf
_WinHttpCloseHandle($hRequest)
sleep(500)
;====================================================================================================
;========================================download file 4=============================================
  $hRequest = _WinHttpSimpleSendSSLRequest($hConnect, "GET", $sTarget4)
  If _WinHttpQueryDataAvailable($hRequest) Then

    $headers = _WinHttpQueryHeaders($hRequest)
    $Length = StringRegExpReplace($headers, '(?s).*Content-Length:\h*(\d+).*', "$1")
    Local $bChunk, $bData = Binary("")

    ProgressOn($sFile4, "Downloading", "0 %")
    While 1
         $bChunk = _WinHttpReadData($hRequest, 2, 8192) ; binary
         If @error Then ExitLoop
         $bData &= $bChunk
         $percent = Int((BinaryLen($bData)/$Length)*100)
         ProgressSet($percent, "Downloading", $percent & " %")
     WEnd
     ProgressSet(100, "Done !", "100 %")
     FileWrite($sFldr & $sFile4, $bData)
     Sleep(1000)
     ProgressOff()
  Else
      MsgBox(48, "", "connection error")
  EndIf
;====================================================================================================

; close handles
_WinHttpCloseHandle($hRequest)
_WinHttpCloseHandle($hConnect)
_WinHttpCloseHandle($hOpen)

EndIf

 

Edited September 6, 2017 by Jos
Please use Codebox for script source.

[Xenobiologist]

I did not look at your code, but I suggest to use wget to download your files. You can script around wget with Autoit.

wget supports many parameters to get the job done.

[careca]

After download use "fileexists", if false, re-download.

[Dante_t]

On 06 September 2017 at 11:40 AM, Xenobiologist said:

I did not look at your code, but I suggest to use wget to download your files. You can script around wget with Autoit.

wget supports many parameters to get the job done.

Hi Xenobiologist,

My script is working fine, I just want to put measures in place in case something goes wrong.

On 06 September 2017 at 0:12 PM, careca said:

After download use "fileexists", if false, re-download.

Hi Careca,

How do i use that? can you use parts of my script as an example?

 

Regards

[careca]

expandcollapse popup

#include "WinHttp.au3"
#include "Date.au3"
#include <FileConstants.au3>

$sServerAddress = "https://rda.ucar.edu"
$sGeneratorLocation = "/cgi-bin/login"

$sEmail = "xxxxx@xxx.co.za"
$sPassword = "xxxxxxx"

$hOpen = _WinHttpOpen()

; collect access cookie first
$hConnect = _WinHttpConnect($hOpen, $sServerAddress)
_WinHttpSimpleSSLRequest($hConnect)
_WinHttpCloseHandle($hConnect)

; build and fill the login form
$sForm = _
        '<form action="' & $sServerAddress & $sGeneratorLocation & '" method="post">' & _
        '   <input name="email" />' & _
        '   <input name="passwd" />' & _
        '   <input name="remember" />' & _
        '   <input name="do" />' & _
        '   <input name="url" />' & _
        '</form>'
$hConnect = $sForm

$sReturned = _WinHttpSimpleFormFill($hConnect, $hOpen, _
        Default, _
        "name:email", $sEmail, _
        "name:passwd", $sPassword, _
        "name:remember", "on", _
        "name:do", "login", _
        "name:url", "/")
If @error Then
    MsgBox(4096, "Error", @error)
Else


;=========================setup file naming according to date========================================

Local $Today = _Date_Time_GetSystemTime()
Local $CalDate = _Date_Time_SystemTimeToDateStr($Today)
Local $Yesterday = _DateTimeFormat(_DateAdd('d', -1, _NowCalcDate()), 2)
 ; Split DTG
 $aYesterday = StringSplit($Yesterday, "/")
 ; Reassemble in desired order
 $sYear = $aYesterday[1]
 $sMonth = _DateToMonth($aYesterday[2])
 $sYearMonth = $aYesterday[1] & "." & $aYesterday[2]
 $sYesterdayF = $aYesterday[1] & $aYesterday[2] & $aYesterday[3]
 $sFile1 = "fnl_" & $sYesterdayF & "_00_00.grib2"
 $sFile2 = "fnl_" & $sYesterdayF & "_06_00.grib2"
 $sFile3 = "fnl_" & $sYesterdayF & "_12_00.grib2"
 $sFile4 = "fnl_" & $sYesterdayF & "_18_00.grib2"

 $sTarget1 = "data/ds083.2/grib2/" & $sYear & "/" & $sYearMonth & "/" & $sFile1
 $sTarget2 = "data/ds083.2/grib2/" & $sYear & "/" & $sYearMonth & "/" & $sFile2
 $sTarget3 = "data/ds083.2/grib2/" & $sYear & "/" & $sYearMonth & "/" & $sFile3
 $sTarget4 = "data/ds083.2/grib2/" & $sYear & "/" & $sYearMonth & "/" & $sFile4

;create a folder where
 $sFldr = "C:\data\"& $sMonth & " " & $sYear &"\"
 DirCreate($sFldr)

 Download1()
  Download2()
   Download3()
    Download4()
EndIf

Func Download1()
;====================================================================================================
;========================================download file 1=============================================
  _WinHttpCloseHandle($hConnect)
  $hConnect = _WinHttpConnect($hOpen, $sServerAddress)
  $hRequest = _WinHttpSimpleSendSSLRequest($hConnect, "GET", $sTarget1)
  If _WinHttpQueryDataAvailable($hRequest) Then

    $headers = _WinHttpQueryHeaders($hRequest)
    $Length = StringRegExpReplace($headers, '(?s).*Content-Length:\h*(\d+).*', "$1")
    Local $bChunk, $bData = Binary("")

    ProgressOn($sFile1, "Downloading", "0 %")
    While 1
         $bChunk = _WinHttpReadData($hRequest, 2, 8192) ; binary
         If @error Then ExitLoop
         $bData &= $bChunk
         $percent = Int((BinaryLen($bData)/$Length)*100)
         ProgressSet($percent, "Downloading", $percent & " %")
     WEnd
     ProgressSet(100, "Done !", "100 %")

     FileWrite($sFldr & $sFile1, $bData )
     Sleep(1000)
     ProgressOff()
  Else
      MsgBox(48, "", "connection error")
  EndIf
  _WinHttpCloseHandle($hRequest)
  sleep(500)
EndFunc
;=============================================================================
Func Download2()
;====================================================================================================
;========================================download file 2=============================================
 $hRequest = _WinHttpSimpleSendSSLRequest($hConnect, "GET", $sTarget2)
  If _WinHttpQueryDataAvailable($hRequest) Then

    $headers = _WinHttpQueryHeaders($hRequest)
    $Length = StringRegExpReplace($headers, '(?s).*Content-Length:\h*(\d+).*', "$1")
    Local $bChunk, $bData = Binary("")

    ProgressOn($sFile2, "Downloading", "0 %")
    While 1
         $bChunk = _WinHttpReadData($hRequest, 2, 8192) ; binary
         If @error Then ExitLoop
         $bData &= $bChunk
         $percent = Int((BinaryLen($bData)/$Length)*100)
         ProgressSet($percent, "Downloading", $percent & " %")
     WEnd
     ProgressSet(100, "Done !", "100 %")
     FileWrite($sFldr & $sFile2, $bData)
     Sleep(1000)
     ProgressOff()
  Else; retry connecting

    $headers = _WinHttpQueryHeaders($hRequest)
    $Length = StringRegExpReplace($headers, '(?s).*Content-Length:\h*(\d+).*', "$1")
    Local $bChunk, $bData = Binary("")

    ProgressOn($sFile2, "Downloading", "0 %")
    While 1
         $bChunk = _WinHttpReadData($hRequest, 2, 8192) ; binary
         If @error Then ExitLoop
         $bData &= $bChunk
         $percent = Int((BinaryLen($bData)/$Length)*100)
         ProgressSet($percent, "Downloading", $percent & " %")
     WEnd
     ProgressSet(100, "Done !", "100 %")
     FileWrite($sFldr & $sFile2, $bData)
     Sleep(1000)
     ProgressOff()
  EndIf
  _WinHttpCloseHandle($hRequest)
sleep(500)
EndFunc
;=============================================================================
Func Download3()
;====================================================================================================
;========================================download file 3=============================================
  $hRequest = _WinHttpSimpleSendSSLRequest($hConnect, "GET", $sTarget3)
  If _WinHttpQueryDataAvailable($hRequest) Then

    $headers = _WinHttpQueryHeaders($hRequest)
    $Length = StringRegExpReplace($headers, '(?s).*Content-Length:\h*(\d+).*', "$1")
    Local $bChunk, $bData = Binary("")

    ProgressOn($sFile3, "Downloading", "0 %")
    While 1
         $bChunk = _WinHttpReadData($hRequest, 2, 8192) ; binary
         If @error Then ExitLoop
         $bData &= $bChunk
         $percent = Int((BinaryLen($bData)/$Length)*100)
         ProgressSet($percent, "Downloading", $percent & " %")
     WEnd
     ProgressSet(100, "Done !", "100 %")
     FileWrite($sFldr & $sFile3, $bData)
     Sleep(1000)
     ProgressOff()
  Else
      MsgBox(48, "", "connection error")
  EndIf
_WinHttpCloseHandle($hRequest)
sleep(500)
EndFunc
;=============================================================================
Func Download4()
;====================================================================================================
;========================================download file 4=============================================
  $hRequest = _WinHttpSimpleSendSSLRequest($hConnect, "GET", $sTarget4)
  If _WinHttpQueryDataAvailable($hRequest) Then

    $headers = _WinHttpQueryHeaders($hRequest)
    $Length = StringRegExpReplace($headers, '(?s).*Content-Length:\h*(\d+).*', "$1")
    Local $bChunk, $bData = Binary("")

    ProgressOn($sFile4, "Downloading", "0 %")
    While 1
         $bChunk = _WinHttpReadData($hRequest, 2, 8192) ; binary
         If @error Then ExitLoop
         $bData &= $bChunk
         $percent = Int((BinaryLen($bData)/$Length)*100)
         ProgressSet($percent, "Downloading", $percent & " %")
     WEnd
     ProgressSet(100, "Done !", "100 %")
     FileWrite($sFldr & $sFile4, $bData)
     Sleep(1000)
     ProgressOff()
  Else
      MsgBox(48, "", "connection error")
  EndIf
EndFunc
;====================================================================================================
Func Check()
    If FileExists($sFldr & $sFile1) = 0 Then
        Download1()
    EndIf
    If FileExists($sFldr & $sFile2) = 0 Then
        Download2()
    EndIf
    If FileExists($sFldr & $sFile3) = 0 Then
        Download3()
    EndIf
    If FileExists($sFldr & $sFile4) = 0 Then
        Download4()
    EndIf
EndFunc
;=============================================================================

; close handles
_WinHttpCloseHandle($hRequest)
_WinHttpCloseHandle($hConnect)
_WinHttpCloseHandle($hOpen)

Now it all depends on how you want to implement this, you can run it at the end of each download, or after all 4 downloads, and re-download if failed.