Managing Large Strings

[hunte922]

I'm using some code that adds string to the end of a larger string $text, and repeats doing so until a file has been completely scanned.

My dillema is that after the string $text reaches 200,000, the script becomes extremely slow. What is the best way to go about speeding up this script?

(This file needs to be in the script directory, and SCP-173.txt can go anywhere.)

(Click on Mesh Convert, select the file you've downloaded called "SCP-173.txt", and go with the default options in the next 4 prompts)

expandcollapse popup

#include <ButtonConstants.au3>
#include <File.au3>
#include <FileConstants.au3>
#include <GUIConstantsEx.au3>
#include <InetConstants.au3>
#include <StaticConstants.au3>
#include <WindowsConstants.au3>
#include <WinHttp.au3>
#include <WinHttpConstants.au3>
AutoItSetOption("GUICloseOnESC", 0)
AutoItSetOption("GUIOnEventMode", 1)
AutoItSetOption("TrayAutoPause", 0)
AutoItSetOption("TrayIconHide", 1)
Global $title = "RBXMC"

$Form1 = GUICreate($title, 380, 50)
GUISetBkColor(0xFFFFFF, $Form1)
$DownloadAsset = GUICtrlCreateButton("Download Asset", 0, 0, 95, 25)
GUICtrlSetOnEvent($DownloadAsset, "DownloadAsset")
$B64D = GUICtrlCreateButton("B64D", 95, 0, 47, 25)
GUICtrlSetOnEvent($B64D, "B64D")
$B64E = GUICtrlCreateButton("B64E", 142, 0, 47, 25)
GUICtrlSetOnEvent($B64E, "B64E")
$MC = GUICtrlCreateButton("Mesh Convert", 0, 25, 190, 25)
GUICtrlSetOnEvent($MC, "MC")
$progress = GUICtrlCreateProgress(191, 26, 188, 23)
$plabel = GUICtrlCreateLabel("0/0", 190, 0, 190, 25)
GUICtrlSetFont($plabel, 18)
GUICtrlSetStyle($plabel, $SS_RIGHT)

GUISetOnEvent($GUI_EVENT_CLOSE, "_exit")
GuiSetState()

Func DownloadAsset()
   Local $inpt = InputBox($title, "Please enter in the asset ID you wish to download.", "0", " M", -1 , -1, Default, Default, 0, $Form1)
   If Not @error Then
      Local $sFilePath = FileSaveDialog($title, @UserProfileDir, "All (*.*)", 3, $title & " Downloaded Asset.txt", $Form1)
      If Not @error Then
         HttpSetUserAgent("Roblox/WinInet")
         Local $hDownload = InetGet("http://www.roblox.com/asset/?id=" & $inpt, $sFilePath, $INET_FORCERELOAD, $INET_DOWNLOADBACKGROUND)
         Do
            Sleep(250)
         Until InetGetInfo($hDownload, $INET_DOWNLOADCOMPLETE)
         Local $iBytesSize = InetGetInfo($hDownload, $INET_DOWNLOADREAD)
         Local $iFileSize = FileGetSize($sFilePath)
         If $iBytesSize <= 0 Then
            MsgBox(262144+0, $title, "Failed to download asset.", 0, $Form1)
         Else
            ShellExecute($sFilePath)
         EndIf
         HttpSetUserAgent("")
      EndIf
   EndIf
EndFunc

Func B64D()
   Local $inpt = FileOpenDialog($title, @UserProfileDir, "Base64 encoded files (*.txt)|All (*.*)", 3, "", $Form1)
   If Not @error Then
      Local $sFilePath = FileSaveDialog($title, @UserProfileDir, "Base64 decoded files (*.txt)|All (*.*)", 3, $title & " Base64 Decoded File.txt", $Form1)
      If Not @error Then
         Local $sFileRead = FileRead($inpt)
         Local $text = _Base64Decode($sFileRead)
         Local $file = _FileCreate($sFilePath)
         Local $fileo = FileOpen($sFilePath, $FO_OVERWRITE)
         FileWrite($fileo, $text)
         FileClose($fileo)
         If Not @error Then
            MsgBox(262144+0, $title, "Successfully decoded file.", 0, $Form1)
            ShellExecute($sFilePath)
         Else
            MsgBox(262144+0, $title, "Failed to decode file.", 0, $Form1)
         EndIf
      EndIf
   EndIf
EndFunc

Func B64E()
   Local $inpt = FileOpenDialog($title, @UserProfileDir, "Base64 decoded files (*.txt)|All (*.*)", 3, "", $Form1)
   If Not @error Then
      Local $sFilePath = FileSaveDialog($title, @UserProfileDir, "Base64 encoded files (*.txt)|All (*.*)", 3, $title & " Base64 Encoded File.txt", $Form1)
      If Not @error Then
         Local $sFileRead = FileRead($inpt)
         Local $text = _Base64Encode($sFileRead)
         Local $file = _FileCreate($sFilePath)
         Local $fileo = FileOpen($sFilePath, $FO_OVERWRITE)
         FileWrite($fileo, $text)
         FileClose($fileo)
         If Not @error Then
            MsgBox(262144+0, $title, "Successfully encoded file.", 0, $Form1)
            ShellExecute($sFilePath)
         Else
            MsgBox(262144+0, $title, "Failed to encode file.", 0, $Form1)
         EndIf
      EndIf
   EndIf
EndFunc

Func MC()
   Local $inpt = FileOpenDialog($title, @UserProfileDir, "Roblox Mesh (*.txt;*.mesh)|All (*.*)", 3, "", $Form1)
   If Not @error Then
      Local $sFilePath = FileSaveDialog($title, @UserProfileDir, "Roblox Model Files (*.rbxm)|All (*.*)", 3, $title & " Converted Mesh.rbxm", $Form1)
      If Not @error Then
         Local $decalstipend = "0"
         Local $scale[4]
         $scale[1] = 1
         $scale[2] = 1
         $scale[3] = 1
         Local $offset[4]
         $offset[1] = 0
         $offset[2] = 0
         $offset[3] = 0
         Local $v0 = InputBox($title, "Please enter in the asset ID you wish texture your mesh.", "0", " M", -1 , -1, Default, Default, 0, $Form1)
         Local $v1 = InputBox($title, "Please enter in the Vector3 you wish to scale your mesh. (X,Y,Z)", "1,1,1", " M", -1 , -1, Default, Default, 0, $Form1)
         Local $v2 = InputBox($title, "Please enter in the Vector3 you wish to offset your mesh. (X,Y,Z)", "0,0,0", " M", -1 , -1, Default, Default, 0, $Form1)
         If $v0 And $v0 <> "0" Then
            $decalstipend = "http://www.roblox.com/asset/?id=" & $v0
         EndIf
         If $v1 And $v1 <> "1,1,1" Then
            Local $na = StringSplit(StringReplace($v1, " ", ""), ",")
            $scale[1] = $na[1]
            $scale[2] = $na[2]
            $scale[3] = $na[3]
         EndIf
         If $v2 And $v2 <> "0,0,0" Then
            Local $na = StringSplit(StringReplace($v2, " ", ""), ",")
            $offset[1] = $na[1]
            $offset[2] = $na[2]
            $offset[3] = $na[3]
         EndIf
         GUICtrlSetState($DownloadAsset, $GUI_DISABLE)
         GUICtrlSetState($B64D, $GUI_DISABLE)
         GUICtrlSetState($B64E, $GUI_DISABLE)
         GUICtrlSetState($MC, $GUI_DISABLE)
         Local $sFileRead = FileRead($inpt)
         Local $temp = StringSplit(FileRead(@ScriptDir & "\MeshTemplate.rbxm"), "#")
         Local $flen = StringLen($sFileRead)
         Local $limiter = 1
         Local $text = "1:"
         Local $order = ":"
         Local $verticy = 0
         Local $connector = 0
         Local $vproperty = 0
         While 1
            Local $lbfind = StringInStr($sFileRead, "[", 0, $limiter)
            Local $rbfind = StringInStr($sFileRead, "]", 0, $limiter)
            GUICtrlSetData($progress, ($rbfind/$flen)*100)
            GUICtrlSetData($plabel, String($flen-$rbfind) & "/" & String($flen))
            $limiter = $limiter + 1
            If $lbfind Then
               $vproperty = $vproperty + 1
               If $vproperty > 3 Then
                  $vproperty = 1
                  $verticy = $verticy + 1
                  $connector = $connector + 1
                  If $verticy > 3 Then
                     $verticy = 1
                  EndIf
               EndIf
               If $vproperty = 1 Then
                  Local $nums = StringSplit(StringReplace(StringTrimLeft(StringLeft($sFileRead, $rbfind - 1), $lbfind), " ", ""), ",")
                  $text = $text & String((Number($nums[1])*$scale[1])+$offset[1]) & "," & String((Number($nums[2])*$scale[2])+$offset[2]) & "," & String((Number($nums[3])*$scale[3])+$offset[3]) & "|#1|255,255,255,255|0,0,0,0|#2|0,0|0,0|"
                  ;$text = $text & StringReplace(StringTrimLeft(StringLeft($sFileRead, $rbfind - 1), $lbfind), " ", "") & "|#1|255,255,255,255|0,0,0,0|#2|0,0|0,0|"
               ElseIf $vproperty = 2 Then
                  $text = $text & StringReplace(StringTrimLeft(StringLeft($sFileRead, $rbfind - 1), $lbfind), " ", "") & "|0,0,0,0|"
                  $text = StringReplace($text, "#1", StringReplace(StringTrimLeft(StringLeft($sFileRead, $rbfind - 1), $lbfind), " ", ""))
               ElseIf $vproperty = 3 Then
                  Local $nums = StringSplit(StringReplace(StringTrimRight(StringTrimLeft(StringLeft($sFileRead, $rbfind - 1), $lbfind), 3), " ", ""), ",")
                  $text = StringReplace($text, "#2", String(Number($nums[1])) & "," & String(1-Number($nums[2]))) & ";"
                  $order = $order & String($connector) & ";"
               EndIf
               ;MsgBox(0, "", $text & $order)
            Else
               ExitLoop
            EndIf
            _ReduceMemory()
         WEnd
         GUICtrlSetData($progress, 100)
         $text = $temp[1] & _Base64Encode($text & $order) & $temp[2] & $decalstipend & $temp[3]
         Local $file = _FileCreate($sFilePath)
         Local $fileo = FileOpen($sFilePath, $FO_OVERWRITE)
         FileWrite($fileo, $text)
         FileClose($fileo)
         If Not @error Then
            MsgBox(262144+0, $title, "Successfully converted file.", 0, $Form1)
         Else
            MsgBox(262144+0, $title, "Failed to convert file.", 0, $Form1)
         EndIf
         GUICtrlSetData($progress, 0)
         GUICtrlSetData($plabel, "0/0")
         GUICtrlSetState($DownloadAsset, $GUI_ENABLE)
         GUICtrlSetState($B64D, $GUI_ENABLE)
         GUICtrlSetState($B64E, $GUI_ENABLE)
         GUICtrlSetState($MC, $GUI_ENABLE)
      EndIf
   EndIf
EndFunc

Func _Base64Decode($sData)
   Local $oXml = ObjCreate("Msxml2.DOMDocument")
   If Not IsObj($oXml) Then
      SetError(1, 1, 0)
   EndIf
   Local $oElement = $oXml.createElement("b64")
   If Not IsObj($oElement) Then
      SetError(2, 2, 0)
   EndIf
   $oElement.dataType = "bin.base64"
   $oElement.Text = $sData
   Local $sReturn = BinaryToString($oElement.nodeTypedValue, 4)
   If StringLen($sReturn) = 0 Then
      SetError(3, 3, 0)
   EndIf
   Return $sReturn
EndFunc

Func _Base64Encode($sData)
    Local $oXml = ObjCreate("Msxml2.DOMDocument")
    If Not IsObj($oXml) Then
        SetError(1, 1, 0)
    EndIf
    Local $oElement = $oXml.createElement("b64")
    If Not IsObj($oElement) Then
        SetError(2, 2, 0)
    EndIf
    $oElement.dataType = "bin.base64"
    $oElement.nodeTypedValue = Binary($sData)
    Local $sReturn = $oElement.Text
    If StringLen($sReturn) = 0 Then
        SetError(3, 3, 0)
    EndIf
    Return $sReturn
 EndFunc

Func _ReduceMemory($i_PID = -1)
   If $i_PID <> -1 Then
      Local $ai_Handle = DllCall("kernel32.dll", 'int', 'OpenProcess', 'int', 0x1f0fff, 'int', False, 'int', $i_PID)
      $ai_Return = DllCall("psapi.dll", 'int', 'EmptyWorkingSet', 'long', $ai_Handle[0])
      DllCall('kernel32.dll', 'int', 'CloseHandle', 'int', $ai_Handle[0])
   Else
      $ai_Return = DllCall("psapi.dll", 'int', 'EmptyWorkingSet', 'long', -1)
   EndIf
   Return $ai_Return[0]
EndFunc   ;==>_ReduceMemory

Func _exit()
   Exit
EndFunc

While 1
WEnd

Edited April 18, 2014 by hunte922

[orbs]

i added a timing log to your main loop in Func MC(), like this:

expandcollapse popup

Func MC()
   Local $inpt = FileOpenDialog($title, 'E:\Downloads', "Roblox Mesh (*.txt;*.mesh)|All (*.*)", 3, "", $Form1)
   If Not @error Then
      Local $sFilePath = FileSaveDialog($title, 'E:\Downloads', "Roblox Model Files (*.rbxm)|All (*.*)", 3, $title & " Converted Mesh.rbxm", $Form1)
      If Not @error Then
         Local $decalstipend = "0"
         Local $scale[4]
         $scale[1] = 1
         $scale[2] = 1
         $scale[3] = 1
         Local $offset[4]
         $offset[1] = 0
         $offset[2] = 0
         $offset[3] = 0
         Local $v0 = InputBox($title, "Please enter in the asset ID you wish texture your mesh.", "0", " M", -1 , -1, Default, Default, 0, $Form1)
         Local $v1 = InputBox($title, "Please enter in the Vector3 you wish to scale your mesh. (X,Y,Z)", "1,1,1", " M", -1 , -1, Default, Default, 0, $Form1)
         Local $v2 = InputBox($title, "Please enter in the Vector3 you wish to offset your mesh. (X,Y,Z)", "0,0,0", " M", -1 , -1, Default, Default, 0, $Form1)
         If $v0 And $v0 <> "0" Then
            $decalstipend = "http://www.roblox.com/asset/?id=" & $v0
         EndIf
         If $v1 And $v1 <> "1,1,1" Then
            Local $na = StringSplit(StringReplace($v1, " ", ""), ",")
            $scale[1] = $na[1]
            $scale[2] = $na[2]
            $scale[3] = $na[3]
         EndIf
         If $v2 And $v2 <> "0,0,0" Then
            Local $na = StringSplit(StringReplace($v2, " ", ""), ",")
            $offset[1] = $na[1]
            $offset[2] = $na[2]
            $offset[3] = $na[3]
         EndIf
         GUICtrlSetState($DownloadAsset, $GUI_DISABLE)
         GUICtrlSetState($B64D, $GUI_DISABLE)
         GUICtrlSetState($B64E, $GUI_DISABLE)
         GUICtrlSetState($MC, $GUI_DISABLE)
         Local $sFileRead = FileRead($inpt)
         Local $temp = StringSplit(FileRead(@ScriptDir & "\MeshTemplate.rbxm"), "#")
         Local $flen = StringLen($sFileRead)
         Local $limiter = 1
         Local $text = "1:"
         Local $order = ":"
         Local $verticy = 0
         Local $connector = 0
         Local $vproperty = 0

        ; >>> initializing timer
        Local $iTimerOpCount=0
        Local $hTimer
        $hTimer=TimerInit()
        ; >>> initializing timer end
        
         While 1
            Local $lbfind = StringInStr($sFileRead, "[", 0, $limiter)
            Local $rbfind = StringInStr($sFileRead, "]", 0, $limiter)
            GUICtrlSetData($progress, ($rbfind/$flen)*100)
            GUICtrlSetData($plabel, String($flen-$rbfind) & "/" & String($flen))
            $limiter = $limiter + 1
            If $lbfind Then
               $vproperty = $vproperty + 1
               If $vproperty > 3 Then
                  $vproperty = 1
                  $verticy = $verticy + 1
                  $connector = $connector + 1
                  If $verticy > 3 Then
                     $verticy = 1
                  EndIf
               EndIf
               If $vproperty = 1 Then
                  Local $nums = StringSplit(StringReplace(StringTrimLeft(StringLeft($sFileRead, $rbfind - 1), $lbfind), " ", ""), ",")
                  $text = $text & String((Number($nums[1])*$scale[1])+$offset[1]) & "," & String((Number($nums[2])*$scale[2])+$offset[2]) & "," & String((Number($nums[3])*$scale[3])+$offset[3]) & "|#1|255,255,255,255|0,0,0,0|#2|0,0|0,0|"
                  ;$text = $text & StringReplace(StringTrimLeft(StringLeft($sFileRead, $rbfind - 1), $lbfind), " ", "") & "|#1|255,255,255,255|0,0,0,0|#2|0,0|0,0|"
               ElseIf $vproperty = 2 Then
                  $text = $text & StringReplace(StringTrimLeft(StringLeft($sFileRead, $rbfind - 1), $lbfind), " ", "") & "|0,0,0,0|"
                  $text = StringReplace($text, "#1", StringReplace(StringTrimLeft(StringLeft($sFileRead, $rbfind - 1), $lbfind), " ", ""))
               ElseIf $vproperty = 3 Then
                  Local $nums = StringSplit(StringReplace(StringTrimRight(StringTrimLeft(StringLeft($sFileRead, $rbfind - 1), $lbfind), 3), " ", ""), ",")
                  $text = StringReplace($text, "#2", String(Number($nums[1])) & "," & String(1-Number($nums[2]))) & ";"
                  $order = $order & String($connector) & ";"
               EndIf
               ;MsgBox(0, "", $text & $order)
            Else
               ExitLoop
            EndIf
            _ReduceMemory()
            
            ; >>> log timing
            $iTimerOpCount+=1
            If $iTimerOpCount=100 Then
                FileWriteLine(@ScriptDir&'\RXBM-timing.txt',TimerDiff($hTimer))
                $iTimerOpCount=0
                $hTimer=TimerInit()
            EndIf
            ; >>> log timing end
            
         WEnd
         GUICtrlSetData($progress, 100)
         $text = $temp[1] & _Base64Encode($text & $order) & $temp[2] & $decalstipend & $temp[3]
         Local $file = _FileCreate($sFilePath)
         Local $fileo = FileOpen($sFilePath, $FO_OVERWRITE)
         FileWrite($fileo, $text)
         FileClose($fileo)
         If Not @error Then
            MsgBox(262144+0, $title, "Successfully converted file.", 0, $Form1)
         Else
            MsgBox(262144+0, $title, "Failed to convert file.", 0, $Form1)
         EndIf
         GUICtrlSetData($progress, 0)
         GUICtrlSetData($plabel, "0/0")
         GUICtrlSetState($DownloadAsset, $GUI_ENABLE)
         GUICtrlSetState($B64D, $GUI_ENABLE)
         GUICtrlSetState($B64E, $GUI_ENABLE)
         GUICtrlSetState($MC, $GUI_ENABLE)
      EndIf
   EndIf
EndFunc

time is measured every 100 iterations (arbitrary). my results (did not go through the entire process):

154.214431886744

248.084207776375

342.071232638133

425.65544352456

519.108161399275

608.166375358433

695.432266170717

793.182143323651

883.277248723324

952.823305487998

1040.67761819754

1137.2590486972

1208.94077838389

1299.41251111844

1390.48666098564

1472.43622837736

1568.33842345228

1648.50313182268

1719.71835366271

1804.22438405414

1889.87584671174

1966.48669830627

2056.56065535022

2158.60271569768

2225.97266886029

2325.4349408157

2409.23032425405

2501.81316049736

2607.09247429542

2689.79778439873

 

you see a gradual increase in processing time.

i still did not determine the cause; but this is a benchmark you can use for investigation. if i can find something further, i'll post it here.