_FileListToArray Alternative

[aslani]

Here my script so far

expandcollapse popup

;
; AutoIt Version: 3.0
; Language:       English
; Platform:       WinXP
; Author:         aslani
;
; Script Function:
;   File Audit - Search for duplicates in the File Directory
; Version:
;   0.1 (beta)
;
; Include
#include "Process.au3"
#include <GUIConstants.au3>
#Include <Array.au3>
#include <Misc.au3>
#include <File.au3>

; Credits
$author = "aslani"
$version = "v.0.1"
$tools_1 = "AutoIt v.3.2.6.0 © 1999-2007 Jonathan Bennett & AutoIt Team (http://www.autoitscript.com)"
$tools_2 = "SciTE v.1.74 © 2007 by Neil Hodgson and SciTE Team (http://www.scintilla.org)"

; Variable
$title = "File Audit " & $version & " (beta)"
$wd = 35
$ht = 20
$row0 = 25
$row1 = 50
$row2 = 75
$row3 = 100
$row4 = 125
$offset_1 = 10
$offset_2 = $offset_1+65
$offset_3 = $offset_1+($wd*2)+10
$offset_4 = $offset_1+($wd*3)+15
$offset_5 = $offset_1+($wd*4)+20

; GUI Location Depeding on Screen Resolution: Min 800x600
Switch @DesktopHeight
    Case 960
        $guiloc_y = 700
    Case 768
        $guiloc_y = 578
    Case 600
        $guiloc_y = 410
EndSwitch

; .INI file for saving information.
If FileExists(@DesktopDir & "\file_list.ini") Then
    FileSetAttrib(@DesktopDir & "\file_list.ini", "-RASHNOT")
Else
    FileSetAttrib(@DesktopDir & "\file_list.ini", "-AHNOT+RS")
EndIf

; GUI - Main
$gui_main = GuiCreate("" & $title, 140, 105, -1, $guiloc_y)
;GuiSetIcon(@DesktopDir & "\My Desktop\ICO\Symbols\Find.ico", 0)

; Pull Down Menu
$helpmenu = GUICtrlCreateMenu ("&Help")
$h_about = GUICtrlCreateMenuItem ("&About File Audit", $helpmenu)

$label = GUICtrlCreateLabel ("Click Search to Start", $offset_1, $row0, 115, $ht)
;$progressbar = GuiCtrlCreateProgress($offset_1, $row0, 115, 10)
;$filepn = GuiCtrlCreateInput("", $offset_2-10, $row0-2, $wd+25, $ht, $ES_CENTER)
$button = GuiCtrlCreateButton("Search", $offset_1, $row1, 115, 25)

GuiCtrlCreateGroup("File Audit",  5, 5, 125, 75)

GUISetState (@SW_SHOW)

While 1
    $msg = GUIGetMsg()
    
    HotKeySet("+{ESC}", "_Terminate")
    
    Select
        Case $msg = $GUI_EVENT_CLOSE
            ExitLoop
            
        Case $msg = $h_about
            $about = "File Audit " & $version & @CRLF
            $about &= "Author: " & $author & @CRLF & @CRLF
            $about &= "Tools: " & @CRLF & $tools_1 & @CRLF
            $about &= $tools_2 & @CRLF & @CRLF
            $about &= "Made for Everyone"
            MsgBox(0, "About File Audit", $about)
            
        Case $msg = $button
            GUICtrlSetState($button, $GUI_DISABLE)
            GUICtrlSetData($label, "Searching...")
            $FileList=_FileListToArray("C:\File")
            ;$FileList = _ArrayCreate("1900123.pdf", "1900124a.pdf", "1900124b.pdf", "1900125.pdf", "1900126.pdf", "1900127.pdf", "1900128a.pdf", "1900128b.pdf") ;debug
            If @Error=1 Then
                MsgBox (0,"","No Files\Folders Found.")
                Exit
            EndIf
            ;_ArraySort($FileList)
            ;_ArrayDisplay($FileList,"$FileList") ;debug
            
            $NoDupList = _ArrayElementUniques($FileList)
            _ArrayDelete($NoDupList, "0")
            ;_ArrayDisplay($NoDupList,"$NoDupList") ;debug
            
            For $x = 0 To UBound($NoDupList) - 1
                $getpos = _ArraySearch($FileList, StringLeft($NoDupList[$x], 7), 0, 0, 0, True)
                _ArrayDelete($FileList, $getpos)
            Next
            
            ;_ArrayDisplay($FileList,"$FileList") ;debug
            
            FileSetAttrib(@DesktopDir & "\file_list.ini", "-RASHNOT")
            For $i = 0 To Ubound($FileList) - 1
                IniWrite(@DesktopDir & "\file_list.ini", "PDF List", "" & $FileList[$i], "")
            Next    
            FileSetAttrib(@DesktopDir & "\file_list.ini", "-AHNOT+RS")
            
            ShellExecute(@DesktopDir & "\file_list.ini")
            
            GUICtrlSetState($button, $GUI_ENABLE)
            GUICtrlSetData($label, "Click Search to Start") 
    EndSelect
WEnd

Func _waitRelease()
    $dll = DllOpen("user32.dll")
    While 1
        Sleep (100)
        If _IsPressed("A1", $dll) Or _IsPressed("A3", $dll) Then
            ; assume a function key is still pressed and loop again
        Else
            ExitLoop
        EndIf
    WEnd
    DllClose($dll)
EndFunc

Func _Terminate()
    _waitRelease()
    TrayTip("Status: Unloading", "Stoping Script", 10)
    HotKeySet("^+p")
    Sleep(2000)
    Exit
EndFunc

Func _ArrayElementUniques(ByRef $av_Array)
    If Not IsArray($av_Array) Then Return $av_Array
    Ubound($av_Array, 2)
    If Not @error Then Return $av_Array
    Local $i_Bound = Ubound($av_Array), $i_Count_A, $i_Count_B, $av_NewArray[$i_Bound + 1], $i_Number
    $av_NewArray[0] = 0
    For $i_Count_A = 0 To $i_Bound - 1
        For $i_Count_B = 0 To $i_Bound - 1
            If StringLeft($av_Array[$i_Count_A], 7) == StringLeft($av_Array[$i_Count_B], 7) And $i_Count_B <> $i_Count_A Then ContinueLoop(2)
        Next
        $av_NewArray[0] = $av_NewArray[0] + 1
        $av_NewArray[$av_NewArray[0]] = $av_Array[$i_Count_A]
    Next
    $i_Number = $av_NewArray[0]
    Redim $av_NewArray[$i_Number + 1]
    Return $av_NewArray
EndFunc

Is there an alternative to _FileListToArray because the number of files in my directory is way over 40,000 and the script was running for 2hrs and still not done.

I'm thinking of breaking the process down into sessions or something.

The goal is to have a list of files that are duplicates.

Example result (using my debug array):

1900124a.pdf

1900124b.pdf

1900128a.pdf

1900128b.pdf

All non-duplicates will be removed from the list. Any idea on how to make this work with 40,000 files?

Thanks in advance.