Jump to content
Sign in to follow this  
kor

trying to speed up script that takes 14 hours

Recommended Posts

kor

This script loops through users and compares an AD attribute to an ini file and puts them in an AD group or takes them out of AD groups based on that attribute. It has to loop through ~68,000 users comparing against about 83 different sites. It is taking about 14 hours to complete each pass. Wondering if there is any way to speed things up or gain some sense of parallelism.

#include <AD.au3>
#include <File.au3>
#include <Array.au3>

Global $iRunMode = 0 ;                                              ###
Global $iRunSection = 0 ;                                           ###

Global $sStudentQuery = "(&(objectCategory=person)(objectClass=user)(!userAccountControl:1.2.840.113556.1.4.803:=2)(company=stu)(!(!company=*)))", _ ; LDAP Query | person, user, not disabled, not student, company not null
$aUsers, $aDepartments, $aUserGroups, $sText, $sType, $aSchools, $aAdmins, $aStudents, $aInput, $aAllOtherAdmins, $iRunMode, $hLogFile

_AD_Open() ; open connection to Active Directory
_Students()
_AD_Close() ; close connection to Active Directory

Func _Students()
    ConsoleWrite("[INFO][BEGIN STUDENTS]")
    ConsoleWrite("[INFO][Running Student LDAP Query]")
    $aUsers = _AD_GetObjectsInOU("", $sStudentQuery, 2, "sAMAccountName,physicalDeliveryOfficeName,title", "") ; query AD (4.439s), not sorting to save 500ms
    ; $aUsers[$i][0] = sAMAccountName (username)
    ; $aUsers[$i][1] = physicalDeliveryOfficeName (site)
    ; $aUsers[$i][2] = title (grade)

    ConsoleWrite("[INFO][Reading Schools INI Section]")
    $aSchools = IniReadSection(@ScriptDir & "\GMAR-students-settings.ini", "Schools") ; read ini file for list of schools
    ; $aSchools[$i][0] = school number
    ; $aSchools[$i][1] = school group prefix

    ConsoleWrite("[INFO][Reading Student INI Section]")
    $aStudents = IniReadSection(@ScriptDir & "\GMAR-students-settings.ini", "Students") ; read ini file for student group name
    ; $aStudents[$i][0] = user type
    ; $aStudents[$i][1] = group name

    ConsoleWrite("[INFO][Running Add/Remove Checks]")
    ConsoleWrite("[VERBOSE] - Student Array contains " & UBound($aUsers) - 1 & " objects")
    ConsoleWrite("[VERBOSE] - School Array contains " & UBound($aSchools) - 1 & " objects")
    For $i = 1 To UBound($aUsers) - 1 ; loop for each user
        If Mod($i, 500) = 0 Then ConsoleWrite("Processed " & $i & " of " & UBound($aUsers) - 1 & " records") ; log entry at every 500 records
        ConsoleWrite("[VERBOSE] - User " & $i & " - " & $aUsers[$i][0])
        For $n = 1 to UBound($aSchools) - 1 ; loop for each school number
            If StringInStr($aUsers[$i][1], $aSchools[$n][0]) > 0 Then ; if users location matches school number
                If _AD_IsMemberOf($aSchools[$n][1] & " - " & $aStudents[1][1], $aUsers[$i][0]) <> 1 Then ; if user is not already a member of the group
                    ConsoleWrite("[ADD] - " & $aUsers[$i][0] & " - [TO] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
                    If $iRunMode < 2 Then
                        _AD_AddUserToGroup($aSchools[$n][1] & " - " & $aStudents[1][1], $aUsers[$i][0]) ; add user to group
                        If @error Then ConsoleWrite("[ERROR][ADD] - " & $aUsers[$i][0] & " - [TO] - " & $aSchools[$i][1] & " - " & $aStudents[1][1])
                    EndIf
                Else
                    ConsoleWrite("[VERBOSE][SKIP] - " & $aUsers[$i][0] & " - [MEMBER] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
                EndIf
            ElseIf StringInStr($aUsers[$i][1], $aSchools[$n][0]) = 0 Then ; if users location does not match school number
                If _AD_IsMemberOf($aSchools[$n][1] & " - " & $aStudents[1][1], $aUsers[$i][0]) = 1 Then ; if user is a member of the group
                    ConsoleWrite("[REMOVE] - " & $aUsers[$i][0] & " - [FROM] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
                    If $iRunMode < 2 Then
                        _AD_RemoveUserFromGroup($aSchools[$n][1] & " - " & $aStudents[1][1], $aUsers[$i][0]) ; remove user to group
                        If @error Then ConsoleWrite("[ERROR][REMOVE] - " & $aUsers[$i][0] & " - [FROM] - " & $aSchools[$i][1] & " - " & $aStudents[1][1])
                    EndIf
                Else
                    ConsoleWrite("[VERBOSE][SKIP] - " & $aUsers[$i][0] & " - [NOT MATCH] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
                EndIf
            EndIf
        Next
    Next
    ConsoleWrite("[INFO][END STUDENTS]")
EndFunc   ;==>_Students

 

example options in the INI it reads from

[schools]
101=Smith
103=Jersey
104=Washington
105=Jefferson
106=Newman

Edited by kor

Share this post


Link to post
Share on other sites
kor

I replaced them with consolewrite's for this example. It actually writes to a log file because I have to know what the script is doing. 

Share this post


Link to post
Share on other sites
jguinch

For $i = 1 To UBound($aUsers) - 1 ; loop for each user
    If Mod($i, 500) = 0 Then ConsoleWrite("Processed " & $i & " of " & UBound($aUsers) - 1 & " records") ; log entry at every 500 records
    ConsoleWrite("[VERBOSE] - User " & $i & " - " & $aUsers[$i][0])
    For $n = 1 to UBound($aSchools) - 1 ; loop for each school number

        If _AD_IsMemberOf($aSchools[$n][1] & " - " & $aStudents[1][1], $aUsers[$i][0]) <> 1 Then ; if user is already a member of the group
            If StringInStr($aUsers[$i][1], $aSchools[$n][0]) Then
                ConsoleWrite("[VERBOSE][SKIP] - " & $aUsers[$i][0] & " - [MEMBER] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
            ElseIf $iRunMode < 2 Then
                _AD_RemoveUserFromGroup($aSchools[$n][1] & " - " & $aStudents[1][1], $aUsers[$i][0]) ; remove user to group
                If @error Then ConsoleWrite("[ERROR][REMOVE] - " & $aUsers[$i][0] & " - [FROM] - " & $aSchools[$i][1] & " - " & $aStudents[1][1])
            EndIf
        Else ; if user is already a not member of the group
            If NOT StringInStr($aUsers[$i][1], $aSchools[$n][0]) Then
                ConsoleWrite("[VERBOSE][SKIP] - " & $aUsers[$i][0] & " - [NOT MATCH] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
            ElseIf $iRunMode < 2 Then
                ConsoleWrite("[ADD] - " & $aUsers[$i][0] & " - [TO] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
                _AD_AddUserToGroup($aSchools[$n][1] & " - " & $aStudents[1][1], $aUsers[$i][0]) ; add user to group
                If @error Then ConsoleWrite("[ERROR][ADD] - " & $aUsers[$i][0] & " - [TO] - " & $aSchools[$i][1] & " - " & $aStudents[1][1])
            EndIf
        EndIf
        
    Next
Next

With something like this, _AD_IsMemberOf will be use only one time, instead of two in your code.

Edit : I think you can skip the _AD_IsMemberOf, because _AD_AddUserToGroup and _AD_RemoveUserFromGroup already do thsi test.

Edit 2 : another way, without _AD_IsMemberOf :

If $iRunMode < 2 Then
    If StringInStr($aUsers[$i][1], $aSchools[$n][0]) Then 
        ConsoleWrite("[ADD] - " & $aUsers[$i][0] & " - [TO] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
        $iRet = _AD_AddUserToGroup($aSchools[$n][1] & " - " & $aStudents[1][1], $aUsers[$i][0]) ; add user to group
        If @error Then
            If $iRet = 3
                ConsoleWrite("[VERBOSE][SKIP] - " & $aUsers[$i][0] & " - [ALREADY MEMBER OF] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
            Else
                ConsoleWrite("[ERROR][ADD] - " & $aUsers[$i][0] & " - [TO] - " & $aSchools[$i][1] & " - " & $aStudents[1][1] & " (error code " & @error & ")")
            EndIf
        Else
            ConsoleWrite("[VERBOSE][OK] - " & $aUsers[$i][0] & " - [SUCCESSFULLY ADDED TO] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
        EndIf
    Else
        ConsoleWrite("[REMOVE] - " & $aUsers[$i][0] & " - [FROM] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
        $iRet = _AD_RemoveUserFromGroup($aSchools[$n][1] & " - " & $aStudents[1][1], $aUsers[$i][0]) ; remove user to group
        If @error Then
            If $iRet = 3
                ConsoleWrite("[VERBOSE][SKIP] - " & $aUsers[$i][0] & " - [NOT MEMBER OF] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
            Else
                ConsoleWrite("[ERROR][REMOVE] - " & $aUsers[$i][0] & " - [FROM] - " & $aSchools[$i][1] & " - " & $aStudents[1][1] & " (error code " & @error & ")")
            EndIf
        Else
            ConsoleWrite("[VERBOSE][OK] - " & $aUsers[$i][0] & " - SUCCESSFULLY REMOVED FROM] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
        EndIf
    EndIf
EndIf
Edited by jguinch

Share this post


Link to post
Share on other sites
water

If the attributes are replicated to the Global Catalog you could drop the queries to 83 sites and simply query the GC.


My UDFs and Tutorials:

Spoiler

UDFs:
Active Directory (NEW 2018-10-19 - Version 1.4.10.0) - Download - General Help & Support - Example Scripts - Wiki
OutlookEX (2018-09-01 - Version 1.3.4.0) - Download - General Help & Support - Example Scripts - Wiki
ExcelChart (2017-07-21 - Version 0.4.0.1) - Download - General Help & Support - Example Scripts
PowerPoint (2017-06-06 - Version 0.0.5.0) - Download - General Help & Support
Excel - Example Scripts - Wiki
Word - Wiki
 
Tutorials:

ADO - Wiki

 

Share this post


Link to post
Share on other sites
junkew

@OP: How do you write to your logfile?

filewrite("log.txt", <yourText>)

or do you have somewhere a

$hFileLog=fileopen("log.txt",$FO_OVERWRITE)
filewrite($hFileLog, <yourText>)

The latter filewrite is much faster

First putting everything in a string with concatenate & and then writing at once to disk will speed up

Share this post


Link to post
Share on other sites
kor

 

Edit : I think you can skip the _AD_IsMemberOf, because _AD_AddUserToGroup and _AD_RemoveUserFromGroup already do thsi test.

Edit 2 : another way, without _AD_IsMemberOf :

If $iRunMode < 2 Then
    If StringInStr($aUsers[$i][1], $aSchools[$n][0]) Then 
        ConsoleWrite("[ADD] - " & $aUsers[$i][0] & " - [TO] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
        $iRet = _AD_AddUserToGroup($aSchools[$n][1] & " - " & $aStudents[1][1], $aUsers[$i][0]) ; add user to group
        If @error Then
            If $iRet = 3
                ConsoleWrite("[VERBOSE][SKIP] - " & $aUsers[$i][0] & " - [ALREADY MEMBER OF] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
            Else
                ConsoleWrite("[ERROR][ADD] - " & $aUsers[$i][0] & " - [TO] - " & $aSchools[$i][1] & " - " & $aStudents[1][1] & " (error code " & @error & ")")
            EndIf
        Else
            ConsoleWrite("[VERBOSE][OK] - " & $aUsers[$i][0] & " - [SUCCESSFULLY ADDED TO] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
        EndIf
    Else
        ConsoleWrite("[REMOVE] - " & $aUsers[$i][0] & " - [FROM] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
        $iRet = _AD_RemoveUserFromGroup($aSchools[$n][1] & " - " & $aStudents[1][1], $aUsers[$i][0]) ; remove user to group
        If @error Then
            If $iRet = 3
                ConsoleWrite("[VERBOSE][SKIP] - " & $aUsers[$i][0] & " - [NOT MEMBER OF] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
            Else
                ConsoleWrite("[ERROR][REMOVE] - " & $aUsers[$i][0] & " - [FROM] - " & $aSchools[$i][1] & " - " & $aStudents[1][1] & " (error code " & @error & ")")
            EndIf
        Else
            ConsoleWrite("[VERBOSE][OK] - " & $aUsers[$i][0] & " - SUCCESSFULLY REMOVED FROM] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
        EndIf
    EndIf
EndIf

the only problem is, without a check first, I can't run my script it logging mode. I have a run mode that doesn't actually modify AD it just logs what it would do if it were going to modify. this way I can run it and go look at the log to see if the script is behaving as I expect

Share this post


Link to post
Share on other sites
jguinch

OK. Maybe something like this :

If StringInStr($aUsers[$i][1], $aSchools[$n][0]) Then 
    If $iRunMode < 2 Then
        If _AD_IsMemberOf($aSchools[$n][1] & " - " & $aStudents[1][1], $aUsers[$i][0]) Then
            ConsoleWrite("[VERBOSE][SKIP] - " & $aUsers[$i][0] & " - [ALREADY MEMBER OF] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
        Else
            ConsoleWrite("[VERBOSE][ADD] - " & $aUsers[$i][0] & " - [MUST BE ADDED TO] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
        EndIf
    Else
        ConsoleWrite("[ADD] - " & $aUsers[$i][0] & " - [TO] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
        $iRet = _AD_AddUserToGroup($aSchools[$n][1] & " - " & $aStudents[1][1], $aUsers[$i][0]) ; add user to group
        If @error Then
            If $iRet = 3
                ConsoleWrite("[VERBOSE][SKIP] - " & $aUsers[$i][0] & " - [ALREADY MEMBER OF] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
            Else
                ConsoleWrite("[ERROR][ADD] - " & $aUsers[$i][0] & " - [ADD TO] - " & $aSchools[$i][1] & " - " & $aStudents[1][1] & " (error code " & @error & ")")
            EndIf
        Else
            ConsoleWrite("[VERBOSE][OK] - " & $aUsers[$i][0] & " - [SUCCESSFULLY ADDED TO] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
        EndIf
    EndIf
Else
    If $iRunMode < 2 Then
        If _AD_IsMemberOf($aSchools[$n][1] & " - " & $aStudents[1][1], $aUsers[$i][0]) Then
            ConsoleWrite("[VERBOSE][REMOVE] - " & $aUsers[$i][0] & " - [MUST BE REMOVED FROM] - " & $aSchools[$i][1] & " - " & $aStudents[1][1] & " (error code " & @error & ")")
        Else
            ConsoleWrite("[VERBOSE][SKIP REMOVE] - " & $aUsers[$i][0] & " - [NOT MEMBER OF] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
        EndIf
    ELse
        $iRet = _AD_RemoveUserFromGroup($aSchools[$n][1] & " - " & $aStudents[1][1], $aUsers[$i][0]) ; remove user to group
        If @error Then
            If $iRet = 3
                ConsoleWrite("[VERBOSE][SKIP REMOVE] - " & $aUsers[$i][0] & " - [NOT MEMBER OF] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
            Else
                ConsoleWrite("[ERROR][REMOVE] - " & $aUsers[$i][0] & " - [REMOVE FROM] - " & $aSchools[$i][1] & " - " & $aStudents[1][1] & " (error code " & @error & ")")
            EndIf
        Else
            ConsoleWrite("[VERBOSE][OK] - " & $aUsers[$i][0] & " - SUCCESSFULLY REMOVED FROM] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
        EndIf
    EndIf
EndIf

Share this post


Link to post
Share on other sites

Create an account or sign in to comment

You need to be a member in order to leave a comment

Create an account

Sign up for a new account in our community. It's easy!

Register a new account

Sign in

Already have an account? Sign in here.

Sign In Now
Sign in to follow this  

×