How do i go about making a program that look on the screen for on image and then click on it?

[abrobot]

Im look for guidance on how to make a program. I want to make a program that checks the screen to see if it find a image that matches another image I provide it. then clicks that area on the screen where it found the image.

I am fairly new to autoit and I'm not quite sure how to go about this. Any help would be really appreciated. im not looking for someone to write me code or anything just a push in the right direction.

[Jos]

Welcome,

So you think this is a question that wasn't asked before so I guess your search didn't return any hits?  

For starters: Be xplicit what it is you want to do and what you have tried thus far! 

Jos

Edited January 29, 2021 by Jos

[robertocm]

1 hour ago, abrobot said:

just a push in the right direction

I'm using this:

An example working with remote desktop and Windows Task Scheduler:

expandcollapse popup

#include <File.au3>
#include "..\OpenCV\OpenCV_Match\OpenCV-Match_UDF.au3"

#include <Timers.au3>
#include <WinAPIFiles.au3>

;Permitir unha única instancia da aplicación
#include <Misc.au3>

;_Singleton("test_OpenCV", 0)
If _Singleton("test_OpenCV", 1) = 0 Then
   Msgbox(48,"Atención", "Ya se está ejecutando." & @LF & "Esta instancia se cerrará.", 5)
   Exit
EndIf

Opt('WinWaitDelay', 250)
Opt('WinDetectHiddenText', 1)
Opt('MouseCoordMode', 0)
Opt("WinTitleMatchMode", 1) ;1=start, 2=subStr, 3=exact, 4=advanced, -1 to -4=Nocase
Opt("SendKeyDelay", 75)
Opt("SendKeyDownDelay", Default)
Opt("SendCapslockMode", 1)

Opt("MustDeclareVars", 1)
Opt("TrayIconDebug", 1)
;Be sure to register this to reenable power-saving, screensaver, etc
OnAutoItExitRegister("OnAutoItExit")

#===== CONFIG =====
Global $sOpenCVDir = "C:\Users\..MyPath....\Scripts\OpenCV\OpenCV_Match"
Global $hFile = FileOpen(@DesktopDir & "\Events.log", 2)
Global $sFilePath = "\\XX.X.X.XX\SharedFolder\switch"

#===== OpenCV-Match_UDF =====
;From Windows Task Sheduler: avoid error Failed loading DLLs
FileChangeDir($sOpenCVDir)
_OpenCV_Startup();loads opencv DLLs
_OpenCV_EnableLogging(True,True,True) ;Logs matches, errors in a log file and autoit console output.

;https://stackoverflow.com/questions/3867584/autoit-how-to-get-system-idle-time-or-if-screensaver-is-active
;How to get system idle time, or if screensaver is active?
;The _Timer_GetIdleTime() function uses GetLastInputInfo from user32.dll.
Global $Sair, $iLimit = 120 ; idle limit in seconds  (5 segundos)
If _Timer_GetIdleTime() > $iLimit * 1000 Then
   ;MsgBox(0, "", "Continue >>")
Else
   AdlibRegister("_CheckIdleTime", 1000)
   While 1 ;use infinite loop since ExitLoop will get called
      If $Sair = True Then
         ;ConsoleWrite("Fin")
         ExitLoop
      EndIf
      Sleep(1000)
   WEnd
   ;Outra forma
   ;Do
      ;Sleep(1000)
   ;Until $Sair = True
EndIf

Global $iFileExists
For $i = 0 To 30
   If FileExists("\\XX.X.X.XX\SharedFolder") Then
      ExitLoop
   Else
      If $i < 30 Then
         Sleep(5000)
      Else
         _FileWriteLog($hFile, "Error Acceso")
         Exit
      EndIf
   EndIf
Next

;from pc in suspension mode
Send("{DOWN}")
Sleep(20000)
MouseMove(@DesktopWidth - 100, @DesktopHeight -  100)

If WinExists("XX.X.X.XX - Conexión a Escritorio remoto") Then
   ProcessClose("mstsc.exe")
   Sleep(2000)
EndIf

;https://www.autoitscript.com/forum/topic/97501-auto-mstsc/
;RunWait('"' & @ComSpec & '" /c MSTSC.EXE C:/yourfile.rdp', '', @SW_HIDE)
;http://support.microsoft.com/kb/941641
Run("C:\windows\system32\mstsc.exe /v:" & "XX.X.X.XX")
;RunWait("C:\windows\system32\mstsc.exe /v:" & "10.0.0.126")

;Wait 90 seconds for the window to appear.
Global $hWnd = WinWait("XX.X.X.XX - Conexión a Escritorio remoto", "", 90)
;WinActivate($hWnd)
;Global $hWnd = WinActivate("XX.X.X.XX - Conexión a Escritorio remoto")
;WinWaitActive($hWnd, "", 90)
;WinSetState($hWnd, "", @SW_MAXIMIZE)
;Sleep(3000)
;MouseClick($MOUSE_CLICK_LEFT, 1533, 26)
Sleep(5000)

;Test if the window is now activated
;You can use the WinActive() function to check if WinActivate() succeeded.
If WinActive($hWnd, "") Then
   ;WinSetState($hWnd, "", @SW_MAXIMIZE)
   ;MsgBox(0, "", "Window NOW active")

   ;clean
   FileRecycle($sFilePath)

   ;Image of Desktop in remote
   Sleep(5000)
   Global $sCoords[4] = [1412, 644, 1420, 651]
   Global $Match1 = _MatchPicture($sOpenCVDir & "\Match\000.png", 0.70, $sCoords, 120, 1000);Try to find the match picture on the screen. Number of tries: 10, Sleep between each try: 500ms.
   If Not @error Then
      _MarkMatch($Match1) ;Debugging: Draws a rect on the screen/coordinates of the match to show the user where the match was found
      Sleep(100)
      ;_ClickMouse($Match1, "left",1) ;Calculates the center of the match and clicks the left mouse once on click position
      ;Sleep(1000)
   Else
      Exit
   EndIf
   Sleep(10000)

   ;https://stackoverflow.com/questions/26505329/autoit-controlsend-remote-desktop
   If ControlSend($hWnd,"", "", "#r") = 1 Then
      _FileWriteLog($hFile, "OK ControlSend: #r")
      Sleep(2000)
      If ControlSend($hWnd, "", "", "C:\SomePath\SomeProgram.EXE") = 1 Then
         _FileWriteLog($hFile, "OK ControlSend: command")
         Sleep(2000)
         ControlSend($hWnd, "", "", "{ENTER}")
      Else
         _FileWriteLog($hFile, "Error ControlSend: command")
         Exit
      EndIf
   Else
      _FileWriteLog($hFile, "Error ControlSend: #r")
      Exit
   EndIf

   Sleep(4000)

   If FileExists($sFilePath) Then
      _FileWriteLog($hFile, "OK switch file")

      ;Please note that these examples might not work as the match pictures have to be found with the exact same size on your screen.
      Global $sCoords[4] = [771, 471, 779, 482]
      Global $Match1 = _MatchPicture($sOpenCVDir & "\Match\a3\01.png", 0.70, $sCoords, 30, 1000);Try to find the match picture on the screen. Number of tries: 10, Sleep between each try: 500ms.
      If Not @error Then
         _MarkMatch($Match1) ;Debugging: Draws a rect on the screen/coordinates of the match to show the user where the match was found
         Sleep(100)

         If ControlSend($hWnd, "", "", "SomeText") = 1 Then
            _FileWriteLog($hFile, "OK ControlSend: SomeText")
         Else
            _FileWriteLog($hFile, "Error ControlSend: SomeText")
            Exit
         EndIf
         Sleep(1000)

         _ClickMouse($Match1, "left",1) ;Calculates the center of the match and clicks the left mouse once on click position
         Sleep(1000)
      Else
         Exit
      EndIf

      Global $sCoords[4] = [269, 91, 277, 102]
      Global $Match1 = _MatchPicture($sOpenCVDir & "\Match\a3\02.png", 0.70, $sCoords, 30, 1000);Try to find the match picture on the screen. Number of tries: 10, Sleep between each try: 500ms.
      If Not @error Then
         _MarkMatch($Match1) ;Debugging: Draws a rect on the screen/coordinates of the match to show the user where the match was found
         Sleep(100)

         For $i = 0 To 1
            If ControlSend($hWnd, "", "", "SomeText") = 1 Then
               Sleep(500)
               If ControlSend($hWnd, "", "", "{TAB}") = 1 Then
                  Sleep(200)
                  _FileWriteLog($hFile, "OK ControlSend: {TAB}")
               Else
                  _FileWriteLog($hFile, "Error ControlSend: {TAB}")
                  Exit
               EndIf
               _FileWriteLog($hFile, "OK ControlSend: SomeText")
            Else
               _FileWriteLog($hFile, "Error ControlSend: SomeText")
               Exit
            EndIf
         Next
         Sleep(1000)

         _ClickMouse($Match1, "left",1) ;Calculates the center of the match and clicks the left mouse once on click position
         Sleep(1000)
      Else
         Exit
      EndIf

      If WinActive($hWnd, "") Then
         MouseClick("left", 118, 180, 10)
         _FileWriteLog($hFile, "Click")
         Sleep(1000)
      EndIf

      ;....

      While 1
         If Not WinExists("XX.X.X.XX - Conexión a Escritorio remoto") Then
            _FileWriteLog($hFile, "Error Not WinExists: XX.X.X.XX")
            ExitLoop
         Else
            If FileExists($sFilePath) Then
               If Not WinActive($hWnd, "") Then
                  If _Timer_GetIdleTime() > $iLimit * 1000 Then
                     WinActivate($hWnd)
                     WinWaitActive($hWnd, "", 10)
                     WinSetState($hWnd, "", @SW_MAXIMIZE)
                  EndIf
               EndIf
               Sleep(25000)
            Else
               _FileWriteLog($hFile, "Deleted switch file")
               ExitLoop
            EndIf
         EndIf
      WEnd
   Else
      _FileWriteLog($hFile, "ERROR: Not found switch file")
      Exit
   EndIf

   If WinActive($hWnd, "") Then
      ;https://stackoverflow.com/questions/26505329/autoit-controlsend-remote-desktop
      If ControlSend($hWnd,"", "", "#r") = 1 Then
         _FileWriteLog($hFile, "OK ControlSend: #r")
         Sleep(2000)
         If ControlSend($hWnd, "", "", "TASKKILL /F /IM SomeProgram.EXE") = 1 Then
            _FileWriteLog($hFile, "OK ControlSend: TASKKILL")
            Sleep(2000)
            ControlSend($hWnd, "", "", "{ENTER}")
         Else
            _FileWriteLog($hFile, "Error ControlSend: TASKKILL")
            Exit
         EndIf
      Else
         _FileWriteLog($hFile, "Error ControlSend: #r")
         Exit
      EndIf
      Sleep(2000)

      ;https://stackoverflow.com/questions/26505329/autoit-controlsend-remote-desktop
      If ControlSend($hWnd,"", "", "#r") = 1 Then
         _FileWriteLog($hFile, "OK ControlSend: #r")
         Sleep(2000)
         If ControlSend($hWnd, "", "", "shutdown -L") = 1 Then
            _FileWriteLog($hFile, "OK ControlSend: shutdown -L")
            Sleep(2000)
            ControlSend($hWnd, "", "", "{ENTER}")
         Else
            _FileWriteLog($hFile, "Error ControlSend: shutdown -L")
            Exit
         EndIf
      Else
         _FileWriteLog($hFile, "Error ControlSend: #r")
         Exit
      EndIf
      Sleep(2000)
   EndIf
Else
   ;MsgBox(0, "Error", "Window not active")
   ;Local $hActive = WinGetHandle("[active]")
   ;Local $sTitle = WinGetTitle("[active]")
   ;MsgBox(0, "Error", $sTitle)

   ;Open the file for write access.
   Global $hFileOpen = FileOpen(@DesktopDir & "\RDP_ERROR.txt", 2)
   ;If $hFileOpen = -1 Then
    ;MsgBox(0, "", "An error occurred when reading/writing the file.")
   ;EndIf
   ;Write some data.
   FileWrite($hFileOpen, "ERROR")
   ;Close the handle returned by FileOpen.
   FileClose($hFileOpen)
EndIf

Func _CheckIdleTime()
   If _Timer_GetIdleTime() > $iLimit * 1000 Then
      ;AdLib functions don't run while a blocking function is shown e.g. MsgBox, InputBox, WinWait, WinWaitClose etc.
      ;MsgBox(16, "Timeout", "You haven't done anything in " & $iLimit & " seconds...  Get busy!")
      $Sair = True
      AdlibUnRegister("_CheckIdleTime")
   EndIf
EndFunc

Func OnAutoItExit()
   _OpenCV_Shutdown();Closes DLLs
   FileClose($hFile)
EndFunc

 

Edited January 29, 2021 by robertocm

[junkew]

Read the wiki and faq 31 and just start on the first pages of the help file