SmartNote

ioa747 · July 31, 2022

SmartNote is a screen snip tool to snip screen area with OCR ability from tesseract 100+ languages

I did some upgrades

can work on multiple monitors
I changed the hotkey so it can grab menus and context menus
I changed the environment to OnEvent Mode for smoother operation
Add entry to send picture to ShareX image editor

I tried it in Windows 10 with the version tesseract-ocr-w32-setup-v5.3.0.20221214.exe

Snipe new note

While Right Windows Key IsPressed, tap 2 times the Right Shift key to start NewCapture,
is useful when Capture menu or context.
While start NewCapture point click anywhere to escape. (Does not catch smaller than 9*9 px)
While start NewCapture escape with ESC

Spoiler

Tesseract info

Of course, you will have to install it first. tesseract-installer-for-windows

Languages supported https://tesseract-ocr.github.io/tessdoc/Data-Files-in-different-versions.html

More info Command-Line-Usage

SmartNote.au3

; https://www.autoitscript.com/forum/topic/208600-smartnote/
;----------------------------------------------------------------------------------------
; Title...........: SmartNote.au3
; Description.....: SmartNote is a screen snip tool to take Screenshot with OCR ability from Tesseract (100+ languages)
; AutoIt Version..: 3.3.16.1   Author: ioa747
;----------------------------------------------------------------------------------------
#Region ;**** Directives created by AutoIt3Wrapper_GUI ****
#AutoIt3Wrapper_Icon=mmcndmgr-106.ico
#AutoIt3Wrapper_Res_Description=SmartNote is a screen snip tool to take Screenshot with OCR ability from Tesseract (100+ languages)
#AutoIt3Wrapper_Res_Fileversion=0.0.4.20230924
#AutoIt3Wrapper_Res_ProductName=SmartNote.au3
#AutoIt3Wrapper_AU3Check_Parameters=-d -w 1 -w 2 -w 3 -w 4 -w 5 -w 6 -w 7
#EndRegion ;**** Directives created by AutoIt3Wrapper_GUI ****

#include <MsgBoxConstants.au3>
#include <ScreenCapture.au3>
#include <GUIConstantsEx.au3>
#include <WindowsConstants.au3>
#include <StaticConstants.au3>
#include <TrayConstants.au3>
#include <Misc.au3>
#include <Array.au3>
#include <GDIPlus.au3>
#include <Clipboard.au3>
#include <String.au3>

; Initialization
DllCall("user32.dll", "bool", "SetProcessDpiAwarenessContext", "int", -4) ; -4=PerMonitorAwareV2

Opt("MouseCoordMode", 1)  ; 1=absolute, 0=relative, 2=client
Opt("TrayMenuMode", 3)    ; 0=append, 1=no default menu, 2=no automatic check, 4=menuitemID  not return
Opt("TrayOnEventMode", 1) ; Enable TrayOnEventMode.
Opt("GUIOnEventMode", 1)  ; 0=disabled, 1=OnEvent mode enabled

; tray entry to exit script
Global $TRAY_ExitScript = TrayCreateItem("Exit")
TrayItemSetOnEvent(-1, "TRAY_EVENT")
; separator
TrayCreateItem("")
; tray entry to capture
Global $TRAY_NewCapture = TrayCreateItem("Capture")
TrayItemSetOnEvent(-1, "TRAY_EVENT")

TraySetOnEvent($TRAY_EVENT_PRIMARYDOUBLE, "TRAY_EVENT")
TraySetIcon("mmcndmgr.dll", -106)
TraySetState($TRAY_ICONSTATE_SHOW)
TraySetToolTip("SmartNote" & @CRLF & "Double Click to take new note")

Global $mPos, $aRecPos[4], $hGUICapture, $block_gui
Global $hDLL = DllOpen("user32.dll")
; array to hold the Contextmenu data
Global $NoteGui[1][12]

$NoteGui[0][0] = 0 ; cnt, Note_Handle
$NoteGui[0][1] = "jpgPath"
$NoteGui[0][2] = "Gui_Size"
$NoteGui[0][3] = "ID_PicCtrl"
$NoteGui[0][4] = "ID_ContextMenu"
$NoteGui[0][5] = "ID_cMenuCopy"
$NoteGui[0][6] = "ID_cMenuShareX"
$NoteGui[0][7] = "ID_cMenuSave"
$NoteGui[0][8] = "ID_cMenuOpen"
$NoteGui[0][9] = "ID_cMenuOCR-eng"
$NoteGui[0][10] = "ID_cMenuOCR-eng+ell"
$NoteGui[0][11] = "ID_cMenuClose"

; Check for leftover files
Global $iFileExists = FileExists(@ScriptDir & "\tmp")
If $iFileExists Then
    FileDelete(@ScriptDir & "\tmp\*.*")
Else
    DirCreate(@ScriptDir & "\tmp")
EndIf

; ℹ️  While Right Windows Key IsPressed, tap 2 times the Right SHIFT key to start NewCapture,
;     is useful when Capture menu or contex.
; While start NewCapture point click anywhere to escape. (does not catch smaller than 9*9 px)
; While start NewCapture escape with ESC

Global $iShift

; loop until program exit
;************************************************
While Sleep(100)
    $iShift = 0
    While _IsPressed("5C", $hDLL) ; 5C = Right Windows key
        Sleep(100)
        If _IsPressed("A1", $hDLL) Then ; A1 = Right SHIFT key
            While _IsPressed("A1", $hDLL)
                Sleep(100)
            WEnd
            $iShift += 1
        EndIf

        If $iShift = 2 Then
            NewCapture()
            ExitLoop
        EndIf
    WEnd
WEnd
;************************************************

;----------------------------------------------------------------------------------------
Func NewCapture()                       ; NewCapture <<- HotKey WIN /

    Local $aRecPos[4], $aMPos[2], $tPos ;, $aTipPos[4], $iX, $iY
    Local $iDeskWidth, $iDeskHeight, $iDeskLeft, $iDeskTop
    Local $sDevice, $hMonitor, $sCurDevice, $aData, $Status = 0

    ; make Capture gui
    $hGUICapture = GUICreate("Capture_gui", 1, 1, 1, 1, $WS_POPUP, BitOR($WS_EX_TOOLWINDOW, $WS_EX_TOPMOST))
    GUISetBkColor("0xFFFF00", $hGUICapture) ; $COLOR_YELLOW
    WinSetTrans($hGUICapture, "", 50)

    ; make mouse block gui
    $block_gui = GUICreate("block_gui", 1, 1, 1, 1, $WS_POPUP, BitOR($WS_EX_TOOLWINDOW, $WS_EX_TOPMOST))
    WinSetTrans($block_gui, "", 1)
    GUISetState(@SW_SHOW, $block_gui)
    GUISetCursor($MCID_CROSS, 1, $block_gui)

    Sleep(200)

    Local $iMaxLoop = 1200, $iCntLoop = 0

    While Sleep(10)
        $iCntLoop += 1
        If $iCntLoop = $iMaxLoop Then ExitLoop

        ; get mouse coordinates
        $tPos = _WinAPI_GetMousePos()
        $aMPos[0] = DllStructGetData($tPos, 1)
        $aMPos[1] = DllStructGetData($tPos, 2)

        ; get $hMonitor from previously defined Mouse coordinates
        $hMonitor = _WinAPI_MonitorFromPoint($tPos)

        ; get monitor $aData appropriate for previously defined coordinates
        $aData = _WinAPI_GetMonitorInfo($hMonitor)
        If Not @error Then
            $sDevice = $aData[3]
            $iDeskLeft = DllStructGetData($aData[0], 1)
            $iDeskTop = DllStructGetData($aData[0], 2)
            $iDeskWidth = DllStructGetData($aData[0], 3)
            $iDeskHeight = DllStructGetData($aData[0], 4)
        EndIf

        ;move the $block_gui to active monitor
        If $sCurDevice <> $sDevice Then
            $sCurDevice = $sDevice
            ;ConsoleWrite("- $sCurDevice=" & $sCurDevice & @CRLF)
            WinMove($block_gui, "", $iDeskLeft, $iDeskTop, $iDeskWidth, $iDeskHeight)
        EndIf

        ; whait  Left_mouse_button _IsPressed
        If _IsPressed("01", $hDLL) Then
            $Status = 1
            $aMPos = MouseGetPos()
            $aRecPos[0] = $aMPos[0]
            $aRecPos[1] = $aMPos[1]

            ; Wait until key is released.
            While _IsPressed("01", $hDLL)
                Sleep(50)
                $aMPos = MouseGetPos()
                $aRecPos[2] = $aMPos[0]
                $aRecPos[3] = $aMPos[1]

                ; show Capture gui
                GUISetState(@SW_SHOW, $hGUICapture)
                WinMove($hGUICapture, "", $aRecPos[0], $aRecPos[1], $aRecPos[2] - $aRecPos[0], $aRecPos[3] - $aRecPos[1])
            WEnd
        ElseIf _IsPressed("1B", $hDLL) Then ;1B=ESC key - emergency exit
            GUIDelete($hGUICapture)
            GUIDelete($block_gui)
            Return SetError(1, 1, 0)
        EndIf

        If $Status = 1 Then ExitLoop
    WEnd

    GUIDelete($hGUICapture)
    GUIDelete($block_gui)

    ;ConsoleWrite($aRecPos[0] & ";" & $aRecPos[1] & ";" & $aRecPos[2] - $aRecPos[0] & ";" & $aRecPos[3] - $aRecPos[1] & @CRLF)

    ; if bigger from 9*9 px then create note
    If ($aRecPos[2] - $aRecPos[0]) > 9 And ($aRecPos[3] - $aRecPos[1]) > 9 Then CreateNew_NoteGui($aRecPos)  ; create new note

    ;Return $FilePath

EndFunc   ;==>NewCapture
;----------------------------------------------------------------------------------------
Func CreateNew_NoteGui($aRecPos)        ; create new note gui

    Local $n, $aSize[2]

    ReDim $NoteGui[UBound($NoteGui) + 1][12]
    $NoteGui[0][0] += 1

    $n = $NoteGui[0][0]

    $aSize[0] = $aRecPos[2] - $aRecPos[0] ; width
    $aSize[1] = $aRecPos[3] - $aRecPos[1] ; height

    ; create note GUI
    $NoteGui[$n][0] = GUICreate($NoteGui[$n][1], $aSize[0], $aSize[1], $aRecPos[0], $aRecPos[1], $WS_POPUP, BitOR($WS_EX_TOOLWINDOW, $WS_EX_TOPMOST))
    GUISetOnEvent($GUI_EVENT_RESIZED, "GUI_EVENT", $NoteGui[$n][0])

    ; jpg Path for _ScreenCapture_Capture
    $NoteGui[$n][1] = @ScriptDir & "\tmp\image_" & $NoteGui[$n][0] & ".jpg"
    _ScreenCapture_Capture($NoteGui[$n][1], $aRecPos[0], $aRecPos[1], $aRecPos[2], $aRecPos[3])

    ; save the Gui_Size
    $NoteGui[$n][2] = $aSize

    ; set jpg Path as GUI title
    WinSetTitle($NoteGui[$n][0], "", $NoteGui[$n][1])

    ; Creates a Picture control
    $NoteGui[$n][3] = GUICtrlCreatePic($NoteGui[$n][1], 0, 0, 0, 0, -1, $GUI_WS_EX_PARENTDRAG)

    ; Creates a Label control just for $SS_GRAYFRAME
    GUICtrlCreateLabel("", 0, 0, $aSize[0], $aSize[1], $SS_GRAYFRAME)

    ; Creates a context menu
    $NoteGui[$n][4] = GUICtrlCreateContextMenu($NoteGui[$n][3])

    ; MenuItem for the context menu "Copy to Clipboard"
    $NoteGui[$n][5] = GUICtrlCreateMenuItem("Copy to Clipboard", $NoteGui[$n][4])
    GUICtrlSetOnEvent(-1, "cm_CopyToClipboard")

    ; MenuItem for the context menu "ID_cMenuShareX"
    $NoteGui[$n][6] = GUICtrlCreateMenuItem("Send to ShareX", $NoteGui[$n][4])
    GUICtrlSetOnEvent(-1, "cm_ShareX")

    ; MenuItem for the context menu "Save as..."
    $NoteGui[$n][7] = GUICtrlCreateMenuItem("Save as...", $NoteGui[$n][4])
    GUICtrlSetOnEvent(-1, "cm_SaveAs")

    ; MenuItem for the context menu "Open"
    $NoteGui[$n][8] = GUICtrlCreateMenuItem("Open", $NoteGui[$n][4])
    GUICtrlSetOnEvent(-1, "cm_Open")

    ; separator
    GUICtrlCreateMenuItem("", $NoteGui[$n][4])

    ; MenuItem for the context menu "OCR - eng"
    $NoteGui[$n][9] = GUICtrlCreateMenuItem("OCR - eng", $NoteGui[$n][4])
    GUICtrlSetOnEvent(-1, "cm_OCR_Eng")

    ; MenuItem for the context menu "OCR - eng+ell"
    $NoteGui[$n][10] = GUICtrlCreateMenuItem("OCR - eng+ell", $NoteGui[$n][4])
    GUICtrlSetOnEvent(-1, "cm_OCR_EngEll")

    ; separator
    GUICtrlCreateMenuItem("", $NoteGui[$n][4])

    ; MenuItem for the context menu "Close"
    $NoteGui[$n][11] = GUICtrlCreateMenuItem("Close", $NoteGui[$n][4])
    GUICtrlSetOnEvent(-1, "cm_Close")

    ; Display the GUI.
    GUISetState(@SW_SHOW, $NoteGui[$n][0])

    Return $NoteGui[$n][0]

EndFunc   ;==>CreateNew_NoteGui
;----------------------------------------------------------------------------------------
Func SaveFileDlg($Active_title)         ; Save file Dialog
    ; Create a constant variable in Local scope of the message to display in FileSaveDialog.
    Local Const $sMessage = "Choose a filename."

    ; Display a save dialog to select a file.
    Local $sFileSaveDialog = FileSaveDialog($sMessage, @ScriptDir & "\SET\", "image (*.jpg)", $FD_PATHMUSTEXIST)
    If @error Then
        ; Display the error message.
        MsgBox($MB_SYSTEMMODAL, "", "No file was saved.")
    Else
        ; Retrieve the filename from the filepath e.g. Example.jpg
        Local $sFileName = StringTrimLeft($sFileSaveDialog, StringInStr($sFileSaveDialog, "\", $STR_NOCASESENSEBASIC, -1))

        ; Check if the extension .jpg is appended to the end of the filename.
        Local $iExtension = StringInStr($sFileName, ".", $STR_NOCASESENSEBASIC)

        ; If a period (dot) is found then check whether or not the extension is equal to .jpg
        If $iExtension Then
            ; If the extension isn't equal to .jpg then append to the end of the filepath.
            If Not (StringTrimLeft($sFileName, $iExtension - 1) = ".jpg") Then $sFileSaveDialog &= ".jpg"
        Else
            ; If no period (dot) was found then append to the end of the file.
            $sFileSaveDialog &= ".jpg"
        EndIf

        ; Display the saved file.
        ;ConsoleWrite("You saved the following file:" & @CRLF & $sFileSaveDialog & @CRLF)

        FileCopy($Active_title, $sFileSaveDialog, $FC_OVERWRITE + $FC_CREATEPATH)

    EndIf
EndFunc   ;==>SaveFileDlg
;----------------------------------------------------------------------------------------
Func NoteGui_Delete($hWnd)              ; NoteGui_Delete
    For $i = 1 To $NoteGui[0][0]
        If $NoteGui[$i][0] = $hWnd Then
            _ArrayDelete($NoteGui, $i)
            $NoteGui[0][0] -= 1
            ExitLoop
        EndIf
    Next
EndFunc   ;==>NoteGui_Delete
;----------------------------------------------------------------------------------------
Func PicToClip($Path)                   ; put image to clipboard (Thanks to @Nine)
    _GDIPlus_Startup()
    Local $hImage = _GDIPlus_ImageLoadFromFile($Path)
    Local $hBitmap1 = _GDIPlus_BitmapCreateHBITMAPFromBitmap($hImage)
    _GDIPlus_ImageDispose($hImage)

    Local $hBitmap2 = _WinAPI_CopyImage($hBitmap1, $IMAGE_BITMAP, 0, 0, $LR_COPYDELETEORG + $LR_COPYRETURNORG)
    _WinAPI_DeleteObject($hBitmap1)
    _GDIPlus_Shutdown()

    _ClipBoard_Open(0)
    _ClipBoard_Empty()
    _ClipBoard_SetDataEx($hBitmap2, $CF_BITMAP)
    _ClipBoard_Close()
    _WinAPI_DeleteObject($hBitmap2)

EndFunc   ;==>PicToClip
;----------------------------------------------------------------------------------------
Func TesseracOCR($ImagePath, $lang)     ; perform ocr (Thanks to JohnOne)

;~ ; $lang = more LangCode  ->  https://tesseract-ocr.github.io/tessdoc/Data-Files-in-different-versions.html
;~ ; more info at  ->  https://tesseract-ocr.github.io/tessdoc/Command-Line-Usage.html#simplest-invocation-to-ocr-an-image
;~ ; Of course you will have to install it first. -> https://github.com/UB-Mannheim/tesseract/wiki
;~ ; I tried it in Windows 10 with the version ->  https://digi.bib.uni-mannheim.de/tesseract/tesseract-ocr-w32-setup-v5.3.0.20221214.exe

    Local $ResultTextPath, $OutPutPath, $Result

    ;C:\Program Files (x86)\Tesseract-OCR\tesseract.exe
    Local Const $TesseractExePath = @ProgramFilesDir & "\Tesseract-OCR\tesseract.exe" ; ℹ️ Give your path * <-

    $ResultTextPath = @ScriptDir & "\tmp\Result"
    $OutPutPath = $ResultTextPath & ".txt"

    ShellExecuteWait($TesseractExePath, '"' & $ImagePath & '" "' & $ResultTextPath & '" -l ' & $lang & ' --psm 6', "", "", @SW_HIDE)
    ;ConsoleWrite($TesseractExePath & ' "' & $ImagePath & '" "' & $ResultTextPath & '" -l ' & $lang & ' --psm 6' & @CRLF)

    If @error Then
        Exit MsgBox($MB_OK + $MB_TOPMOST + $MB_ICONERROR, "Error", @error)
    EndIf

    $Result = FileRead($OutPutPath)

    Local $Msg = @CRLF & @CRLF & _StringRepeat(@TAB, 4) & "Copy to ClipBoard ?"
    Local $iMsgBoxAnswer = MsgBox(4, "OCR Result", $Result & $Msg)
    If $iMsgBoxAnswer = 6 Then ;Yes
        ClipPut($Result)
        ToolTip("The text was copied to the clipboard")
        Sleep(1000)
        ToolTip("")
    EndIf
    FileDelete($OutPutPath)
EndFunc   ;==>TesseracOCR
;----------------------------------------------------------------------------------------
Func TRAY_EVENT()         ; TRAY_Event
    Switch @TRAY_ID ; Check the last tray item identifier.
        Case $TRAY_EVENT_PRIMARYDOUBLE
            NewCapture()

        Case $TRAY_NewCapture
            NewCapture()

        Case $TRAY_ExitScript
            DllClose($hDLL)
            Exit

    EndSwitch
EndFunc   ;==>TRAY_EVENT
;----------------------------------------------------------------------------------------
Func GUI_EVENT()          ; GUI_EVENT
    Select
        Case @GUI_CtrlId = $GUI_EVENT_RESIZED
            Local $aSize
            For $i = 1 To $NoteGui[0][0]
                If $NoteGui[$i][0] = @GUI_WinHandle Then
                    $aSize = $NoteGui[$i][2]
                    WinMove(@GUI_WinHandle, "", Default, Default, $aSize[0], $aSize[1])
                    ControlMove(@GUI_WinHandle, "", "Static1", 0, 0, $aSize[0], $aSize[1])
                    ControlMove(@GUI_WinHandle, "", "Static2", 0, 0, $aSize[0], $aSize[1])
                    ExitLoop
                EndIf
            Next
    EndSelect
EndFunc   ;==>GUI_EVENT
;----------------------------------------------------------------------------------------
Func cm_CopyToClipboard() ; ContextMenu  "Copy to Clipboard"
    Local $Active_title = WinGetTitle(@GUI_WinHandle)
    PicToClip($Active_title)
EndFunc   ;==>cm_CopyToClipboard
;----------------------------------------------------------------------------------------
Func cm_ShareX()          ; ContextMenu  "Send to ShareX""
    Local $Active_title = WinGetTitle(@GUI_WinHandle)
    PicToClip($Active_title)
    Sleep(100)
    Local $strLen = StringLen(@ScriptDir & "\tmp\image_")
    If StringLeft($Active_title, $strLen) = @ScriptDir & "\tmp\image_" Then
        GUIDelete(@GUI_WinHandle)
        FileDelete($Active_title)
        NoteGui_Delete(@GUI_WinHandle)
    EndIf

    ShellExecute("C:\Program Files\ShareX\ShareX.exe", "-imageeditor ") ; ℹ️ Give your path * <-
    Local $hShareXImageEd = WinWait("[TITLE:ShareX - Image editor; REGEXPCLASS:WindowsForms10\.Window\..\.app\.0\..+_r\d+_ad1]", "", 2)
    While WinExists($hShareXImageEd)
        ControlClick($hShareXImageEd, "", "[NAME:btnLoadImageFromClipboard]")
        Sleep(300)
    WEnd
    $hShareXImageEd = WinWait("[TITLE:ShareX - Image editor; REGEXPCLASS:WindowsForms10\.Window\..\.app\.0\..+_r\d+_ad1]", "", 2)
    WinActivate($hShareXImageEd)
    WinSetState($hShareXImageEd, "", @SW_MAXIMIZE)

EndFunc   ;==>cm_ShareX
;----------------------------------------------------------------------------------------
Func cm_SaveAs()          ; ContextMenu  "Save as..."
    Local $Active_title = WinGetTitle(@GUI_WinHandle)
    SaveFileDlg($Active_title)
EndFunc   ;==>cm_SaveAs
;----------------------------------------------------------------------------------------
Func cm_Open()            ; ContextMenu  "Open"
    Local $Active_title = WinGetTitle(@GUI_WinHandle)
    ShellExecute($Active_title)
EndFunc   ;==>cm_Open
;----------------------------------------------------------------------------------------
Func cm_OCR_Eng()         ; ContextMenu  "OCR - eng"
    Local $Active_title = WinGetTitle(@GUI_WinHandle)
    TesseracOCR($Active_title, "eng")
EndFunc   ;==>cm_OCR_Eng
;----------------------------------------------------------------------------------------
Func cm_OCR_EngEll()      ; ContextMenu  "OCR - eng+ell"
    Local $Active_title = WinGetTitle(@GUI_WinHandle)
    TesseracOCR($Active_title, "eng+ell")
EndFunc   ;==>cm_OCR_EngEll
;----------------------------------------------------------------------------------------
Func cm_Close()           ; ContextMenu  "Close"
    Local $Active_title = WinGetTitle(@GUI_WinHandle)
    Local $strLen = StringLen(@ScriptDir & "\tmp\image_")
    If StringLeft($Active_title, $strLen) = @ScriptDir & "\tmp\image_" Then
        GUIDelete(@GUI_WinHandle)
        FileDelete($Active_title)
        NoteGui_Delete(@GUI_WinHandle)
        ;ConsoleWrite("- WinClose " & @GUI_WinHandle & " & FileDelete " & $Active_title & @CRLF)
    EndIf
EndFunc   ;==>cm_Close
;----------------------------------------------------------------------------------------

Please, every comment is appreciated!
leave your comments and experiences here!
Thank you very much

Edited September 27 by ioa747
UpDate to Version 0.0.4.20230924

ioa747 · September 26

On 7/31/2022 at 7:33 PM, ioa747 said:

UpDate to Version 0.0.3.20230924

Sign In

SmartNote

Recommended Posts

ioa747

SmartNote is a screen snip tool to snip screen area with OCR ability from tesseract 100+ languages

I did some upgrades

Snipe new note

Tesseract info

Link to comment

Share on other sites

ioa747

Link to comment

Share on other sites

Create an account or sign in to comment

Create an account

Sign in

Recently Browsing 0 members

Similar Content

OCRSpace UDF

OCR and MODI MiRect property

UWPOCR - Windows Platform Optical character recognition API Implementation 1 2 3

Website screenshot / information

Tesseract OCR - Reading text

Browse

AutoIt Resources

Release

Beta