Mediapipe UDF

[smbape 43]

After a long hesitation about whether it would be fun to do or not, I finally did it.

This UDF provides a way to use mediapipe in AutoIt

The usage is similar to the python usage of mediapipe

Prerequisites

• Download and extract autoit-mediapipe-0.8.11-opencv-4.6.0-com-v0.1.0.7z into a folder
• Download the opencv UDF from here

Sources

Here

Documentation

A generated documentation for functions is available here

Examples

More examples can be found here

To run them, please follow these instructions

Face mesh

expandcollapse popup

#Region ;**** Directives created by AutoIt3Wrapper_GUI ****
#AutoIt3Wrapper_UseX64=y
#AutoIt3Wrapper_Change2CUI=y
#AutoIt3Wrapper_Au3Check_Parameters=-d -w 1 -w 2 -w 3 -w 4 -w 5 -w 6
#AutoIt3Wrapper_AU3Check_Stop_OnWarning=y
#EndRegion ;**** Directives created by AutoIt3Wrapper_GUI ****

#include "autoit-mediapipe-com\udf\mediapipe_udf_utils.au3"
#include "autoit-opencv-com\udf\opencv_udf_utils.au3"

_Mediapipe_Open_And_Register("opencv-4.6.0-vc14_vc15\opencv\build\x64\vc15\bin\opencv_world460.dll", "autoit-mediapipe-com\autoit_mediapipe_com-0.8.11-460.dll")
_OpenCV_Open_And_Register("opencv-4.6.0-vc14_vc15\opencv\build\x64\vc15\bin\opencv_world460.dll", "autoit-opencv-com\autoit_opencv_com460.dll")
OnAutoItExitRegister("_OnAutoItExit")

; Tell mediapipe where to look its resource files
_Mediapipe_SetResourceDir()

Global $mp = _Mediapipe_get()
If Not IsObj($mp) Then
  ConsoleWriteError("Failed to load mediapipe" & @CRLF)
  Exit
EndIf

Global $cv = _OpenCV_get()
If Not IsObj($cv) Then
  ConsoleWriteError("Failed to load opencv" & @CRLF)
  Exit
EndIf

Example()

Func Example()
  Local $download_utils = $mp.solutions.download_utils

  $download_utils.download( _
      "https://github.com/tensorflow/tfjs-models/raw/master/face-detection/test_data/portrait.jpg", _
      @ScriptDir & "/testdata/portrait.jpg" _
      )

  Local $mp_face_mesh = $mp.solutions.face_mesh
  Local $mp_drawing = $mp.solutions.drawing_utils
  Local $mp_drawing_styles = $mp.solutions.drawing_styles

  Local $image_path = @ScriptDir & "/testdata/portrait.jpg"
  Local $image = $cv.imread($image_path)

  ; Preview the images.
  Local $ratio = resize_and_show("preview", $image)
  Local $scale = 1 / $ratio

  ; Run MediaPipe Face Mesh
  Local $face_mesh = $mp_face_mesh.FaceMesh(_Mediapipe_Params( _
      "static_image_mode", True, _
      "refine_landmarks", True, _
      "max_num_faces", 2, _
      "min_detection_confidence", 0.5 _
      ))

  ; Convert the BGR image to RGB and process it with MediaPipe Face Mesh.
  Local $results = $face_mesh.process($cv.cvtColor($image, $CV_COLOR_BGR2RGB))
  If $results("multi_face_landmarks") == Default Then
    ConsoleWrite("No face detection for " & $image_path & @CRLF)
    Return
  EndIf

  Local $annotated_image = $image.copy()

  ; Draw face detections of each face.
  For $face_landmarks In $results("multi_face_landmarks")
    $mp_drawing.draw_landmarks(_Mediapipe_Params( _
        "image", $annotated_image, _
        "landmark_list", $face_landmarks, _
        "connections", $mp_face_mesh.FACEMESH_TESSELATION, _
        "landmark_drawing_spec", Null, _
        "connection_drawing_spec", $mp_drawing_styles.get_default_face_mesh_tesselation_style($scale)))
    $mp_drawing.draw_landmarks(_Mediapipe_Params( _
        "image", $annotated_image, _
        "landmark_list", $face_landmarks, _
        "connections", $mp_face_mesh.FACEMESH_CONTOURS, _
        "landmark_drawing_spec", Null, _
        "connection_drawing_spec", $mp_drawing_styles.get_default_face_mesh_contours_style($scale)))
    $mp_drawing.draw_landmarks(_Mediapipe_Params( _
        "image", $annotated_image, _
        "landmark_list", $face_landmarks, _
        "connections", $mp_face_mesh.FACEMESH_IRISES, _
        "landmark_drawing_spec", Null, _
        "connection_drawing_spec", $mp_drawing_styles.get_default_face_mesh_iris_connections_style($scale)))
  Next

  resize_and_show("face mesh", $annotated_image)

  ; display images until a keyboard action is detected
  $cv.waitKey()
  $cv.destroyAllWindows()
EndFunc   ;==>Example

Func resize_and_show($title, $image)
  Local Const $DESIRED_HEIGHT = 480
  Local Const $DESIRED_WIDTH = 480
  Local $w = $image.width
  Local $h = $image.height

  If $h < $w Then
    $h = $h / ($w / $DESIRED_WIDTH)
    $w = $DESIRED_WIDTH
  Else
    $w = $w / ($h / $DESIRED_HEIGHT)
    $h = $DESIRED_HEIGHT
  EndIf

  Local $interpolation = ($DESIRED_WIDTH > $image.width Or $DESIRED_HEIGHT > $image.height) ? $CV_INTER_CUBIC : $CV_INTER_AREA

  Local $img = $cv.resize($image, _OpenCV_Size($w, $h), _OpenCV_Params("interpolation", $interpolation))
  $cv.imshow($title, $img.convertToShow())

  Return $img.width / $image.width
EndFunc   ;==>resize_and_show

Func _OnAutoItExit()
  _OpenCV_Unregister_And_Close()
  _Mediapipe_Unregister_And_Close()
EndFunc   ;==>_OnAutoItExit

 

[smbape 43]

Hi @malcev

The UDF of mediapipe is out.

Do you happen to know of any mediapipe python examples that are different from what is available at MediaPipe in Python?

Everything I come across is a variation of those examples.

[malcev 0]

Hi smbape!

Sorry, but I dont know any other resources.

Only some video tutorials like this:

Thank You very much for such great work!