Can ZIM do eye and body tracking

I found this simple code in p5
but I want it in ZIM ofcourse


waving works

 <script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/1.0.0/p5.js"></script>
  <script src="https://unpkg.com/ml5@0.4.3/dist/ml5.min.js"></script>
 <style>
    canvas {
      border: 5px solid white;
      border-radius: 20px;
      box-shadow: 5px 5px 5px grey;
    }
  </style>
</head>

<body>
  <div class="copy_right_footer"> Developed by - Arnav Jaiswal </div>
  <script>
    let nosex, nosey, learx, leary, rearx, reary, leyex, leyey, reyex, reyey, lhipx, lhipy, rhipx, rhipy, lsholx, lsholy, rsholx, rsholy, lelbx, lelby, relbx, relby, lwrix, lwriy, rwrix, rwriy, lankx, lanky, rankx, ranky, lknex, lkney, rknex, rkney, angel;

    function setup() {
      canvas = createCanvas(window.innerWidth, window.innerHeight);
      canvas.position(0, 0);
      video = createCapture(VIDEO);
      video.size(window.innerWidth, window.innerHeight);
      poseNet = ml5.poseNet(video, modelLoaded);
      poseNet.on('pose', gotPoses);
      angleMode(DEGREES);
      video.hide()
    }

    function draw() {
      image(video, 0, 0, window.innerWidth, window.innerHeight);
      background("#ffffff80");
      fill('#ebca7f');
      stroke('#ebca7f');
      quad(nosex - ((learx + rearx) / 2) + rearx, nosey - 20, nosex + ((learx - rearx) / 2), nosey - 20, ((lsholx - rsholx) * 0.60) + rsholx, lsholy - 30, ((lsholx - rsholx) * 0.40) + rsholx, rsholy - 30);
      strokeWeight(20);
      fill(color);
      stroke('#000000');
      quad(lhipx, lhipy, rhipx, rhipy, rsholx, rsholy - 30, lsholx, lsholy - 30);
      line(lsholx, lsholy, lelbx, lelby);
      line(lelbx, lelby, lwrix, lwriy);
      line(rsholx, rsholy, relbx, relby);
      line(relbx, relby, rwrix, rwriy);
      stroke('#000000');
      line(lhipx, lhipy, lknex, lkney);
      line(lknex, lkney, lankx, lanky);
      line(rhipx, rhipy, rknex, rkney);
      line(rknex, rkney, rankx, ranky);
      angel = leary - reary;
      translate(nosex, nosey);
      rotate(angel / 2.5);
      rectMode(CENTER);
      strokeWeight(0);
      fill('#ffda82');
      stroke('#ffda82');
      ellipse(0, 0, rearx - learx * 1.1, rearx - learx * 1.1);
      translate(-nosex, -nosey);
      fill('#000000');
      stroke('#000000');
      rect(leyex, leyey, (rearx - learx) / 10, (rearx - learx) / 10);
      fill('#000000');
      stroke('#000000');
      rect(reyex, reyey, (rearx - learx) / 10, (rearx - learx) / 10);
      fill('#c7a75b');
      stroke('#c7a75b');
      rect(nosex, nosey, (rearx - learx) / 8, (rearx - learx) / 8);
      fill('#f0c3a5');
      stroke('#f0c3a5');
      rect(nosex, nosey + (rearx - learx * 1.1) * -0.194, (rearx - learx) / 1.5, (rearx - learx) / 10);
      fill('#734603');
      stroke('#734603');
      rect(nosex, nosey - (rearx - learx * 1.1) * -0.44444, (rearx - learx), (rearx - learx) * -0.2)
    }

    function modelLoaded() {
      console.log('PoseNet is Initialized')
    }

    function gotPoses(results) {
      if (results.length > 0) {
        leary = results[0].pose.leftEar.y;
        learx = results[0].pose.leftEar.x;
        reary = results[0].pose.rightEar.y;
        rearx = results[0].pose.rightEar.x;
        leyey = results[0].pose.leftEye.y;
        leyex = results[0].pose.leftEye.x;
        reyey = results[0].pose.rightEye.y;
        reyex = results[0].pose.rightEye.x;
        nosey = results[0].pose.nose.y;
        nosex = results[0].pose.nose.x;
        lhipx = results[0].pose.leftHip.x;
        lhipy = results[0].pose.leftHip.y;
        rhipx = results[0].pose.rightHip.x;
        rhipy = results[0].pose.rightHip.y;
        lsholx = results[0].pose.leftShoulder.x;
        lsholy = results[0].pose.leftShoulder.y;
        rsholx = results[0].pose.rightShoulder.x;
        rsholy = results[0].pose.rightShoulder.y;
        lelbx = results[0].pose.leftElbow.x;
        lelby = results[0].pose.leftElbow.y;
        relbx = results[0].pose.rightElbow.x;
        relby = results[0].pose.rightElbow.y;
        lwrix = results[0].pose.leftWrist.x;
        lwriy = results[0].pose.leftWrist.y;
        rwrix = results[0].pose.rightWrist.x;
        rwriy = results[0].pose.rightWrist.y;
        lknex = results[0].pose.leftKnee.x;
        lkney = results[0].pose.leftKnee.y;
        rknex = results[0].pose.rightKnee.x;
        rkney = results[0].pose.rightKnee.y;
        lankx = results[0].pose.leftAnkle.x;
        lanky = results[0].pose.leftAnkle.y;
        rankx = results[0].pose.rightAnkle.x;
        ranky = results[0].pose.rightAnkle.y
      }
    }

    function generateRainbowHex() {
      time = new Date().getTime();
      frequency = 0.001;
      amplitude = 127;
      red = Math.sin(frequency * time + 0) * amplitude + 128;
      green = Math.sin(frequency * time + 2 * Math.PI / 3) * amplitude + 128;
      blue = Math.sin(frequency * time + 4 * Math.PI / 3) * amplitude + 128;
      hexCode = `#${Math.round(red).toString(16).padStart(2,'0')}${Math.round(green).toString(16).padStart(2,'0')}${Math.round(blue).toString(16).padStart(2,'0')}`;
      return hexCode
    }
    setInterval(() => {
      color = generateRainbowHex()
    }, 1);
  </script>

can be found on codepen also
https://codepen.io/Awesome_Billy_Bob/pen/MWZzMRG
(found it via stackoverflow article)
https://stackoverflow.com/questions/77248149/i-am-using-p5-js-and-ml5-js-how-do-i-set-the-background-of-a-canvas-to-the-came


would be a lot of fun for apps
thanks

anybody want an update about tracking check
https://thecodingtrain.com/tracks/ml5js-beginners-guide/ml5/hand-pose
video of a month ago

for example
https://editor.p5js.org/codingtrain/sketches/o5wnL6esQ
so I tested


and yes it works!
check the emoji of the thumb below the video in the example
https://editor.p5js.org/codingtrain/sketches/0_qPHtsF_

now.. also in ZIM integration?

also moving video can now be tracked in 3D with not using poseNet now moveNet but better BlazePose to have also Z-estimation of movement opensource


so x, y, z

can be cool for your dancing videos Dan :slight_smile:

there is now ml5@1 version much better


check the video at

https://youtu.be/2h8VArJ3gnQ?si=y612Z2vMXXP2Rl6A&t=304

Does the ml5 also do eyetracking - is it an external library - so not really p5js and p5js is just showing the results. If that is the case, you can probably already do this in ZIM using the ml5 library. Someone should try it out and let us know.

happy birthday ZIM ..
I found cool webcam motion dectection app with with an emitter check the ZIM emitter tool https://zimjs.com/016/emitter


https://editor.p5js.org/codingtrain/sketches/e_vqrMWcL
found under the video on theCodeTrain :slight_smile: https://thecodingtrain.com/tracks/ml5js-beginners-guide/ml5/7-bodypose/pose-detection
GreetZ

waauw that would be wonderfull if it would work..
maybe you can give me a start how to code the emitter working with the eyetracking and other parts of the body?
or one of your students @danzen ?
ThankZ a lot to test it out :slight_smile:
in the video is also a mouth open/closing and moving pattern as ZIM has already made before.. but now animated with the webcam


image

We have been considering all this stuff since we started the Cam. We wanted to find out how P5js tracked so quickly based on these vids. We asked at the time if anyone wanted to look into the code and improve things. Anyway - we are doing what we are doing... let's leave this - it is plenty for us to think about. You go and do your own thing. The offer is still open, if anyone wants to see how all this works in ZIM.

ok, so physics to catch the circles, I thought @amihanya made already an example
https://editor.p5js.org/pattvira/sketches/kxhf0tEiW
as this p5 tuturial is showing.. maby Ami can help a little; thanks


video

bodyposes can be found here
https://docs.ml5js.org/#/reference/bodypose?id=bodypose

I tried with the ZIM_cam
but no results..
any idea how to solve?
to trace only my nose

https://editor.p5js.org/codingtrain/sketches/AWrwZ_rSv

make code

                        // Update ZIM bitmap
                     //   canvasZim.bitmapCanvas = canvas;
                    }
                }
            });

            // Ticker to keep canvas updated
            Ticker.add(() => {
                stage.update();
            });
        }
    });
}

// Ensure dependencies are loaded before running
window.addEventListener('load', () => {
    // Check if ZIM and ML5 are available
    if (typeof zim !== 'undefined' && typeof ml5 !== 'undefined') {
        Main();
    } else {
        console.error('ZIM or ML5 not loaded. Please include the libraries.');
    }
});
             
    </script>
  <body>
    </body>
  </html>

so in the future to make this