Teachable Machine and p5.js

172 Views Asked by At

I'm trying to import my teachable machine model to my p5.js sketch.

Current Problem: After I flipped the video input to mirror the camera, there's a mismatch of position of the dots and the position of my face is at. Below is the snippets of code that I used to draw circle on the exact position of my face.

  push();
  textSize(8);
  if (poser) { //did we get a skeleton yet;
    for (var i = 0; i < poser.length; i++) {
      let x = poser[i].position.x;
      let y = poser[i].position.y;
      ellipse(x, y, 5, 5);
      text(poser[i].part, x + 4, y);
    }
  }
  pop();

This is the what it currently look like: enter image description here

Here's the link to the full sketch: https://editor.p5js.org/wangz318/sketches/tdJm565Ap and in case the link goes down:

const modelURL = 'https://teachablemachine.withgoogle.com/models/kWOrQ14vz/';
// the json file (model topology) has a reference to the bin file (model weights)
const checkpointURL = modelURL + "model.json";
// the metatadata json file contains the text labels of your model and additional information
const metadataURL = modelURL + "metadata.json";


const flip = true; // whether to flip the webcam

let model;
let totalClasses;
let myCanvas;

let classification = "None Yet";
let probability = "100";
let poser;
let video;
let flippedVideo;

// A function that loads the model from the checkpoint
async function load() {
  model = await tmPose.load(checkpointURL, metadataURL);
  totalClasses = model.getTotalClasses();
  console.log("Number of classes, ", totalClasses);
}


async function setup() {
  myCanvas = createCanvas(400, 400);
  // Call the load function, wait until it finishes loading
  videoCanvas = createCanvas(320, 240)

  await load();
  video = createCapture(VIDEO, videoReady);
  video.size(320, 240);
  video.hide();

}

function draw() {
  background(255);
  if(video) {
    flippedVideo = ml5.flipImage(video);
    image(flippedVideo,0,0,video.width,video.height);
  }
  fill(255,0,0)
  textSize(18);
  text("Result:" + classification, 10, 40);

  text("Probability:" + probability, 10, 20)
  ///ALEX insert if statement here testing classification against apppropriate part of array for this time in your video

  push();
  textSize(8);
  if (poser) { //did we get a skeleton yet;
    for (var i = 0; i < poser.length; i++) {
      let x = poser[i].position.x;
      let y = poser[i].position.y;
      ellipse(x, y, 5, 5);
      text(poser[i].part, x + 4, y);
    }
  }
  pop();
  
  
}

function videoReady() {
  console.log("Video Ready");
  predict();
}


async function predict() {
  // Prediction #1: run input through posenet
  // predict can take in an image, video or canvas html element
  const {
    pose,
    posenetOutput
  } = await model.estimatePose(
    flippedVideo.elt //webcam.canvas,
  );
  // Prediction 2: run input through teachable machine assification model
  const prediction = await model.predict(
    posenetOutput,
    totalClasses
  );

  // console.log(prediction);
  
  // Sort prediction array by probability
  // So the first classname will have the highest probability
  const sortedPrediction = prediction.sort((a, b) => -a.probability + b.probability);

  //communicate these values back to draw function with global variables
  classification = sortedPrediction[0].className;
  probability = sortedPrediction[0].probability.toFixed(2);
  if (pose) poser = pose.keypoints; // is there a skeleton
  predict();
}

Anyone know how to get it right?

My goal: Get the flipping, and the facial position dots position right at the same time. https://editor.p5js.org/wangz318/sketches/tdJm565Ap

0

There are 0 best solutions below