three.js raycast on skinning mesh

Question

I'm trying to raycast skinning mesh (of knowing issue) after some skeleton changes (without animation on it, so performance isn't a priority).

The tricky thing i imagine in this attempt is:

1. Load skinned mesh add to scene
2. Make some changes in positions of specific bones at loaded mesh
3. Copy geometries of transformed loaded mesh (maybe from buffer?)
4. Create new mesh (some kind of imitation ghost mesh) from copied geometries and apply to it
5. set raycast on ghost mesh with opacity material= 0.0

Above list should work, but I'm stuck third day on point 3 cause I can't get transformed vertices after skinning.

var scene, camera, renderer, mesh, ghostMesh;

var raycaster = new THREE.Raycaster();
var raycasterMeshHelper;

initScene();
render();

function initScene() {
  scene = new THREE.Scene();
  camera = new THREE.PerspectiveCamera(75, window.innerWidth / window.innerHeight, 0.1, 200);
  camera.position.set(20, 7, 3);
  renderer = new THREE.WebGLRenderer({
    antialias: true
  });
  renderer.setPixelRatio(window.devicePixelRatio);
  renderer.setSize(window.innerWidth, window.innerHeight);

  document.body.appendChild(renderer.domElement);
  window.addEventListener('resize', onWindowResize, false);

  var orbit = new THREE.OrbitControls(camera, renderer.domElement);

  //lights stuff
  var ambientLight = new THREE.AmbientLight(0xffffff, 0.3);
  scene.add(ambientLight);
  var lights = [];
  lights[0] = new THREE.PointLight(0xffffff, 1, 0);
  lights[1] = new THREE.PointLight(0xffffff, 1, 0);
  lights[2] = new THREE.PointLight(0xffffff, 1, 0);
  lights[0].position.set(0, 200, 0);
  lights[1].position.set(100, 200, 100);
  lights[2].position.set(-100, -200, -100);
  scene.add(lights[0]);
  scene.add(lights[1]);
  scene.add(lights[2]);

  //raycaster mesh 
  var raycasterMaterial = new THREE.MeshBasicMaterial({
    color: 0xdddddd,
    opacity: 0.7,
    transparent: true
  });
  var geometrySphere = new THREE.SphereGeometry(0.5, 16, 16);
  raycasterMeshHelper = new THREE.Mesh(geometrySphere, raycasterMaterial);
  raycasterMeshHelper.visible = false;
  scene.add(raycasterMeshHelper);

  renderer.domElement.addEventListener('mousemove', onMouseMove, false);

  //model Loading

  var loader = new THREE.JSONLoader();
  loader.load("https://raw.githubusercontent.com/visus100/skinnedTests/master/js_fiddle/skinned_mesh.json", function(geometry) {
    var meshMaterial = new THREE.MeshStandardMaterial({
      color: 0x00df15,
      skinning: true
    });

    mesh = new THREE.SkinnedMesh(geometry, meshMaterial);
    scene.add(mesh);

    var skeleton = new THREE.SkeletonHelper(mesh);
    scene.add(skeleton);

    //some experimental skeletonal changes
    mesh.skeleton.bones[1].rotation.z += 0.10;
    mesh.skeleton.bones[2].rotation.x += -0.65;
    mesh.skeleton.bones[3].rotation.y += -0.45;
    mesh.skeleton.bones[3].position.x += 0.11;

    //updates matrix
    mesh.updateMatrix();
    mesh.geometry.applyMatrix(mesh.matrix);
    mesh.updateMatrixWorld(true);

    //crate ghost mesh geometry
    createGhostMesh();

    //crate point cloud helper from buffergeometry
    var bufferGeometry = new THREE.BufferGeometry().fromGeometry(mesh.geometry);

    var particesMaterial = new THREE.PointsMaterial({
      color: 0xff00ea,
      size: 0.07,
      sizeAttenuation: false
    });
    particles = new THREE.Points(bufferGeometry, particesMaterial);
    particles.sortParticles = true;
    scene.add(particles);

  });
}

function createGhostMesh() {
  var geometryForGhostMesh = new THREE.Geometry();

  //push vertices and other stuff to geometry
  for (i = 0; i < mesh.geometry.vertices.length; i++) {
    var temp = new THREE.Vector3(mesh.geometry.vertices[i].x, mesh.geometry.vertices[i].y, mesh.geometry.vertices[i].z);
    geometryForGhostMesh.vertices.push(temp);

    //////
    //here should be the code for calc translation vertices of skinned mesh and added to geometryForGhostMesh
    //////

    geometryForGhostMesh.skinIndices.push(mesh.geometry.skinIndices[i]);
    geometryForGhostMesh.skinWeights.push(mesh.geometry.skinWeights[i]);
  }

  for (i = 0; i < mesh.geometry.faces.length; i++) {
    geometryForGhostMesh.faces.push(mesh.geometry.faces[i]);
  }

  //create material and add to scene

  var ghostMaterial = new THREE.MeshBasicMaterial({
    color: 0xff0000,
    opacity: 0.1,
    transparent: true,
    skinning: true
  });
  ghostMesh = new THREE.Mesh(geometryForGhostMesh, ghostMaterial);
  scene.add(ghostMesh);
}

function onWindowResize() {
  camera.aspect = window.innerWidth / window.innerHeight;
  camera.updateProjectionMatrix();
  renderer.setSize(window.innerWidth, window.innerHeight);
}

function render() {
  requestAnimationFrame(render);
  renderer.render(scene, camera);
};

function onMouseMove(event) {
  //raycaster for ghostMesh 
  if (ghostMesh) {
    var rect = renderer.domElement.getBoundingClientRect();
    var mouseX = ((event.clientX - rect.left) / rect.width) * 2 - 1;
    var mouseY = -((event.clientY - rect.top) / rect.height) * 2 + 1;
    raycaster.setFromCamera(new THREE.Vector2(mouseX, mouseY), camera);

    var intersects = raycaster.intersectObject(ghostMesh);
    if (intersects.length > 0) {
      raycasterMeshHelper.visible = true;
      raycasterMeshHelper.position.set(0, 0, 0);
      raycasterMeshHelper.lookAt(intersects[0].face.normal);
      raycasterMeshHelper.position.copy(intersects[0].point);
    } else {
      raycasterMeshHelper.visible = false;
    }
  }
}

body {
  margin: 0px;
  background-color: #000000;
  overflow: hidden;
}

<script src="https://cdnjs.cloudflare.com/ajax/libs/three.js/98/three.min.js"></script>
<script src="https://threejs.org/examples/js/controls/OrbitControls.js"></script>

Please note that I need this in thre.js build r98 or less, because the rest of my code (not included here) and without morph tangents only skinning bones.

I tried to write it clearly and please if anyone want help do it so because I'm not a pro.

I not including my approach of calculating transformed geometries because I failed too hard.

I dug a lot about this problem here e.g. issue6440 and for today it's still not fixed.

But there existing methods to work with it e.g https://jsfiddle.net/fnjkeg9x/1/ but after several of attempts I failed and my conclusion is that the stormtrooper works on morph tanges and this could be the reason I failed.

EDIT:

I created next codepen based on this topics get-the-global-position-of-a-vertex-of-a-skinned-mesh and Stormtrooper. Decided to start with simple box to make bounding around skinned transformed mesh.

Result is fail because it giving 0 at line:
boneMatrix.fromArray(skeleton.boneMatrices, si * 16);
Here i comparing stormtrooper with my example output from console: Screen shot image

Codpen with new progress: https://codepen.io/donkeyLuck0/pen/XQbBMQ

My other idea is to apply this bones form loaded model and rig as a morph tangent programmatically (but i don't even know if it is possible and how to figure it out)

Founded example of animated model Sketchfab animation with points tracking

Answer 1

You can use GPU picking to "pick" skinned object. It won't give you a position though

Note: GPU picking requires rendering every pickable object with a custom material. How you implement that is up to you. This article does it by making 2 scenes. That might not be as useful for skinned objects.

Unfortunately three.js provides no way to override materials AFAICT. Here's an example that replaces the materials on the pickable objects before rendering for picking and then restores them after. You would also need to hide any objects you don't want picked.

const renderer = new THREE.WebGLRenderer({
  antialias: true,
  canvas: document.querySelector('canvas'),
});
const scene = new THREE.Scene();
const camera = new THREE.PerspectiveCamera(75, 2, 0.1, 200);
camera.position.set(20, 7, 3);

const orbit = new THREE.OrbitControls(camera, renderer.domElement);

//lights stuff
const ambientLight = new THREE.AmbientLight(0xffffff, 0.3);
scene.add(ambientLight);
const lights = [];
lights[0] = new THREE.PointLight(0xffffff, 1, 0);
lights[1] = new THREE.PointLight(0xffffff, 1, 0);
lights[2] = new THREE.PointLight(0xffffff, 1, 0);
lights[0].position.set(0, 200, 0);
lights[1].position.set(100, 200, 100);
lights[2].position.set(-100, -200, -100);
scene.add(lights[0]);
scene.add(lights[1]);
scene.add(lights[2]);


//raycaster mesh 
const raycasterMaterial = new THREE.MeshBasicMaterial({
  color: 0xdddddd,
  opacity: 0.7,
  transparent: true
});
const geometrySphere = new THREE.SphereGeometry(0.5, 16, 16);
raycasterMeshHelper = new THREE.Mesh(geometrySphere, raycasterMaterial);
raycasterMeshHelper.visible = false;
scene.add(raycasterMeshHelper);


//model Loading
const pickableObjects = [];

const loader = new THREE.JSONLoader();
loader.load("https://raw.githubusercontent.com/visus100/skinnedTests/master/js_fiddle/skinned_mesh.json", (geometry) => {
  const meshMaterial = new THREE.MeshStandardMaterial({
    color: 0x00df15,
    skinning: true
  });

  const mesh = new THREE.SkinnedMesh(geometry, meshMaterial);
  scene.add(mesh);

  const id = pickableObjects.length + 1;
  pickableObjects.push({
    mesh,
    renderingMaterial: meshMaterial,
    pickingMaterial: new THREE.MeshPhongMaterial({
      skinning: true,
      emissive: new THREE.Color(id),
      color: new THREE.Color(0, 0, 0),
      specular: new THREE.Color(0, 0, 0),
      //map: texture,
      //transparent: true,
      //side: THREE.DoubleSide,
      //alphaTest: 0.5,
      blending: THREE.NoBlending,
    }),
  });

  //some experimental skeletonal changes
  mesh.skeleton.bones[1].rotation.z += 0.10;
  mesh.skeleton.bones[2].rotation.x += -0.65;
  mesh.skeleton.bones[3].rotation.y += -0.45;
  mesh.skeleton.bones[3].position.x += 0.11;

  //updates matrix
  mesh.updateMatrix();
  mesh.geometry.applyMatrix(mesh.matrix);
  mesh.updateMatrixWorld(true);

});

class GPUPickHelper {
  constructor() {
    // create a 1x1 pixel render target
    this.pickingTexture = new THREE.WebGLRenderTarget(1, 1);
    this.pixelBuffer = new Uint8Array(4);
  }
  pick(cssPosition, scene, camera) {
    const {
      pickingTexture,
      pixelBuffer
    } = this;

    // set the view offset to represent just a single pixel under the mouse
    const pixelRatio = renderer.getPixelRatio();
    camera.setViewOffset(
      renderer.context.drawingBufferWidth, // full width
      renderer.context.drawingBufferHeight, // full top
      cssPosition.x * pixelRatio | 0, // rect x
      cssPosition.y * pixelRatio | 0, // rect y
      1, // rect width
      1, // rect height
    );
    // render the scene
    // r102
    //renderer.setRenderTarget(pickingTexture);
    //renderer.render(scene, camera);
    //renderer.setRenderTarget(null);
    // r98
    renderer.render(scene, camera, pickingTexture);
    // clear the view offset so rendering returns to normal
    camera.clearViewOffset();
    //read the pixel
    renderer.readRenderTargetPixels(
      pickingTexture,
      0, // x
      0, // y
      1, // width
      1, // height
      pixelBuffer);

    const id =
      (pixelBuffer[0] << 16) |
      (pixelBuffer[1] << 8) |
      (pixelBuffer[2]);
    return id;
  }
}

function resizeRendererToDisplaySize(renderer) {
  const canvas = renderer.domElement;
  const width = canvas.clientWidth;
  const height = canvas.clientHeight;
  const needResize = canvas.width !== width || canvas.height !== height;
  if (needResize) {
    renderer.setSize(width, height, false);
  }
  return needResize;
}

const pickPosition = {
  x: 0,
  y: 0,
};
const pickHelper = new GPUPickHelper();
let lastPickedId = 0;
let lastPickedObjectSavedEmissive;

function render(time) {
  time *= 0.001;  // convert to seconds;

  if (resizeRendererToDisplaySize(renderer)) {
    const canvas = renderer.domElement;
    camera.aspect = canvas.clientWidth / canvas.clientHeight;
    camera.updateProjectionMatrix();
  }

  if (lastPickedId) {
    pickableObjects[lastPickedId - 1].renderingMaterial.emissive.setHex(lastPickedObjectSavedEmissive);
    lastPickedId = 0;
  }

  for (pickableObject of pickableObjects) {
    pickableObject.mesh.material = pickableObject.pickingMaterial;
  }
  
  const id = pickHelper.pick(pickPosition, scene, camera, time);
  
  for (pickableObject of pickableObjects) {
    pickableObject.mesh.material = pickableObject.renderingMaterial;
  }
  
  const pickedObject = pickableObjects[id - 1];
  if (pickedObject) {
    lastPickedId = id;
    lastPickedObjectSavedEmissive = pickedObject.renderingMaterial.emissive.getHex();
    pickedObject.renderingMaterial.emissive.setHex((time * 8) % 2 > 1 ? 0xFFFF00 : 0xFF0000);
  }
  
  renderer.render(scene, camera);

  requestAnimationFrame(render);
};
requestAnimationFrame(render);

function setPickPosition(event) {
  pickPosition.x = event.clientX;
  pickPosition.y = event.clientY;
}

function clearPickPosition() {
  // unlike the mouse which always has a position
  // if the user stops touching the screen we want
  // to stop picking. For now we just pick a value
  // unlikely to pick something
  pickPosition.x = -100000;
  pickPosition.y = -100000;
}

window.addEventListener('mousemove', setPickPosition);
window.addEventListener('mouseout', clearPickPosition);
window.addEventListener('mouseleave', clearPickPosition);

window.addEventListener('touchstart', (event) => {
  // prevent the window from scrolling
  event.preventDefault();
  setPickPosition(event.touches[0]);
}, {
  passive: false
});

window.addEventListener('touchmove', (event) => {
  setPickPosition(event.touches[0]);
});

window.addEventListener('touchend', clearPickPosition);


window.addEventListener('mousemove', setPickPosition);
window.addEventListener('mouseout', clearPickPosition);
window.addEventListener('mouseleave', clearPickPosition);

body {
  margin: 0;
}
canvas {
  width: 100vw;
  height: 100vh;
  display: block;
}

<script src="https://threejsfundamentals.org/threejs/resources/threejs/r98/three.min.js"></script>
<script src="https://threejsfundamentals.org/threejs/resources/threejs/r98/js/controls/OrbitControls.js"></script>
<canvas></canvas>

Answer 2

This is super late to the game, but here's an example of GPU picking that works with skinned meshes and doesn't require a separate picking scene to keep in sync with your main scene, nor does it require the user to manage custom materials:

https://github.com/bzztbomb/three_js_gpu_picking

The trick that allows for easy material overriding and scene re-use is here: https://github.com/bzztbomb/three_js_gpu_picking/blob/master/gpupicker.js#L58

Answer 3

A proper support for raycasting for skinned meshes was added in https://github.com/mrdoob/three.js/pull/19178 in revision 116.