Mesh result is not correct by running utils_mpi_model.py using freihand ground truth data

[youngstu]

Mesh result is not correct by running utils_mpi_model.py using freihand ground truth data.

    ### test freihand dataset ###
    freihand_root_dir = "FreiHAND_pub_v2/"
    freihand_K_path = os.path.join(freihand_root_dir, "training_K.json")
    freihand_mano_path = os.path.join(freihand_root_dir, "training_mano.json")
    freihand_xyz_path = os.path.join(freihand_root_dir, "training_xyz.json")

    K_list = json.load(open(freihand_K_path, 'r'))
    mano_list = json.load(open(freihand_mano_path, 'r'))
    xyz_list = json.load(open(freihand_xyz_path, 'r'))
    img_path_list = [os.path.join(freihand_root_dir, "training/rgb/%08d.jpg" % idx) \
                     for idx in range(len(mano_list))]

    select_index = 0
    img_path = img_path_list[select_index]
    K_param = K_list[select_index]
    mano_param = mano_list[select_index]
    mano_param = np.array(mano_param)[0]
    xyz_param = xyz_list[select_index]

    pose_param = mano_param[:48]
    shape_param = mano_param[48:58]
    uv_root_param = mano_param[58:60]
    scale_param = mano_param[60:]

    beta = torch.Tensor(shape_param).reshape([bs, 10])
    pose = torch.Tensor(pose_param[3:]).reshape([bs, 15, 3])
    rvec = torch.Tensor(pose_param[:3]).reshape([bs, 3])
    tvec = torch.Tensor(mano_param[58:61]).reshape([bs, 1, 3])
    xyz_root = torch.Tensor(xyz_param[0])

[gmntu]

@youngstu Could you also provide the part of comparing the mesh result, after getting the ground truth data?

[youngstu]

[gmntu]

@youngstu you will need to add the mean pose i.e. pose += model.M_.reshape(bs,15,3) # Add model mean pose

It is similar to this issue found on Freihand dataset github and can also be found in Freihand view_samples.py which include the use of mean pose.

Below is the sample code:

import os
import json
import torch
import numpy as np
import open3d as o3d

from utils_mpi_model import MANO


### test freihand dataset ###
freihand_root_dir = 'FreiHAND_pub_v2/'
freihand_K_path = os.path.join(freihand_root_dir, "training_K.json")
freihand_mano_path = os.path.join(freihand_root_dir, "training_mano.json")
freihand_xyz_path = os.path.join(freihand_root_dir, "training_xyz.json")

K_list = json.load(open(freihand_K_path, 'r'))
mano_list = json.load(open(freihand_mano_path, 'r'))
xyz_list = json.load(open(freihand_xyz_path, 'r'))
img_path_list = [os.path.join(freihand_root_dir, "training/rgb/%08d.jpg" % idx) \
                 for idx in range(len(mano_list))]

select_index = 0
img_path = img_path_list[select_index]
K_param = K_list[select_index]
mano_param = mano_list[select_index]
mano_param = np.array(mano_param)[0]
xyz_param = xyz_list[select_index]

pose_param = mano_param[:48]
shape_param = mano_param[48:58]
uv_root_param = mano_param[58:60]
scale_param = mano_param[60:]

bs = 1
beta = torch.Tensor(shape_param).reshape([bs,10])
pose = torch.Tensor(pose_param[3:]).reshape([bs,15,3])
rvec = torch.Tensor(pose_param[:3]).reshape([bs,3])
tvec = torch.Tensor(mano_param[58:61]).reshape([bs,3])
xyz_root = torch.Tensor(xyz_param[0])


model = MANO('right')
pose += model.M_.reshape(bs,15,3) # Add model mean pose
tvec = torch.zeros([bs,3], dtype=torch.float32) # Set tvec to zero else the hand will be out of Open3D FoV

vertices, joints = model.forward(beta, pose, rvec, tvec)
print('vertices', vertices.shape, vertices.dtype) # torch.Size([10, 778, 3]) torch.float32
print('joints', joints.shape, joints.dtype)       # torch.Size([10, 21, 3]) torch.float32

# Create a reference frame 10 cm
mesh_frame = o3d.geometry.TriangleMesh.create_coordinate_frame(size=0.1)        
# Draw mesh model
mesh = o3d.geometry.TriangleMesh()
mesh.vertices = o3d.utility.Vector3dVector(vertices[0,:,:])
mesh.triangles = o3d.utility.Vector3iVector(model.F)
mesh.compute_vertex_normals()
mesh.paint_uniform_color([0.75, 0.75, 0.75])
# Draw wireframe
ls = o3d.geometry.LineSet.create_from_triangle_mesh(mesh)
ls.paint_uniform_color([0.75, 0.75, 0.75])
# Draw joints
mesh_spheres = []
for j in joints[0,:,:]:
    m = o3d.geometry.TriangleMesh.create_sphere(radius=0.001)
    m.compute_vertex_normals()
    m.paint_uniform_color([1,0,0])
    m.translate(j)
    mesh_spheres.append(m)

o3d.visualization.draw_geometries([mesh, mesh_frame])
o3d.visualization.draw_geometries([ls, mesh_frame] + mesh_spheres)

[youngstu]

That's right. Thank you very much for the information.