adding evaluation code for H3.6M

cgopen · Jun 17, 2018 · 537f1b2 · 537f1b2
1 parent d2ead07
commit 537f1b2
Show file tree

Hide file tree

Showing 7 changed files with 614 additions and 2 deletions.
diff --git a/doc/train.md b/doc/train.md
@@ -105,3 +105,28 @@ Setup tensorboard to this directory to monitor the training progress like so:
 It's important to visually monitor the training! Make sure that the images
 loaded look right.
 
+
+## Evaluation
+Provided is an evaluation code for Human3.6M. It uses the test tf_records,
+provided with the training tf_records available above and [here (11GB)](https://drive.google.com/open?id=1tquavoVWSdGeOn9P6zwoffIMoCRElzEO).
+
+To evaluate a model, run
+```
+python -m src.benchmark.evaluate_h36m --batch_size=500
+--load_path=<model_to_eval.ckpt> --tfh36m_dir <path to tf_records_human36m_wjoints/>
+```
+for example for the provided model, use:
+```
+python -m src.benchmark.evaluate_h36m --batch_size=500
+--load_path=models/model.ckpt-667589 --tfh36m_dir <path to tf_records_human36m_wjoints/>
+```
+
+This writes intermediate output to a temp directory, which you can specify by pred_dir
+With the provided model, this outputs errors per action and overall MPE for P1
+(corresponding to Table 2 in paper -- this retrained model gets slightly lower
+MPJPE and a comparable PA-MPJPE):
+```
+MPJPE: 86.20, PA-MPJPE: 58.47, Median: 79.47, PA-Median: 52.95
+```
+
+Run it with `--vis` option, it visualizes the top/worst 30 results. 
diff --git a/src/RunModel.py b/src/RunModel.py
@@ -87,7 +87,7 @@ def build_test_model_ief(self):
         self.all_cams = []
         self.all_Js = []
         self.final_thetas = []
-        theta_prev = self.mean_var
+        theta_prev = tf.tile(self.mean_var, [self.batch_size, 1])
         for i in np.arange(self.num_stage):
             print('Iteration %d' % i)
             # ---- Compute outputs

diff --git a/src/benchmark/__init__.py b/src/benchmark/__init__.py
diff --git a/src/benchmark/eval_util.py b/src/benchmark/eval_util.py
@@ -0,0 +1,102 @@
+"""
+Utils for evaluation.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+
+def compute_similarity_transform(S1, S2):
+    '''
+    Computes a similarity transform (sR, t) that takes
+    a set of 3D points S1 (3 x N) closest to a set of 3D points S2,
+    where R is an 3x3 rotation matrix, t 3x1 translation, s scale.
+    i.e. solves the orthogonal Procrutes problem.
+    '''
+    transposed = False
+    if S1.shape[0] != 3 and S1.shape[0] != 2:
+        S1 = S1.T
+        S2 = S2.T
+        transposed = True
+    assert(S2.shape[1] == S1.shape[1])
+
+    # 1. Remove mean.
+    mu1 = S1.mean(axis=1, keepdims=True)
+    mu2 = S2.mean(axis=1, keepdims=True)
+    X1 = S1 - mu1
+    X2 = S2 - mu2
+
+    # 2. Compute variance of X1 used for scale.
+    var1 = np.sum(X1**2)
+
+    # 3. The outer product of X1 and X2.
+    K = X1.dot(X2.T)
+
+    # 4. Solution that Maximizes trace(R'K) is R=U*V', where U, V are
+    # singular vectors of K.
+    U, s, Vh = np.linalg.svd(K)
+    V = Vh.T
+    # Construct Z that fixes the orientation of R to get det(R)=1.
+    Z = np.eye(U.shape[0])
+    Z[-1, -1] *= np.sign(np.linalg.det(U.dot(V.T)))
+    # Construct R.
+    R = V.dot(Z.dot(U.T))
+
+    # 5. Recover scale.
+    scale = np.trace(R.dot(K)) / var1
+
+    # 6. Recover translation.
+    t = mu2 - scale*(R.dot(mu1))
+
+    # 7. Error:
+    S1_hat = scale*R.dot(S1) + t
+
+    if transposed:
+        S1_hat = S1_hat.T
+
+    return S1_hat
+
+
+def align_by_pelvis(joints, get_pelvis=False):
+    """
+    Assumes joints is 14 x 3 in LSP order.
+    Then hips are: [3, 2]
+    Takes mid point of these points, then subtracts it.
+    """
+    left_id = 3
+    right_id = 2
+
+    pelvis = (joints[left_id, :] + joints[right_id, :]) / 2.
+    if get_pelvis:
+        return joints - np.expand_dims(pelvis, axis=0), pelvis
+    else:
+        return joints - np.expand_dims(pelvis, axis=0)
+
+
+def compute_errors(gt3ds, preds):
+    """
+    Gets MPJPE after pelvis alignment + MPJPE after Procrustes.
+    Evaluates on the 14 common joints.
+    Inputs:
+      - gt3ds: N x 14 x 3
+      - preds: N x 14 x 3
+    """
+    errors, errors_pa = [], []
+    for i, (gt3d, pred) in enumerate(zip(gt3ds, preds)):
+        gt3d = gt3d.reshape(-1, 3)
+        # Root align.
+        gt3d = align_by_pelvis(gt3d)
+        pred3d = align_by_pelvis(pred)
+
+        joint_error = np.sqrt(np.sum((gt3d - pred3d)**2, axis=1))
+        errors.append(np.mean(joint_error))
+
+        # Get PA error.
+        pred3d_sym = compute_similarity_transform(pred3d, gt3d)
+        pa_error = np.sqrt(np.sum((gt3d - pred3d_sym)**2, axis=1))
+        errors_pa.append(np.mean(pa_error))
+
+    return errors, errors_pa