[Refactor] refactor evaluation for fourdag (#91)

* refactor evaluation for fourdag * fix bug of fourdag_associator
openxrlab · Mar 14, 2023 · 15a71a5 · 15a71a5
1 parent d81e48b
commit 15a71a5
Show file tree

Hide file tree

Showing 9 changed files with 354 additions and 195 deletions.
diff --git a/configs/fourdag/campus_config/eval_keypoints3d.py b/configs/fourdag/campus_config/eval_keypoints3d.py
@@ -7,11 +7,6 @@
 output_dir = './output/fourdag/CampusSeq1_fourdag_19_FourDAGOptimization/'
 pred_kps3d_convention = 'fourdag_19'
 eval_kps3d_convention = 'campus'
-selected_limbs_name = [
-    'left_lower_leg', 'right_lower_leg', 'left_upperarm', 'right_upperarm',
-    'left_forearm', 'right_forearm', 'left_thigh', 'right_thigh'
-]
-additional_limbs_names = [['jaw', 'headtop']]
 
 associator = dict(
     type='FourDAGAssociator',
@@ -60,7 +55,44 @@
     ),
     logger=logger,
 )
-
+metric_list = [
+    dict(
+        type='PredictionMatcher',
+        name='matching',
+    ),
+    dict(type='MPJPEMetric', name='mpjpe', unit_scale=1),
+    dict(type='PAMPJPEMetric', name='pa_mpjpe', unit_scale=1),
+    dict(
+        type='PCKMetric',
+        name='pck',
+        use_pa_mpjpe=True,
+        threshold=[50, 100],
+    ),
+    dict(
+        type='PCPMetric',
+        name='pcp',
+        threshold=0.5,
+        show_table=True,
+        selected_limbs_names=[
+            'left_lower_leg', 'right_lower_leg', 'left_upperarm',
+            'right_upperarm', 'left_forearm', 'right_forearm', 'left_thigh',
+            'right_thigh'
+        ],
+    ),
+    dict(
+        type='PrecisionRecallMetric',
+        name='precision_recall',
+        show_table=False,
+        threshold=list(range(25, 155, 25)) + [500],
+    )
+]
+pick_dict = dict(
+    mpjpe=['mpjpe_mean', 'mpjpe_std'],
+    pa_mpjpe=['pa_mpjpe_mean', 'pa_mpjpe_std'],
+    pck=['pck@50', 'pck@100'],
+    pcp=['pcp_total_mean'],
+    precision_recall=['recall@500'],
+)
 dataset = dict(
     type='BottomUpMviewMpersonDataset',
     data_root=__data_root__,

diff --git a/configs/fourdag/fourdag_config/eval_keypoints3d_seq2.py b/configs/fourdag/fourdag_config/eval_keypoints3d_seq2.py
@@ -7,11 +7,6 @@
 output_dir = './output/fourdag/fourdag_fourdag_19_FourDAGOptimization/'
 pred_kps3d_convention = 'fourdag_19'
 eval_kps3d_convention = 'campus'
-selected_limbs_name = [
-    'left_lower_leg', 'right_lower_leg', 'left_upperarm', 'right_upperarm',
-    'left_forearm', 'right_forearm', 'left_thigh', 'right_thigh'
-]
-# additional_limbs_names = [['jaw', 'headtop']]
 
 associator = dict(
     type='FourDAGAssociator',
@@ -60,7 +55,44 @@
     ),
     logger=logger,
 )
-
+metric_list = [
+    dict(
+        type='PredictionMatcher',
+        name='matching',
+    ),
+    dict(type='MPJPEMetric', name='mpjpe', unit_scale=1),
+    dict(type='PAMPJPEMetric', name='pa_mpjpe', unit_scale=1),
+    dict(
+        type='PCKMetric',
+        name='pck',
+        use_pa_mpjpe=True,
+        threshold=[50, 100],
+    ),
+    dict(
+        type='PCPMetric',
+        name='pcp',
+        threshold=0.5,
+        show_table=True,
+        selected_limbs_names=[
+            'left_lower_leg', 'right_lower_leg', 'left_upperarm',
+            'right_upperarm', 'left_forearm', 'right_forearm', 'left_thigh',
+            'right_thigh'
+        ],
+    ),
+    dict(
+        type='PrecisionRecallMetric',
+        name='precision_recall',
+        show_table=False,
+        threshold=list(range(25, 155, 25)) + [500],
+    )
+]
+pick_dict = dict(
+    mpjpe=['mpjpe_mean', 'mpjpe_std'],
+    pa_mpjpe=['pa_mpjpe_mean', 'pa_mpjpe_std'],
+    pck=['pck@50', 'pck@100'],
+    pcp=['pcp_total_mean'],
+    precision_recall=['recall@500'],
+)
 dataset = dict(
     type='BottomUpMviewMpersonDataset',
     data_root=__data_root__,

diff --git a/configs/fourdag/fourdag_config/eval_keypoints3d_seq4.py b/configs/fourdag/fourdag_config/eval_keypoints3d_seq4.py
@@ -7,11 +7,6 @@
 output_dir = './output/fourdag/fourdag_fourdag_19_FourDAGOptimization/'
 pred_kps3d_convention = 'fourdag_19'
 eval_kps3d_convention = 'campus'
-selected_limbs_name = [
-    'left_lower_leg', 'right_lower_leg', 'left_upperarm', 'right_upperarm',
-    'left_forearm', 'right_forearm', 'left_thigh', 'right_thigh'
-]
-# additional_limbs_names = [['jaw', 'headtop']]
 
 associator = dict(
     type='FourDAGAssociator',
@@ -60,7 +55,44 @@
     ),
     logger=logger,
 )
-
+metric_list = [
+    dict(
+        type='PredictionMatcher',
+        name='matching',
+    ),
+    dict(type='MPJPEMetric', name='mpjpe', unit_scale=1),
+    dict(type='PAMPJPEMetric', name='pa_mpjpe', unit_scale=1),
+    dict(
+        type='PCKMetric',
+        name='pck',
+        use_pa_mpjpe=True,
+        threshold=[50, 100],
+    ),
+    dict(
+        type='PCPMetric',
+        name='pcp',
+        threshold=0.5,
+        show_table=True,
+        selected_limbs_names=[
+            'left_lower_leg', 'right_lower_leg', 'left_upperarm',
+            'right_upperarm', 'left_forearm', 'right_forearm', 'left_thigh',
+            'right_thigh'
+        ],
+    ),
+    dict(
+        type='PrecisionRecallMetric',
+        name='precision_recall',
+        show_table=False,
+        threshold=list(range(25, 155, 25)) + [500],
+    )
+]
+pick_dict = dict(
+    mpjpe=['mpjpe_mean', 'mpjpe_std'],
+    pa_mpjpe=['pa_mpjpe_mean', 'pa_mpjpe_std'],
+    pck=['pck@50', 'pck@100'],
+    pcp=['pcp_total_mean'],
+    precision_recall=['recall@500'],
+)
 dataset = dict(
     type='BottomUpMviewMpersonDataset',
     data_root=__data_root__,

diff --git a/configs/fourdag/fourdag_config/eval_keypoints3d_seq5.py b/configs/fourdag/fourdag_config/eval_keypoints3d_seq5.py
@@ -7,11 +7,6 @@
 output_dir = './output/fourdag/fourdag_fourdag_19_FourDAGOptimization/'
 pred_kps3d_convention = 'fourdag_19'
 eval_kps3d_convention = 'campus'
-selected_limbs_name = [
-    'left_lower_leg', 'right_lower_leg', 'left_upperarm', 'right_upperarm',
-    'left_forearm', 'right_forearm', 'left_thigh', 'right_thigh'
-]
-# additional_limbs_names = [['jaw', 'headtop']]
 
 associator = dict(
     type='FourDAGAssociator',
@@ -60,7 +55,44 @@
     ),
     logger=logger,
 )
-
+metric_list = [
+    dict(
+        type='PredictionMatcher',
+        name='matching',
+    ),
+    dict(type='MPJPEMetric', name='mpjpe', unit_scale=1),
+    dict(type='PAMPJPEMetric', name='pa_mpjpe', unit_scale=1),
+    dict(
+        type='PCKMetric',
+        name='pck',
+        use_pa_mpjpe=True,
+        threshold=[50, 100],
+    ),
+    dict(
+        type='PCPMetric',
+        name='pcp',
+        threshold=0.5,
+        show_table=True,
+        selected_limbs_names=[
+            'left_lower_leg', 'right_lower_leg', 'left_upperarm',
+            'right_upperarm', 'left_forearm', 'right_forearm', 'left_thigh',
+            'right_thigh'
+        ],
+    ),
+    dict(
+        type='PrecisionRecallMetric',
+        name='precision_recall',
+        show_table=False,
+        threshold=list(range(25, 155, 25)) + [500],
+    )
+]
+pick_dict = dict(
+    mpjpe=['mpjpe_mean', 'mpjpe_std'],
+    pa_mpjpe=['pa_mpjpe_mean', 'pa_mpjpe_std'],
+    pck=['pck@50', 'pck@100'],
+    pcp=['pcp_total_mean'],
+    precision_recall=['recall@500'],
+)
 dataset = dict(
     type='BottomUpMviewMpersonDataset',
     data_root=__data_root__,

diff --git a/configs/fourdag/shelf_config/eval_keypoints3d.py b/configs/fourdag/shelf_config/eval_keypoints3d.py
@@ -7,11 +7,6 @@
 output_dir = './output/fourdag/shelf_fourdag_19_FourDAGOptimization/'
 pred_kps3d_convention = 'fourdag_19'
 eval_kps3d_convention = 'campus'
-selected_limbs_name = [
-    'left_lower_leg', 'right_lower_leg', 'left_upperarm', 'right_upperarm',
-    'left_forearm', 'right_forearm', 'left_thigh', 'right_thigh'
-]
-additional_limbs_names = [['jaw', 'headtop']]
 
 associator = dict(
     type='FourDAGAssociator',
@@ -76,6 +71,46 @@
     cam_world2cam=True,
 )
 
+metric_list = [
+    dict(
+        type='PredictionMatcher',
+        name='matching',
+    ),
+    dict(type='MPJPEMetric', name='mpjpe', unit_scale=1),
+    dict(type='PAMPJPEMetric', name='pa_mpjpe', unit_scale=1),
+    dict(
+        type='PCKMetric',
+        name='pck',
+        use_pa_mpjpe=True,
+        threshold=[50, 100],
+    ),
+    dict(
+        type='PCPMetric',
+        name='pcp',
+        threshold=0.5,
+        show_table=True,
+        selected_limbs_names=[
+            'left_lower_leg', 'right_lower_leg', 'left_upperarm',
+            'right_upperarm', 'left_forearm', 'right_forearm', 'left_thigh',
+            'right_thigh'
+        ],
+        additional_limbs_names=[['jaw', 'headtop']],
+    ),
+    dict(
+        type='PrecisionRecallMetric',
+        name='precision_recall',
+        show_table=False,
+        threshold=list(range(25, 155, 25)) + [500],
+    )
+]
+pick_dict = dict(
+    mpjpe=['mpjpe_mean', 'mpjpe_std'],
+    pa_mpjpe=['pa_mpjpe_mean', 'pa_mpjpe_std'],
+    pck=['pck@50', 'pck@100'],
+    pcp=['pcp_total_mean'],
+    precision_recall=['recall@500'],
+)
+
 dataset_visualization = dict(
     type='MviewMpersonDataVisualization',
     data_root=__data_root__,

diff --git a/configs/modules/core/evaluation/bottom_up_eval_shelf_unittest.py b/configs/modules/core/evaluation/bottom_up_eval_shelf_unittest.py
@@ -7,11 +7,6 @@
 output_dir = 'tests/data/output/core/test_mvpose_evaluation'
 pred_kps3d_convention = 'fourdag_19'
 eval_kps3d_convention = 'campus'
-selected_limbs_name = [
-    'left_lower_leg', 'right_lower_leg', 'left_upperarm', 'right_upperarm',
-    'left_forearm', 'right_forearm', 'left_thigh', 'right_thigh'
-]
-additional_limbs_names = [['jaw', 'headtop']]
 
 associator = dict(
     type='FourDAGAssociator',
@@ -61,6 +56,46 @@
     logger=logger,
 )
 
+metric_list = [
+    dict(
+        type='PredictionMatcher',
+        name='matching',
+    ),
+    dict(type='MPJPEMetric', name='mpjpe', unit_scale=1),
+    dict(type='PAMPJPEMetric', name='pa_mpjpe', unit_scale=1),
+    dict(
+        type='PCKMetric',
+        name='pck',
+        use_pa_mpjpe=True,
+        threshold=[50, 100],
+    ),
+    dict(
+        type='PCPMetric',
+        name='pcp',
+        threshold=0.5,
+        show_table=True,
+        selected_limbs_names=[
+            'left_lower_leg', 'right_lower_leg', 'left_upperarm',
+            'right_upperarm', 'left_forearm', 'right_forearm', 'left_thigh',
+            'right_thigh'
+        ],
+        additional_limbs_names=[['jaw', 'headtop']],
+    ),
+    dict(
+        type='PrecisionRecallMetric',
+        name='precision_recall',
+        show_table=False,
+        threshold=list(range(25, 155, 25)) + [500],
+    )
+]
+pick_dict = dict(
+    mpjpe=['mpjpe_mean', 'mpjpe_std'],
+    pa_mpjpe=['pa_mpjpe_mean', 'pa_mpjpe_std'],
+    pck=['pck@50', 'pck@100'],
+    pcp=['pcp_total_mean'],
+    precision_recall=['recall@500'],
+)
+
 dataset = dict(
     type='BottomUpMviewMpersonDataset',
     data_root=__data_root__,