{ "baseline_model": "baseline", "cadviga_model": "cadviga", "notes": [ "Failed attempts are excluded from numeric means, variances, and t-tests.", "Welch tests compare successful attempt-level metric distributions.", "Paired tests compare per-sample means for samples with at least one successful attempt in both models.", "Variance fields report sample variance unless explicitly named population_variance." ], "scopes": { "global": { "metrics": { "voxel_iou": { "global_mean_variance": { "baseline": { "n": 17, "mean": 0.4126602353, "sample_variance": 0.0786298661, "population_variance": 0.0740045799 }, "cadviga": { "n": 45, "mean": 0.3800051556, "sample_variance": 0.087418397, "population_variance": 0.0854757659 } }, "welch_t_test_attempts": { "test": "welch_two_sample_t_test", "available": true, "alternative": "two-sided", "mean_difference_cadviga_minus_baseline": -0.0326550797, "t_statistic": -0.4029367796, "degrees_of_freedom": 30.317739467, "p_value": 0.6898221505, "baseline": { "n": 17, "mean": 0.4126602353, "sample_variance": 0.0786298661, "population_variance": 0.0740045799 }, "cadviga": { "n": 45, "mean": 0.3800051556, "sample_variance": 0.087418397, "population_variance": 0.0854757659 } }, "paired_t_test_sample_means": { "test": "paired_t_test_on_sample_means", "available": true, "alternative": "two-sided", "n_pairs": 11, "mean_difference_cadviga_minus_baseline": -0.0814410758, "sample_variance_of_differences": 0.0459250614, "t_statistic": -1.2604190873, "degrees_of_freedom": 10, "p_value": 0.236134404, "paired_record_ids": [ "hinge_000914_s20260506", "j_hook_000905_s20260506", "knob_000898_s20260506", "knob_001348_s20260506", "knob_001352_s20260506", "pillow_block_001313_s20260506", "pipe_elbow_000974_s20260506", "tee_nut_001166_s20260506", "turnbuckle_001266_s20260506", "turnbuckle_001275_s20260506", "u_bolt_001112_s20260506" ] } }, "composite_score": { "global_mean_variance": { "baseline": { "n": 17, "mean": 0.5236411765, "sample_variance": 0.0381580113, "population_variance": 0.0359134224 }, "cadviga": { "n": 45, "mean": 0.5072888889, "sample_variance": 0.056569936, "population_variance": 0.0553128263 } }, "welch_t_test_attempts": { "test": "welch_two_sample_t_test", "available": true, "alternative": "two-sided", "mean_difference_cadviga_minus_baseline": -0.0163522876, "t_statistic": -0.2763370618, "degrees_of_freedom": 34.9538179092, "p_value": 0.7839166158, "baseline": { "n": 17, "mean": 0.5236411765, "sample_variance": 0.0381580113, "population_variance": 0.0359134224 }, "cadviga": { "n": 45, "mean": 0.5072888889, "sample_variance": 0.056569936, "population_variance": 0.0553128263 } }, "paired_t_test_sample_means": { "test": "paired_t_test_on_sample_means", "available": true, "alternative": "two-sided", "n_pairs": 11, "mean_difference_cadviga_minus_baseline": -0.0639, "sample_variance_of_differences": 0.0213887499, "t_statistic": -1.4491204963, "degrees_of_freedom": 10, "p_value": 0.1779273077, "paired_record_ids": [ "hinge_000914_s20260506", "j_hook_000905_s20260506", "knob_000898_s20260506", "knob_001348_s20260506", "knob_001352_s20260506", "pillow_block_001313_s20260506", "pipe_elbow_000974_s20260506", "tee_nut_001166_s20260506", "turnbuckle_001266_s20260506", "turnbuckle_001275_s20260506", "u_bolt_001112_s20260506" ] } } } }, "easy": { "metrics": { "voxel_iou": { "global_mean_variance": { "baseline": { "n": 8, "mean": 0.609924625, "sample_variance": 0.0488845373, "population_variance": 0.0427739702 }, "cadviga": { "n": 15, "mean": 0.4540467333, "sample_variance": 0.0322649032, "population_variance": 0.0301139096 } }, "welch_t_test_attempts": { "test": "welch_two_sample_t_test", "available": true, "alternative": "two-sided", "mean_difference_cadviga_minus_baseline": -0.1558778917, "t_statistic": -1.7149579422, "degrees_of_freedom": 12.0490432934, "p_value": 0.1119329254, "baseline": { "n": 8, "mean": 0.609924625, "sample_variance": 0.0488845373, "population_variance": 0.0427739702 }, "cadviga": { "n": 15, "mean": 0.4540467333, "sample_variance": 0.0322649032, "population_variance": 0.0301139096 } }, "paired_t_test_sample_means": { "test": "paired_t_test_on_sample_means", "available": true, "alternative": "two-sided", "n_pairs": 5, "mean_difference_cadviga_minus_baseline": -0.1131691667, "sample_variance_of_differences": 0.0249003028, "t_statistic": -1.603654494, "degrees_of_freedom": 4, "p_value": 0.184055162, "paired_record_ids": [ "hinge_000914_s20260506", "knob_001352_s20260506", "tee_nut_001166_s20260506", "turnbuckle_001275_s20260506", "u_bolt_001112_s20260506" ] } }, "composite_score": { "global_mean_variance": { "baseline": { "n": 8, "mean": 0.6409, "sample_variance": 0.0399054686, "population_variance": 0.034917285 }, "cadviga": { "n": 15, "mean": 0.5435333333, "sample_variance": 0.0267455567, "population_variance": 0.0249625196 } }, "welch_t_test_attempts": { "test": "welch_two_sample_t_test", "available": true, "alternative": "two-sided", "mean_difference_cadviga_minus_baseline": -0.0973666667, "t_statistic": -1.183250831, "degrees_of_freedom": 12.1241699147, "p_value": 0.2593923875, "baseline": { "n": 8, "mean": 0.6409, "sample_variance": 0.0399054686, "population_variance": 0.034917285 }, "cadviga": { "n": 15, "mean": 0.5435333333, "sample_variance": 0.0267455567, "population_variance": 0.0249625196 } }, "paired_t_test_sample_means": { "test": "paired_t_test_on_sample_means", "available": true, "alternative": "two-sided", "n_pairs": 5, "mean_difference_cadviga_minus_baseline": -0.0907766667, "sample_variance_of_differences": 0.0134474702, "t_statistic": -1.7504065826, "degrees_of_freedom": 4, "p_value": 0.1549426151, "paired_record_ids": [ "hinge_000914_s20260506", "knob_001352_s20260506", "tee_nut_001166_s20260506", "turnbuckle_001275_s20260506", "u_bolt_001112_s20260506" ] } } } }, "medium": { "metrics": { "voxel_iou": { "global_mean_variance": { "baseline": { "n": 6, "mean": 0.3112856667, "sample_variance": 0.0394380574, "population_variance": 0.0328650479 }, "cadviga": { "n": 15, "mean": 0.2711602667, "sample_variance": 0.0606876609, "population_variance": 0.0566418168 } }, "welch_t_test_attempts": { "test": "welch_two_sample_t_test", "available": true, "alternative": "two-sided", "mean_difference_cadviga_minus_baseline": -0.0401254, "t_statistic": -0.3893862015, "degrees_of_freedom": 11.4942877056, "p_value": 0.7041082452, "baseline": { "n": 6, "mean": 0.3112856667, "sample_variance": 0.0394380574, "population_variance": 0.0328650479 }, "cadviga": { "n": 15, "mean": 0.2711602667, "sample_variance": 0.0606876609, "population_variance": 0.0566418168 } }, "paired_t_test_sample_means": { "test": "paired_t_test_on_sample_means", "available": true, "alternative": "two-sided", "n_pairs": 4, "mean_difference_cadviga_minus_baseline": -0.097041375, "sample_variance_of_differences": 0.1092572708, "t_statistic": -0.5871671676, "degrees_of_freedom": 3, "p_value": 0.5983558991, "paired_record_ids": [ "j_hook_000905_s20260506", "knob_000898_s20260506", "knob_001348_s20260506", "pipe_elbow_000974_s20260506" ] } }, "composite_score": { "global_mean_variance": { "baseline": { "n": 6, "mean": 0.4502333333, "sample_variance": 0.0187762667, "population_variance": 0.0156468889 }, "cadviga": { "n": 15, "mean": 0.41466, "sample_variance": 0.0345533069, "population_variance": 0.0322497531 } }, "welch_t_test_attempts": { "test": "welch_two_sample_t_test", "available": true, "alternative": "two-sided", "mean_difference_cadviga_minus_baseline": -0.0355733333, "t_statistic": -0.4826223661, "degrees_of_freedom": 12.6267998601, "p_value": 0.6376231023, "baseline": { "n": 6, "mean": 0.4502333333, "sample_variance": 0.0187762667, "population_variance": 0.0156468889 }, "cadviga": { "n": 15, "mean": 0.41466, "sample_variance": 0.0345533069, "population_variance": 0.0322497531 } }, "paired_t_test_sample_means": { "test": "paired_t_test_on_sample_means", "available": true, "alternative": "two-sided", "n_pairs": 4, "mean_difference_cadviga_minus_baseline": -0.0756958333, "sample_variance_of_differences": 0.0455319745, "t_statistic": -0.7094858333, "degrees_of_freedom": 3, "p_value": 0.5291941342, "paired_record_ids": [ "j_hook_000905_s20260506", "knob_000898_s20260506", "knob_001348_s20260506", "pipe_elbow_000974_s20260506" ] } } } }, "hard": { "metrics": { "voxel_iou": { "global_mean_variance": { "baseline": { "n": 3, "mean": 0.089371, "sample_variance": 0.0160906432, "population_variance": 0.0107270955 }, "cadviga": { "n": 15, "mean": 0.4148084667, "sample_variance": 0.1619260026, "population_variance": 0.1511309358 } }, "welch_t_test_attempts": { "test": "welch_two_sample_t_test", "available": true, "alternative": "two-sided", "mean_difference_cadviga_minus_baseline": 0.3254374667, "t_statistic": 2.5601504604, "degrees_of_freedom": 11.4983684028, "p_value": 0.0257192239, "baseline": { "n": 3, "mean": 0.089371, "sample_variance": 0.0160906432, "population_variance": 0.0107270955 }, "cadviga": { "n": 15, "mean": 0.4148084667, "sample_variance": 0.1619260026, "population_variance": 0.1511309358 } }, "paired_t_test_sample_means": { "test": "paired_t_test_on_sample_means", "available": true, "alternative": "two-sided", "n_pairs": 2, "mean_difference_cadviga_minus_baseline": 0.02907975, "sample_variance_of_differences": 0.0014410486, "t_statistic": 1.0833439404, "degrees_of_freedom": 1, "p_value": 0.4745456706, "paired_record_ids": [ "pillow_block_001313_s20260506", "turnbuckle_001266_s20260506" ] } }, "composite_score": { "global_mean_variance": { "baseline": { "n": 3, "mean": 0.3577666667, "sample_variance": 0.0062180933, "population_variance": 0.0041453956 }, "cadviga": { "n": 15, "mean": 0.5636733333, "sample_variance": 0.1024856035, "population_variance": 0.09565323 } }, "welch_t_test_attempts": { "test": "welch_two_sample_t_test", "available": true, "alternative": "two-sided", "mean_difference_cadviga_minus_baseline": 0.2059066667, "t_statistic": 2.1819847271, "degrees_of_freedom": 14.464472839, "p_value": 0.0460599401, "baseline": { "n": 3, "mean": 0.3577666667, "sample_variance": 0.0062180933, "population_variance": 0.0041453956 }, "cadviga": { "n": 15, "mean": 0.5636733333, "sample_variance": 0.1024856035, "population_variance": 0.09565323 } }, "paired_t_test_sample_means": { "test": "paired_t_test_on_sample_means", "available": true, "alternative": "two-sided", "n_pairs": 2, "mean_difference_cadviga_minus_baseline": 0.0268833333, "sample_variance_of_differences": 0.002850125, "t_statistic": 0.7121412804, "degrees_of_freedom": 1, "p_value": 0.6060418071, "paired_record_ids": [ "pillow_block_001313_s20260506", "turnbuckle_001266_s20260506" ] } } } } } }