{ "version": 1, "study": "FVS detector external validation v1", "corpus_size": 12, "detectable_frames": [ "FVS-001", "FVS-002", "FVS-007", "FVS-008", "FVS-009", "FVS-010", "FVS-011", "FVS-012", "FVS-014", "FVS-015", "FVS-016" ], "per_frame": { "FVS-001": { "n_A_exhibits": 1, "n_B_exhibits": 0, "n_C_exhibits": 4, "kappa_AB": 0.0, "kappa_AC": -0.154, "kappa_BC": -0.0, "f1_C_vs_A": 0.0, "f1_C_vs_B": 0.0, "f1_C_vs_majority_union": 0.0, "f1_C_vs_majority_intersection": 0.0, "precision_C_vs_majority_union": 0.0, "recall_C_vs_majority_union": 0.0, "tp_fp_fn_tn_C_vs_majority_union": [ 0, 4, 1, 7 ] }, "FVS-002": { "n_A_exhibits": 1, "n_B_exhibits": 7, "n_C_exhibits": 1, "kappa_AB": 0.122, "kappa_AC": 1.0, "kappa_BC": 0.122, "f1_C_vs_A": 1.0, "f1_C_vs_B": 0.25, "f1_C_vs_majority_union": 0.25, "f1_C_vs_majority_intersection": 1.0, "precision_C_vs_majority_union": 1.0, "recall_C_vs_majority_union": 0.143, "tp_fp_fn_tn_C_vs_majority_union": [ 1, 0, 6, 5 ] }, "FVS-007": { "n_A_exhibits": 1, "n_B_exhibits": 0, "n_C_exhibits": 3, "kappa_AB": 0.0, "kappa_AC": -0.143, "kappa_BC": 0.0, "f1_C_vs_A": 0.0, "f1_C_vs_B": 0.0, "f1_C_vs_majority_union": 0.0, "f1_C_vs_majority_intersection": 0.0, "precision_C_vs_majority_union": 0.0, "recall_C_vs_majority_union": 0.0, "tp_fp_fn_tn_C_vs_majority_union": [ 0, 3, 1, 8 ] }, "FVS-008": { "n_A_exhibits": 2, "n_B_exhibits": 3, "n_C_exhibits": 3, "kappa_AB": 0.75, "kappa_AC": -0.25, "kappa_BC": -0.333, "f1_C_vs_A": 0.0, "f1_C_vs_B": 0.0, "f1_C_vs_majority_union": 0.0, "f1_C_vs_majority_intersection": 0.0, "precision_C_vs_majority_union": 0.0, "recall_C_vs_majority_union": 0.0, "tp_fp_fn_tn_C_vs_majority_union": [ 0, 3, 3, 6 ] }, "FVS-009": { "n_A_exhibits": 3, "n_B_exhibits": 9, "n_C_exhibits": 2, "kappa_AB": 0.2, "kappa_AC": 0.25, "kappa_BC": 0.125, "f1_C_vs_A": 0.4, "f1_C_vs_B": 0.364, "f1_C_vs_majority_union": 0.364, "f1_C_vs_majority_intersection": 0.4, "precision_C_vs_majority_union": 1.0, "recall_C_vs_majority_union": 0.222, "tp_fp_fn_tn_C_vs_majority_union": [ 2, 0, 7, 3 ] }, "FVS-010": { "n_A_exhibits": 2, "n_B_exhibits": 8, "n_C_exhibits": 2, "kappa_AB": 0.182, "kappa_AC": 0.4, "kappa_BC": -0.091, "f1_C_vs_A": 0.5, "f1_C_vs_B": 0.2, "f1_C_vs_majority_union": 0.2, "f1_C_vs_majority_intersection": 0.5, "precision_C_vs_majority_union": 0.5, "recall_C_vs_majority_union": 0.125, "tp_fp_fn_tn_C_vs_majority_union": [ 1, 1, 7, 3 ] }, "FVS-011": { "n_A_exhibits": 3, "n_B_exhibits": 2, "n_C_exhibits": 7, "kappa_AB": 0.25, "kappa_AC": 0.385, "kappa_BC": 0.25, "f1_C_vs_A": 0.6, "f1_C_vs_B": 0.444, "f1_C_vs_majority_union": 0.727, "f1_C_vs_majority_intersection": 0.25, "precision_C_vs_majority_union": 0.571, "recall_C_vs_majority_union": 1.0, "tp_fp_fn_tn_C_vs_majority_union": [ 4, 3, 0, 5 ] }, "FVS-012": { "n_A_exhibits": 4, "n_B_exhibits": 9, "n_C_exhibits": 2, "kappa_AB": 0.286, "kappa_AC": 0.143, "kappa_BC": -0.125, "f1_C_vs_A": 0.333, "f1_C_vs_B": 0.182, "f1_C_vs_majority_union": 0.182, "f1_C_vs_majority_intersection": 0.333, "precision_C_vs_majority_union": 0.5, "recall_C_vs_majority_union": 0.111, "tp_fp_fn_tn_C_vs_majority_union": [ 1, 1, 8, 2 ] }, "FVS-014": { "n_A_exhibits": 4, "n_B_exhibits": 7, "n_C_exhibits": 0, "kappa_AB": 0.211, "kappa_AC": -0.0, "kappa_BC": 0.0, "f1_C_vs_A": 0.0, "f1_C_vs_B": 0.0, "f1_C_vs_majority_union": 0.0, "f1_C_vs_majority_intersection": 0.0, "precision_C_vs_majority_union": 0.0, "recall_C_vs_majority_union": 0.0, "tp_fp_fn_tn_C_vs_majority_union": [ 0, 0, 8, 4 ] }, "FVS-015": { "n_A_exhibits": 1, "n_B_exhibits": 1, "n_C_exhibits": 2, "kappa_AB": 1.0, "kappa_AC": -0.125, "kappa_BC": -0.125, "f1_C_vs_A": 0.0, "f1_C_vs_B": 0.0, "f1_C_vs_majority_union": 0.0, "f1_C_vs_majority_intersection": 0.0, "precision_C_vs_majority_union": 0.0, "recall_C_vs_majority_union": 0.0, "tp_fp_fn_tn_C_vs_majority_union": [ 0, 2, 1, 9 ] }, "FVS-016": { "n_A_exhibits": 6, "n_B_exhibits": 3, "n_C_exhibits": 0, "kappa_AB": 0.167, "kappa_AC": 0.0, "kappa_BC": 0.0, "f1_C_vs_A": 0.0, "f1_C_vs_B": 0.0, "f1_C_vs_majority_union": 0.0, "f1_C_vs_majority_intersection": 0.0, "precision_C_vs_majority_union": 0.0, "recall_C_vs_majority_union": 0.0, "tp_fp_fn_tn_C_vs_majority_union": [ 0, 0, 7, 5 ] } }, "per_document": { "a01_altman_intelligence_age": { "stratum": "A", "n_A_exhibits": 6, "n_B_exhibits": 5, "n_C_exhibits": 1, "kappa_AB": 0.459, "kappa_AC": 0.154, "kappa_BC": 0.214, "f1_C_vs_majority_union": 0.25, "f1_C_vs_majority_intersection": 0.4, "f1_C_vs_A": 0.286, "f1_C_vs_B": 0.333, "A_labels": [ "FVS-001", "FVS-002", "FVS-007", "FVS-008", "FVS-010", "FVS-014" ], "B_labels": [ "FVS-002", "FVS-008", "FVS-010", "FVS-012", "FVS-014" ], "C_labels": [ "FVS-002" ] }, "a02_fomc_march_2026": { "stratum": "A", "n_A_exhibits": 2, "n_B_exhibits": 3, "n_C_exhibits": 2, "kappa_AB": 0.744, "kappa_AC": 1.0, "kappa_BC": 0.744, "f1_C_vs_majority_union": 0.8, "f1_C_vs_majority_intersection": 1.0, "f1_C_vs_A": 1.0, "f1_C_vs_B": 0.8, "A_labels": [ "FVS-009", "FVS-012" ], "B_labels": [ "FVS-009", "FVS-012", "FVS-014" ], "C_labels": [ "FVS-009", "FVS-012" ] }, "b01_nvidia_investment": { "stratum": "B", "n_A_exhibits": 3, "n_B_exhibits": 7, "n_C_exhibits": 2, "kappa_AB": 0.353, "kappa_AC": -0.279, "kappa_BC": -0.394, "f1_C_vs_majority_union": 0.0, "f1_C_vs_majority_intersection": 0.0, "f1_C_vs_A": 0.0, "f1_C_vs_B": 0.0, "A_labels": [ "FVS-008", "FVS-009", "FVS-012" ], "B_labels": [ "FVS-002", "FVS-008", "FVS-009", "FVS-010", "FVS-012", "FVS-014", "FVS-016" ], "C_labels": [ "FVS-001", "FVS-011" ] }, "b02_automation_policy": { "stratum": "B", "n_A_exhibits": 2, "n_B_exhibits": 5, "n_C_exhibits": 4, "kappa_AB": 0.421, "kappa_AC": 0.56, "kappa_BC": 0.441, "f1_C_vs_majority_union": 0.667, "f1_C_vs_majority_intersection": 0.667, "f1_C_vs_A": 0.667, "f1_C_vs_B": 0.667, "A_labels": [ "FVS-010", "FVS-011" ], "B_labels": [ "FVS-002", "FVS-009", "FVS-010", "FVS-011", "FVS-014" ], "C_labels": [ "FVS-009", "FVS-010", "FVS-011", "FVS-012" ] }, "b03_social_media_adolescents": { "stratum": "B", "n_A_exhibits": 2, "n_B_exhibits": 6, "n_C_exhibits": 1, "kappa_AB": 0.312, "kappa_AC": -0.138, "kappa_BC": -0.185, "f1_C_vs_majority_union": 0.0, "f1_C_vs_majority_intersection": 0.0, "f1_C_vs_A": 0.0, "f1_C_vs_B": 0.0, "A_labels": [ "FVS-012", "FVS-016" ], "B_labels": [ "FVS-002", "FVS-009", "FVS-010", "FVS-012", "FVS-014", "FVS-016" ], "C_labels": [ "FVS-011" ] }, "b04_llm_customer_support": { "stratum": "B", "n_A_exhibits": 2, "n_B_exhibits": 5, "n_C_exhibits": 2, "kappa_AB": 0.035, "kappa_AC": 0.389, "kappa_BC": -0.351, "f1_C_vs_majority_union": 0.25, "f1_C_vs_majority_intersection": 0.0, "f1_C_vs_A": 0.5, "f1_C_vs_B": 0.0, "A_labels": [ "FVS-011", "FVS-015" ], "B_labels": [ "FVS-002", "FVS-008", "FVS-009", "FVS-010", "FVS-015" ], "C_labels": [ "FVS-001", "FVS-011" ] }, "b05_remote_work_productivity": { "stratum": "B", "n_A_exhibits": 1, "n_B_exhibits": 3, "n_C_exhibits": 4, "kappa_AB": -0.158, "kappa_AC": 0.298, "kappa_BC": -0.453, "f1_C_vs_majority_union": 0.25, "f1_C_vs_majority_intersection": 0.0, "f1_C_vs_A": 0.4, "f1_C_vs_B": 0.0, "A_labels": [ "FVS-011" ], "B_labels": [ "FVS-002", "FVS-010", "FVS-012" ], "C_labels": [ "FVS-001", "FVS-007", "FVS-008", "FVS-011" ] }, "b06_quantum_computing_outlook": { "stratum": "B", "n_A_exhibits": 3, "n_B_exhibits": 6, "n_C_exhibits": 2, "kappa_AB": 0.476, "kappa_AC": -0.279, "kappa_BC": -0.375, "f1_C_vs_majority_union": 0.0, "f1_C_vs_majority_intersection": 0.0, "f1_C_vs_A": 0.0, "f1_C_vs_B": 0.0, "A_labels": [ "FVS-012", "FVS-014", "FVS-016" ], "B_labels": [ "FVS-002", "FVS-009", "FVS-010", "FVS-012", "FVS-014", "FVS-016" ], "C_labels": [ "FVS-001", "FVS-015" ] }, "c01_wikipedia_semaglutide": { "stratum": "C", "n_A_exhibits": 1, "n_B_exhibits": 2, "n_C_exhibits": 1, "kappa_AB": -0.138, "kappa_AC": -0.1, "kappa_BC": -0.138, "f1_C_vs_majority_union": 0.0, "f1_C_vs_majority_intersection": 0.0, "f1_C_vs_A": 0.0, "f1_C_vs_B": 0.0, "A_labels": [ "FVS-016" ], "B_labels": [ "FVS-009", "FVS-012" ], "C_labels": [ "FVS-015" ] }, "c02_wikipedia_eu_ai_act": { "stratum": "C", "n_A_exhibits": 2, "n_B_exhibits": 2, "n_C_exhibits": 2, "kappa_AB": 0.389, "kappa_AC": -0.222, "kappa_BC": 0.389, "f1_C_vs_majority_union": 0.4, "f1_C_vs_majority_intersection": 0.0, "f1_C_vs_A": 0.0, "f1_C_vs_B": 0.5, "A_labels": [ "FVS-009", "FVS-016" ], "B_labels": [ "FVS-009", "FVS-011" ], "C_labels": [ "FVS-010", "FVS-011" ] }, "c03_wikipedia_quantum_supremacy": { "stratum": "C", "n_A_exhibits": 2, "n_B_exhibits": 3, "n_C_exhibits": 2, "kappa_AB": 0.233, "kappa_AC": -0.222, "kappa_BC": -0.279, "f1_C_vs_majority_union": 0.0, "f1_C_vs_majority_intersection": 0.0, "f1_C_vs_A": 0.0, "f1_C_vs_B": 0.0, "A_labels": [ "FVS-014", "FVS-016" ], "B_labels": [ "FVS-009", "FVS-012", "FVS-014" ], "C_labels": [ "FVS-007", "FVS-008" ] }, "c04_wikipedia_ubi": { "stratum": "C", "n_A_exhibits": 2, "n_B_exhibits": 2, "n_C_exhibits": 3, "kappa_AB": -0.222, "kappa_AC": -0.279, "kappa_BC": -0.279, "f1_C_vs_majority_union": 0.0, "f1_C_vs_majority_intersection": 0.0, "f1_C_vs_A": 0.0, "f1_C_vs_B": 0.0, "A_labels": [ "FVS-014", "FVS-016" ], "B_labels": [ "FVS-010", "FVS-012" ], "C_labels": [ "FVS-007", "FVS-008", "FVS-011" ] } }, "per_stratum_macro_f1": { "A": 0.525, "B": 0.195, "C": 0.1 }, "per_doc_AB_kappa_all_frames": { "a01_altman_intelligence_age": 0.583, "a02_fomc_march_2026": 0.615, "b01_nvidia_investment": 0.3, "b02_automation_policy": 0.286, "b03_social_media_adolescents": 0.239, "b04_llm_customer_support": 0.048, "b05_remote_work_productivity": -0.094, "b06_quantum_computing_outlook": 0.355, "c01_wikipedia_semaglutide": -0.071, "c02_wikipedia_eu_ai_act": 0.231, "c03_wikipedia_quantum_supremacy": 0.118, "c04_wikipedia_ubi": 0.138 }, "aggregate": { "macro_kappa_AB_detectable_frames": 0.288, "macro_kappa_AC_detectable_frames": 0.137, "macro_kappa_BC_detectable_frames": -0.016, "macro_f1_C_vs_A": 0.258, "macro_f1_C_vs_B": 0.131, "macro_f1_C_vs_majority_union": 0.157, "macro_f1_C_vs_majority_intersection": 0.226, "mean_kappa_AB_all_20_frames": 0.229 }, "verdict": { "macro_f1_C_vs_majority_union": 0.157, "assessment": "F1 < 0.4: detector does not reliably measure what multi-source labeling identifies. H3 fires; project pivot required." } }