{
  "commit": "agent-v1",
  "default": {
    "retrieval_mode": "hybrid",
    "rewrite_mode": "rule",
    "rerank_mode": "off",
    "top_k": 4,
    "context_token_budget": 5000,
    "embedding": "bge-m3:567m",
    "generation": "qwen2.5:7b-instruct",
    "prompt": "grounded-v5-conflict-gated"
  },
  "retrieval": {
    "bm25": {
      "recall_at_5": 0.9839,
      "recall_at_10": 0.9919,
      "mrr": 0.9274,
      "ndcg_at_10": 0.9436
    },
    "dense": {
      "recall_at_5": 0.9032,
      "recall_at_10": 0.9355,
      "mrr": 0.8395,
      "ndcg_at_10": 0.8611
    },
    "hybrid": {
      "recall_at_5": 0.9355,
      "recall_at_10": 0.9758,
      "mrr": 0.9003,
      "ndcg_at_10": 0.9185
    },
    "hybrid_rerank": {
      "recall_at_5": 0.9677,
      "recall_at_10": 0.9758,
      "mrr": 0.9351,
      "ndcg_at_10": 0.9446
    }
  },
  "dev": {
    "bm25": {
      "questions": 147,
      "status_correct": 0.8435,
      "literal_fact_coverage": 0.8462,
      "conflict_recall": "10/13",
      "conflict_precision": "10/10",
      "permission_violations": 0,
      "hidden_source_leaks": 0,
      "citation_identity_all_valid": true,
      "median_seconds": 18.1,
      "p95_seconds": 40.1
    },
    "dense": {
      "questions": 147,
      "status_correct": 0.8503,
      "literal_fact_coverage": 0.8205,
      "conflict_recall": "9/13",
      "conflict_precision": "9/9",
      "permission_violations": 0,
      "hidden_source_leaks": 0,
      "citation_identity_all_valid": true,
      "median_seconds": 26.8,
      "p95_seconds": 59.8
    },
    "hybrid": {
      "questions": 147,
      "status_correct": 0.8912,
      "literal_fact_coverage": 0.891,
      "conflict_recall": "11/13",
      "conflict_precision": "11/11",
      "permission_violations": 0,
      "hidden_source_leaks": 0,
      "citation_identity_all_valid": true,
      "median_seconds": 24.5,
      "p95_seconds": 45.7
    },
    "hybrid_rerank": {
      "questions": 147,
      "status_correct": 0.9048,
      "literal_fact_coverage": 0.9167,
      "conflict_recall": "12/13",
      "conflict_precision": "12/13",
      "permission_violations": 0,
      "hidden_source_leaks": 0,
      "citation_identity_all_valid": true,
      "median_seconds": 30.0,
      "p95_seconds": 47.4
    }
  },
  "holdout": {
    "targets": {
      "status_correct_min": 0.8,
      "literal_fact_coverage_min": 0.8,
      "permission_violations_max": 0,
      "hidden_source_leaks_max": 0,
      "citation_identity_valid": "必须全部通过",
      "tolerated_drop_vs_development": 0.09,
      "reasoning": "开发集 89.1%，允许最多 9 个百分点的回落到 80%：留出题从未被看过，且开发集参与过选型。低于 80% 视为过拟合开发集，必须记录而不是调参。"
    },
    "measured": {
      "questions": 80,
      "status_correct": 0.9,
      "literal_fact_coverage": 0.8194,
      "conflict_recall": null,
      "conflict_precision": null,
      "permission_violations": 0,
      "hidden_source_leaks": 0,
      "citation_identity_all_valid": true,
      "median_seconds": 24.6,
      "p95_seconds": 42.7
    },
    "run_once": true
  },
  "public": {
    "retrieval": {
      "bm25": {
        "recall_at_5": 0.8974,
        "recall_at_10": 0.9744,
        "mrr": 0.8447,
        "ndcg_at_10": 0.8751
      },
      "dense": {
        "recall_at_5": 0.8974,
        "recall_at_10": 0.9487,
        "mrr": 0.845,
        "ndcg_at_10": 0.8696
      },
      "hybrid": {
        "recall_at_5": 0.9744,
        "recall_at_10": 0.9744,
        "mrr": 0.847,
        "ndcg_at_10": 0.8796
      }
    },
    "end_to_end": {
      "bm25": {
        "questions": 39,
        "status_correct": 0.8462,
        "literal_fact_coverage": 0.0079,
        "conflict_recall": null,
        "conflict_precision": null,
        "permission_violations": 0,
        "hidden_source_leaks": 0,
        "citation_identity_all_valid": true,
        "median_seconds": 21.4,
        "p95_seconds": 40.9
      },
      "dense": {
        "questions": 39,
        "status_correct": 0.7436,
        "literal_fact_coverage": 0.0,
        "conflict_recall": null,
        "conflict_precision": null,
        "permission_violations": 0,
        "hidden_source_leaks": 0,
        "citation_identity_all_valid": true,
        "median_seconds": 18.1,
        "p95_seconds": 37.9
      },
      "hybrid": {
        "questions": 39,
        "status_correct": 0.8462,
        "literal_fact_coverage": 0.0079,
        "conflict_recall": null,
        "conflict_precision": null,
        "permission_violations": 0,
        "hidden_source_leaks": 0,
        "citation_identity_all_valid": true,
        "median_seconds": 19.1,
        "p95_seconds": 41.1
      }
    },
    "literal_matching": "not applicable: upstream gold is prose"
  },
  "rewrite": {
    "A": {
      "mode": "off",
      "follow_up_recall_at_5": 0.65,
      "topic_shift_contaminated": 0.0
    },
    "B": {
      "mode": "rule",
      "follow_up_recall_at_5": 0.975,
      "topic_shift_contaminated": 0.0
    },
    "C": {
      "mode": "llm",
      "follow_up_recall_at_5": 0.65,
      "topic_shift_contaminated": 0.0
    },
    "D": {
      "mode": "llm_history",
      "follow_up_recall_at_5": 0.9,
      "topic_shift_contaminated": 0.025
    }
  },
  "agent": {
    "status": "Phase B MVP 可运行，B2 开发回归通过",
    "tools": [
      "search_documents",
      "retrieve_evidence",
      "open_document",
      "get_document_version",
      "compare_versions",
      "verify_chunk_access",
      "search_memory"
    ],
    "workflow": {
      "tool_steps": 2,
      "wall_seconds": 23.1,
      "answer": "生产数据库 RPO 为 15 分钟",
      "citations": 1
    },
    "dynamic": {
      "tool_steps": 2,
      "wall_seconds": 111.3,
      "actions": [
        "search_documents",
        "retrieve_evidence",
        "final"
      ],
      "answer": "SEV1：10 分钟内通知负责人，每 30 分钟同步客户",
      "citations": 1
    },
    "benchmark": {
      "tasks": 30,
      "rag_accuracy": 1.0,
      "workflow_accuracy": 1.0,
      "rag_median_seconds": 22.0,
      "workflow_median_seconds": 21.6,
      "forbidden_leaks": 0,
      "dynamic_legacy": {
        "tasks": 5,
        "accuracy": 0.8,
        "median_seconds": 157.4,
        "mean_steps": 2.4,
        "forbidden_leaks": 0
      },
      "decision": "当前不训练、不做 RL：30 题开发回归已通过，动态模式保留为本地实验。"
    },
    "not_completed": [
      "长期记忆尚未证明能提高企业事实正确率；偏好遵从需要单独预注册评测",
      "动态 Agent 的完整 30 题配对未运行；访客后端固定使用低成本路径"
    ],
    "memory_ab": {
      "tasks": 30,
      "memory_off_accuracy": 1.0,
      "memory_on_accuracy": 0.9667,
      "accuracy_delta": -0.0333,
      "mcnemar_exact_p": 1.0,
      "memory_off_median_seconds": 15.9,
      "memory_on_median_seconds": 21.9,
      "mean_prompt_token_increase": 428.4,
      "forbidden_leaks": 0,
      "targeted_fix": "A26 外租户预检后开关均通过"
    },
    "failure_corpus": {
      "unique_failures": 17,
      "unresolved": 0,
      "decision": "do_not_train"
    },
    "hard": {
      "benchmark": "agent-hard-v2.1",
      "subset": "shared_comparable_subset",
      "rows": [
        {
          "name": "单次 RAG · Hard 共享 21 题",
          "correct": "12/21",
          "median_seconds": 33.4,
          "mean_steps": 1,
          "leaks": 0
        },
        {
          "name": "固定工作流 · Hard 共享 21 题",
          "correct": "19/21",
          "median_seconds": 43.2,
          "mean_steps": 2.33,
          "leaks": 0
        },
        {
          "name": "动态 Agent · Hard 共享 21 题",
          "correct": "14/21",
          "median_seconds": 93.9,
          "mean_steps": 3.48,
          "leaks": 0
        }
      ],
      "note": "Hard 30 是参与调试的开发 benchmark，跨策略只比较同分母的 21 题共享子集；不是泛化成绩。"
    },
    "external": {
      "benchmark": "multihop-rag-external-v1, multihop-rag-external-r2",
      "rows": [
        {
          "name": "单次 RAG · MultiHop-RAG 外部 第 1 批（150 题）",
          "correct": "57/150",
          "median_seconds": 29.5,
          "mean_steps": null,
          "leaks": null,
          "gold_document_recall": 0.518,
          "scoring_rule": "v1"
        },
        {
          "name": "固定工作流 · MultiHop-RAG 外部 第 1 批（150 题）",
          "correct": "62/150",
          "median_seconds": 42.9,
          "mean_steps": null,
          "leaks": null,
          "gold_document_recall": 0.578,
          "scoring_rule": "v1"
        },
        {
          "name": "单次 RAG · MultiHop-RAG 外部 第 2 批（150 题）",
          "correct": "101/150",
          "median_seconds": 53.5,
          "mean_steps": null,
          "leaks": null,
          "gold_document_recall": 0.652,
          "scoring_rule": "v2"
        },
        {
          "name": "固定工作流 · MultiHop-RAG 外部 第 2 批（150 题）",
          "correct": "101/150",
          "median_seconds": 54.0,
          "mean_steps": null,
          "leaks": null,
          "gold_document_recall": 0.667,
          "scoring_rule": "v2"
        }
      ],
      "note": "外部 MultiHop-RAG 固定子集，每批只运行一次，未用于调 prompt、规则或阈值。第 1 批按旧口径（要求答案里出现判断词）评分，第 2 批按显式结论字段评分，因此两批的答案列不可直接相减；Yes/No 题的判断正确率低于「一律答 yes」的平凡基线，结论字段解决的是可读性，不是判断力。"
    }
  },
  "not_measured": [
    "语义正确性与忠实度：评分器已校准但置信区间重叠，仅以影子模式记录，未用于判断",
    "金标与校准标签的独立人工复核：均记为 0",
    "并发与峰值负载：单进程单机，未做压测",
    "真实企业语料：全部资料为本项目自建虚构内容"
  ],
  "tests": {
    "collected": 138,
    "source": "pytest --collect-only"
  }
}