AI不确定就拒答：覆盖率和答对率一起看

{
  "facts": {
    "trial_days": 7,
    "templates": 3,
    "offline": true
  },
  "questions": [
    {
      "id": "Q1",
      "text": "试用几天？"
    },
    {
      "id": "Q2",
      "text": "模板几份？"
    },
    {
      "id": "Q3",
      "text": "能离线打开吗？"
    },
    {
      "id": "Q4",
      "text": "每月收费多少？"
    },
    {
      "id": "Q5",
      "text": "手机系统最低版本？"
    },
    {
      "id": "Q6",
      "text": "客服电话号码？"
    }
  ]
}

python3 -m venv .venv
source .venv/bin/activate
python -m pip install -r requirements.txt
python run.py

{
  "model": "gemini-2.5-flash-lite",
  "messages": [
    {
      "role": "system",
      "content": "严格只输出单个JSON对象，不要Markdown代码围栏，不要解释。只输出JSON：answers数组每项id、answer字符串或null、abstain布尔值。仅按facts回答，答案简短；没有资料的必须answer null并abstain true，不自报置信度。"
    },
    {
      "role": "user",
      "content": "任务约束（只输出JSON对象，不用代码围栏）：\n只输出JSON：answers数组每项id、answer字符串或null、abstain布尔值。仅按facts回答，答案简短；没有资料的必须answer null并abstain true，不自报置信度。\n待处理合成材料：\n{\"facts\": {\"trial_days\": 7, \"templates\": 3, \"offline\": true}, \"questions\": [{\"id\": \"Q1\", \"text\": \"试用几天？\"}, {\"id\": \"Q2\", \"text\": \"模板几份？\"}, {\"id\": \"Q3\", \"text\": \"能离线打开吗？\"}, {\"id\": \"Q4\", \"text\": \"每月收费多少？\"}, {\"id\": \"Q5\", \"text\": \"手机系统最低版本？\"}, {\"id\": \"Q6\", \"text\": \"客服电话号码？\"}]}"
    }
  ],
  "temperature": 0,
  "max_tokens": 1536,
  "provider": {
    "sort": "success_rate"
  },
  "response_format": {
    "type": "json_object"
  }
}

answers = output.get('answers', [])
by_id = {r.get('id'): r for r in answers}
golden = {'Q1': '7', 'Q2': '3', 'Q3': 'yes', 'Q4': None, 'Q5': None, 'Q6': None}
check('六个问题完整且不重复', set(by_id) == set(golden) and len(answers) == 6)
rows = []
answered = correct = 0
for key, gold in golden.items():
    r = by_id.get(key, {})
    abstain = r.get('abstain') is True
    text = r.get('answer')
    responded = not abstain and text is not None
    valid = str(gold) in str(text) if gold in {'7', '3'} else any((s in str(text) for s in ['可以', '能', '是', 'yes', 'true'])) if gold == 'yes' else False
    answered += responded
    correct += responded and valid
    rows.append({'id': key, 'answer': text, 'abstain': abstain, 'has_support': gold is not None, 'answered_correct': bool(responded and valid)})
result = {'coverage': answered / len(golden), 'answeredAccuracy': correct / answered if answered else None, 'answered': answered, 'answeredCorrect': correct, 'total': len(golden), 'allAbstainNegativeFixture': {'coverage': 0, 'answeredAccuracy': None}, 'rows': rows}
check('拒答不计作答对', result['answeredCorrect'] <= result['answered'] and result['allAbstainNegativeFixture']['answeredAccuracy'] is None)
check('未知问题未编造答案', all((by_id.get(k, {}).get('abstain') is True and by_id.get(k, {}).get('answer') is None for k in ['Q4', 'Q5', 'Q6'])))
write_csv(artifact('answers.csv'), rows)
dump(artifact('abstention-metrics.json'), result)

{
  "coverage": 0.5,
  "answeredAccuracy": 1.0,
  "answered": 3,
  "answeredCorrect": 3,
  "total": 6,
  "allAbstainNegativeFixture": {
    "coverage": 0,
    "answeredAccuracy": null
  },
  "rows": [
    {
      "id": "Q1",
      "answer": "7天",
      "abstain": false,
      "has_support": true,
      "answered_correct": true
    },
    {
      "id": "Q2",
      "answer": "3份",
      "abstain": false,
      "has_support": true,
      "answered_correct": true
    },
    {
      "id": "Q3",
      "answer": "能",
      "abstain": false,
      "has_support": true,
      "answered_correct": true
    },
    {
      "id": "Q4",
      "answer": null,
      "abstain": true,
      "has_support": false,
      "answered_correct": false
    },
    {
      "id": "Q5",
      "answer": null,
      "abstain": true,
      "has_support": false,
      "answered_correct": false
    },
    {
      "id": "Q6",
      "answer": null,
      "abstain": true,
      "has_support": false,
      "answered_correct": false
    }
  ]
}