diff --git a/CHANGELOG.md b/CHANGELOG.md
index ef83812d..e28afca5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -180,6 +180,10 @@ All notable changes to SkillOpt are documented here. This project adheres to
Code, Codex and Pi transcript timestamps as instants rather than strings.
East of UTC, sessions that ended in the hours after a harvest were never
harvested (thanks @1AbdullahButt).
+- Treat a SearchQA gold answer given as a single string as one answer instead
+ of a sequence of characters, in scoring and in the gold answers recorded on
+ rollout rows. The correct answer was scored as wrong and a one-letter answer
+ as correct (thanks @Shizoqua).
### Security
- Prevent managed-identity credentials from being sent to non-Azure or
diff --git a/skillopt/envs/searchqa/evaluator.py b/skillopt/envs/searchqa/evaluator.py
index 3c29c89a..666e0eff 100644
--- a/skillopt/envs/searchqa/evaluator.py
+++ b/skillopt/envs/searchqa/evaluator.py
@@ -93,11 +93,16 @@ def sub_em(prediction: str, gold_answers: list[str]) -> float:
return 0.0
-def evaluate(prediction_text: str, gold_answers: list[str]) -> dict:
+def evaluate(prediction_text: str, gold_answers: list[str] | str) -> dict:
"""Evaluate a single QA prediction against gold answers.
+ A single string gold answer is treated as a one-item list, so it is not
+ iterated character by character.
+
Returns dict with: em, f1, sub_em, predicted_answer, gold_answers.
"""
+ if isinstance(gold_answers, str):
+ gold_answers = [gold_answers]
answer = extract_answer(prediction_text)
return {
"em": exact_match(answer, gold_answers),
diff --git a/skillopt/envs/searchqa/rollout.py b/skillopt/envs/searchqa/rollout.py
index 7f96393d..549ec017 100644
--- a/skillopt/envs/searchqa/rollout.py
+++ b/skillopt/envs/searchqa/rollout.py
@@ -27,6 +27,12 @@
_MAX_CONTEXT_CHARS = 6000
+def _gold_answers(item: dict) -> list:
+ """Return the item's gold answers as a list; a single string is one answer."""
+ answers = item.get("answers", [])
+ return [answers] if isinstance(answers, str) else answers
+
+
def _raise_on_systemic_failure(results: list[dict]) -> None:
"""Abort when all rollout rows failed before any agent response."""
if not results or not all(row.get("agent_ok") is False for row in results):
@@ -184,7 +190,7 @@ def process_one(
pred_dir = confined_task_output_dir(out_root, item_id)
question = item["question"]
context = item.get("context", "")
- gold_answers = item.get("answers", [])
+ gold_answers = _gold_answers(item)
result = {
"id": item_id,
@@ -415,7 +421,7 @@ def _timeout_result(item: dict) -> dict:
"fail_reason": f"task-timeout-{task_timeout}s",
"agent_ok": False,
"n_turns": 0,
- "gold_answer": item.get("answers", []),
+ "gold_answer": _gold_answers(item),
"phase": "timeout",
}
diff --git a/tests/test_searchqa_evaluator.py b/tests/test_searchqa_evaluator.py
index 75fa9eea..8eea9a27 100644
--- a/tests/test_searchqa_evaluator.py
+++ b/tests/test_searchqa_evaluator.py
@@ -26,3 +26,17 @@ def test_sub_em_empty_prediction_matches_empty_gold():
result = evaluate("", [""])
assert result["sub_em"] == result["em"] == result["f1"] == 1.0
+
+
+@pytest.mark.parametrize("response", ["P", "s"])
+def test_single_string_gold_is_not_split_into_characters(response):
+ result = evaluate(response, "Paris")
+
+ assert result["em"] == result["f1"] == 0.0
+
+
+def test_single_string_gold_matches_the_full_answer():
+ result = evaluate("Paris", "Paris")
+
+ assert result["em"] == result["f1"] == result["sub_em"] == 1.0
+ assert result["gold_answers"] == ["Paris"]
diff --git a/tests/test_searchqa_rollout_gold_answers.py b/tests/test_searchqa_rollout_gold_answers.py
new file mode 100644
index 00000000..78ec5f27
--- /dev/null
+++ b/tests/test_searchqa_rollout_gold_answers.py
@@ -0,0 +1,41 @@
+"""SearchQA rollout rows record a single string gold answer as a one-item list."""
+
+from __future__ import annotations
+
+from skillopt.envs.searchqa import rollout
+
+ITEM = {
+ "id": "q1",
+ "question": "What is the capital of France?",
+ "context": "Paris is the capital.",
+ "answers": "Paris",
+}
+
+
+def test_process_one_records_string_gold_answer_as_a_list(tmp_path, monkeypatch):
+ monkeypatch.setattr(rollout, "is_target_exec_backend", lambda: False)
+ monkeypatch.setattr(rollout, "chat_target", lambda **kwargs: ("Paris", {}))
+
+ row = rollout.process_one(dict(ITEM), str(tmp_path), "")
+
+ assert row["gold_answers"] == ["Paris"]
+ assert row["hard"] == 1
+
+
+def test_error_row_records_string_gold_answer_as_a_list(tmp_path, monkeypatch):
+ good = {"id": "ok", "agent_ok": True, "hard": 1, "soft": 1.0}
+
+ def fake_process_one(item, *args, **kwargs):
+ if item["id"] == "q1":
+ raise ValueError("boom")
+ return dict(good)
+
+ monkeypatch.setattr(rollout, "process_one", fake_process_one)
+
+ rows = rollout.run_batch(
+ [dict(ITEM), {"id": "ok", "question": "q", "answers": ["x"]}], str(tmp_path), "", workers=1
+ )
+
+ (error_row,) = [row for row in rows if row["id"] == "q1"]
+ assert error_row["phase"] == "error"
+ assert error_row["gold_answer"] == ["Paris"]