让研究 Agent 检索并给出证据

本次运行结果

A fixed local corpus; no external search API or model download.

Verified source: 22a043be7e04532629b6f89f8ba3fecc3accfce744b1ee4fba1212b5813e2a02
def run_episode(
    policy: LinearSoftmaxPolicy,
    task: Task,
    rng: random.Random,
    temperature: float = 1.0,
    greedy: bool = False,
) -> tuple[float, dict[str, object]]:
    q_features = [query_features(task.question, query) for query in task.query_options]
    q_idx, q_probs = policy.choose(policy.query_weights, q_features, temperature, greedy)
    query = task.query_options[q_idx]

    results = search(query, rng)
    d_features = [doc_features(task.question, query, doc, rank) for rank, doc in enumerate(results)]
    d_idx, d_probs = policy.choose(policy.doc_weights, d_features, temperature, greedy)
    doc = results[d_idx]

    score = reward(task, doc.answer, doc.doc_id)
    trace = {
        "query_idx": q_idx,
        "query_probs": q_probs,
        "query_features": q_features,
        "doc_idx": d_idx,
        "doc_probs": d_probs,
        "doc_features": d_features,
        "query": query,
        "doc": doc,
        "answer": doc.answer,
    }
    return score, trace

{'train_questions': 7, 'heldout_questions': 3, 'corpus_documents': 12}
Matched evaluator ready: same seeds, temperature and action mode before/after.

[
  {
    "seed": 3,
    "before": {
      "greedy": {
        "reward": 1.0,
        "answer_exact": 1.0,
        "citation_support": 1.0
      },
      "sampled": {
        "reward": 0.2,
        "answer_exact": 0.2,
        "citation_support": 0.2
      }
    },
    "after": {
      "greedy": {
        "reward": 1.0,
        "answer_exact": 1.0,
        "citation_support": 1.0
      },
      "sampled": {
        "reward": 1.0,
        "answer_exact": 1.0,
        "citation_support": 1.0
      }
    }
  },
  {
    "seed": 7,
    "before": {
      "greedy": {
        "reward": 1.0,
        "answer_exact": 1.0,
        "citation_support": 1.0
      },
      "sampled": {
        "reward": 0.2,
        "answer_exact": 0.2,
        "citation_support": 0.2
      }
    },
    "after": {
      "greedy": {
        "reward": 1.0,
        "answer_exact": 1.0,
        "citation_support": 1.0
      },
      "sampled": {
        "reward": 1.0,
        "answer_exact": 1.0,
        "citation_support": 1.0
      }
    }
  },
  {
    "seed": 11,
    "before": {
      "greedy": {
        "reward": 1.0,
        "answer_exact": 1.0,
        "citation_support": 1.0
      },
      "sampled": {
        "reward": 0.2,
        "answer_exact": 0.2,
        "citation_support": 0.2
      }
    },
    "after": {
      "greedy": {
        "reward": 1.0,
        "answer_exact": 1.0,
        "citation_support": 1.0
      },
      "sampled": {
        "reward": 1.0,
        "answer_exact": 1.0,
        "citation_support": 1.0
      }
    }
  }
]

{
  "question": "Which agentic reasoning report uses a GRPO-style recipe to make smaller models competitive?",
  "query": "GRPO style recipe smaller model competitive agentic reasoning",
  "retrieved": [
    "rstar2-paper",
    "tongyi-paper",
    "web-shepherd-paper",
    "distractor-rag"
  ],
  "chosen_doc": "rstar2-paper",
  "answer": "rStar2-Agent",
  "reward": 1.0
}

Fill answer, then enable RUN_EXERCISE.

{
  "runs": [
    {
      "seed": 3,
      "before": {
        "greedy": {
          "reward": 1.0,
          "answer_exact": 1.0,
          "citation_support": 1.0
        },
        "sampled": {
          "reward": 0.2,
          "answer_exact": 0.2,
          "citation_support": 0.2
        }
      },
      "after": {
        "greedy": {
          "reward": 1.0,
          "answer_exact": 1.0,
          "citation_support": 1.0
        },
        "sampled": {
          "reward": 1.0,
          "answer_exact": 1.0,
          "citation_support": 1.0
        }
      }
    },
    {
      "seed": 7,
      "before": {
        "greedy": {
          "reward": 1.0,
          "answer_exact": 1.0,
          "citation_support": 1.0
        },
        "sampled": {
          "reward": 0.2,
          "answer_exact": 0.2,
          "citation_support": 0.2
        }
      },
      "after": {
        "greedy": {
          "reward": 1.0,
          "answer_exact": 1.0,
          "citation_support": 1.0
        },
        "sampled": {
          "reward": 1.0,
          "answer_exact": 1.0,
          "citation_support": 1.0
        }
      }
    },
    {
      "seed": 11,
      "before": {
        "greedy": {
          "reward": 1.0,
          "answer_exact": 1.0,
          "citation_support": 1.0
        },
        "sampled": {
          "reward": 0.2,
          "answer_exact": 0.2,
          "citation_support": 0.2
        }
      },
      "after": {
        "greedy": {
          "reward": 1.0,
          "answer_exact": 1.0,
          "citation_support": 1.0
        },
        "sampled": {
          "reward": 1.0,
          "answer_exact": 1.0,
          "citation_support": 1.0
        }
      }
    }
  ],
  "fixed_heldout_trace": {
    "question": "Which agentic reasoning report uses a GRPO-style recipe to make smaller models competitive?",
    "query": "GRPO style recipe smaller model competitive agentic reasoning",
    "retrieved": [
      "rstar2-paper",
      "tongyi-paper",
      "web-shepherd-paper",
      "distractor-rag"
    ],
    "chosen_doc": "rstar2-paper",
    "answer": "rStar2-Agent",
    "reward": 1.0
  },
  "source_sha256": "22a043be7e04532629b6f89f8ba3fecc3accfce744b1ee4fba1212b5813e2a02",
  "train_questions": 7,
  "heldout_questions": 3,
  "epochs": 220,
  "algorithm": "Feature policy with REINFORCE-style baseline updates"
}