From fcb943cfe00f9079653f3988ced9f61a66632af2 Mon Sep 17 00:00:00 2001 From: Dylan Hadfield-Menell Date: Mon, 20 Jul 2026 10:29:41 -0400 Subject: [PATCH 1/2] Allow WANDB_ENTITY to be overridden by the caller's environment Previously hardcoded to the original authors' W&B team, which clobbers any pre-set value and breaks logging for anyone running on their own W&B account. --- training/verl_training.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/training/verl_training.sh b/training/verl_training.sh index 0b513369..e944051d 100644 --- a/training/verl_training.sh +++ b/training/verl_training.sh @@ -5,7 +5,7 @@ export PYTHONBUFFERED=1 # export RAY_DEBUG=1 ulimit -c 0 -export WANDB_ENTITY="sample-efficient-rlvr" # team +export WANDB_ENTITY="${WANDB_ENTITY:-sample-efficient-rlvr}" # team (override by pre-setting WANDB_ENTITY) export EXPERIMENT=${1:-"experiment"} CONFIG_NAME=${2:-"ppo_trainer"} export TASK=${3:-"datasets/ttcs/lasgroup_verifiable-corpus_math-ai_math500_1000"} From 34c88598b474b85583b9cd6618f08b511a734a3e Mon Sep 17 00:00:00 2001 From: Dylan Hadfield-Menell Date: Mon, 20 Jul 2026 14:33:30 -0400 Subject: [PATCH 2/2] Remove reward-based gating from self-distillation sample inclusion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit self_distillation_mask previously excluded any sample that didn't have a successful within-group demonstration or usable feedback (gated by reward_tensor via success_reward_threshold). But the self-distillation loss never reads reward/advantages at all (see compute_self_distillation_loss's argument list) — reward's only role here was deciding which samples got distilled, conflating "the task's reward function liked this rollout" with "this sample should be distilled." Those are unrelated once reward is just a placeholder/proxy rather than the actual training objective (as it is for the current Engaging smoke test, and will be once GOOD's context replaces the solution/feedback reprompt entirely). Every sample now participates unconditionally. Verified on Engaging (2x H200, Qwen3-8B, tooluse dataset): 3/3 training steps complete, self_distillation/reprompt_sample_fraction now 1.0 at every step (previously 1.0/0.96875/0.96875, gated by success/feedback availability). Co-Authored-By: Claude Sonnet 5 --- verl/trainer/ppo/ray_trainer.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/verl/trainer/ppo/ray_trainer.py b/verl/trainer/ppo/ray_trainer.py index a7256fd5..95a97156 100644 --- a/verl/trainer/ppo/ray_trainer.py +++ b/verl/trainer/ppo/ray_trainer.py @@ -770,12 +770,15 @@ def _build_teacher_message(i: int) -> list[dict]: for i in range(batch_size) ] - # self_distillation_mask is True if sample has a solution OR feedback is used (i.e., will get a reprompted message) - self_distillation_mask = torch.tensor( - [solution_strs[i] is not None or feedback_used[i] for i in range(batch_size)], - dtype=torch.float32, - device=device - ) + # Every sample participates in self-distillation, regardless of whether the + # task's reward function marked it "successful". Reward is not a training + # signal in this loss (see compute_self_distillation_loss, which never takes + # advantages/reward as input) — it was previously only used to gate which + # samples got a demonstration-based reprompt, which conflates "did the + # reward function like this rollout" with "should this sample be distilled." + # Those are unrelated once the reward function is just a placeholder/proxy + # rather than the actual training objective. + self_distillation_mask = torch.ones(batch_size, dtype=torch.float32, device=device) uids = set(batch.non_tensor_batch["uid"]) num_with_feedback_available = sum(1 for f in feedback_list if f is not None)