{
  "description": "Fixed DPO label flips; clean reward table and update budget unchanged; not held-out evaluation",
  "runs": [
    {
      "settings": {
        "algorithm": "dpo",
        "seed": 7,
        "steps": 120,
        "dtype": "torch.float64",
        "device": "cpu",
        "torch": "2.8.0+cu128",
        "learning_rate": 0.08,
        "update_epochs": 1,
        "clip_epsilon": 0.2,
        "target_old_policy_kl": 0.05,
        "kl_beta": 0,
        "dpo_beta": 0.5,
        "group_size": null,
        "online_batch_prompts": 0,
        "offline_preference_pairs": 12,
        "reward_table": [
          [
            1.0,
            0.2,
            -0.5
          ],
          [
            -0.5,
            1.0,
            0.2
          ],
          [
            0.2,
            -0.5,
            1.0
          ],
          [
            1.0,
            -0.5,
            0.2
          ]
        ],
        "reference": "fixed uniform",
        "scope": "one-step tabular contextual bandit; no language model or held-out generalization",
        "preference_flips": 0,
        "flipped_pair_indices": [],
        "clean_preference_pairs": [
          [
            0,
            0,
            1
          ],
          [
            0,
            0,
            2
          ],
          [
            0,
            1,
            2
          ],
          [
            1,
            1,
            0
          ],
          [
            1,
            1,
            2
          ],
          [
            1,
            2,
            0
          ],
          [
            2,
            0,
            1
          ],
          [
            2,
            2,
            0
          ],
          [
            2,
            2,
            1
          ],
          [
            3,
            0,
            1
          ],
          [
            3,
            0,
            2
          ],
          [
            3,
            2,
            1
          ]
        ],
        "training_preference_pairs": [
          [
            0,
            0,
            1
          ],
          [
            0,
            0,
            2
          ],
          [
            0,
            1,
            2
          ],
          [
            1,
            1,
            0
          ],
          [
            1,
            1,
            2
          ],
          [
            1,
            2,
            0
          ],
          [
            2,
            0,
            1
          ],
          [
            2,
            2,
            0
          ],
          [
            2,
            2,
            1
          ],
          [
            3,
            0,
            1
          ],
          [
            3,
            0,
            2
          ],
          [
            3,
            2,
            1
          ]
        ],
        "preference_columns": [
          "context",
          "chosen",
          "rejected"
        ]
      },
      "initial": {
        "algorithm": "dpo",
        "step": 0,
        "optimizer_steps": 0,
        "sampled_actions": 0,
        "pair_presentations": 0,
        "expected_reward": 0.23333333333333334,
        "best_action_probability": 0.3333333333333333,
        "entropy": 1.0986122886681096,
        "kl_reference": 0.0,
        "loss": 0.0,
        "value_loss": 0.0,
        "clip_fraction": 0.0,
        "old_policy_kl": 0.0,
        "zero_group_fraction": 0.0,
        "dpo_training_loss": 0.6931471805599453,
        "dpo_clean_loss": 0.6931471805599453
      },
      "final": {
        "algorithm": "dpo",
        "step": 120,
        "optimizer_steps": 120,
        "sampled_actions": 0,
        "pair_presentations": 1440,
        "expected_reward": 0.9950378892642915,
        "best_action_probability": 0.993830465052959,
        "entropy": 0.0377723143394775,
        "kl_reference": 1.0608399743286323,
        "loss": 0.052985320336784214,
        "value_loss": 0.0,
        "clip_fraction": 0.0,
        "old_policy_kl": 0.0,
        "zero_group_fraction": 0.0,
        "dpo_training_loss": 0.05246112303660775,
        "dpo_clean_loss": 0.05246112303660775
      }
    },
    {
      "settings": {
        "algorithm": "dpo",
        "seed": 7,
        "steps": 120,
        "dtype": "torch.float64",
        "device": "cpu",
        "torch": "2.8.0+cu128",
        "learning_rate": 0.08,
        "update_epochs": 1,
        "clip_epsilon": 0.2,
        "target_old_policy_kl": 0.05,
        "kl_beta": 0,
        "dpo_beta": 0.5,
        "group_size": null,
        "online_batch_prompts": 0,
        "offline_preference_pairs": 12,
        "reward_table": [
          [
            1.0,
            0.2,
            -0.5
          ],
          [
            -0.5,
            1.0,
            0.2
          ],
          [
            0.2,
            -0.5,
            1.0
          ],
          [
            1.0,
            -0.5,
            0.2
          ]
        ],
        "reference": "fixed uniform",
        "scope": "one-step tabular contextual bandit; no language model or held-out generalization",
        "preference_flips": 3,
        "flipped_pair_indices": [
          0,
          3,
          4
        ],
        "clean_preference_pairs": [
          [
            0,
            0,
            1
          ],
          [
            0,
            0,
            2
          ],
          [
            0,
            1,
            2
          ],
          [
            1,
            1,
            0
          ],
          [
            1,
            1,
            2
          ],
          [
            1,
            2,
            0
          ],
          [
            2,
            0,
            1
          ],
          [
            2,
            2,
            0
          ],
          [
            2,
            2,
            1
          ],
          [
            3,
            0,
            1
          ],
          [
            3,
            0,
            2
          ],
          [
            3,
            2,
            1
          ]
        ],
        "training_preference_pairs": [
          [
            0,
            1,
            0
          ],
          [
            0,
            0,
            2
          ],
          [
            0,
            1,
            2
          ],
          [
            1,
            0,
            1
          ],
          [
            1,
            2,
            1
          ],
          [
            1,
            2,
            0
          ],
          [
            2,
            0,
            1
          ],
          [
            2,
            2,
            0
          ],
          [
            2,
            2,
            1
          ],
          [
            3,
            0,
            1
          ],
          [
            3,
            0,
            2
          ],
          [
            3,
            2,
            1
          ]
        ],
        "preference_columns": [
          "context",
          "chosen",
          "rejected"
        ]
      },
      "initial": {
        "algorithm": "dpo",
        "step": 0,
        "optimizer_steps": 0,
        "sampled_actions": 0,
        "pair_presentations": 0,
        "expected_reward": 0.23333333333333334,
        "best_action_probability": 0.3333333333333333,
        "entropy": 1.0986122886681096,
        "kl_reference": 0.0,
        "loss": 0.0,
        "value_loss": 0.0,
        "clip_fraction": 0.0,
        "old_policy_kl": 0.0,
        "zero_group_fraction": 0.0,
        "dpo_training_loss": 0.6931471805599453,
        "dpo_clean_loss": 0.6931471805599453
      },
      "final": {
        "algorithm": "dpo",
        "step": 120,
        "optimizer_steps": 120,
        "sampled_actions": 0,
        "pair_presentations": 1440,
        "expected_reward": 0.5976731767116585,
        "best_action_probability": 0.49845755332160635,
        "entropy": 0.03777167370627035,
        "kl_reference": 1.0608406149618395,
        "loss": 0.052985309883391014,
        "value_loss": 0.0,
        "clip_fraction": 0.0,
        "old_policy_kl": 0.0,
        "zero_group_fraction": 0.0,
        "dpo_training_loss": 0.05246111267175962,
        "dpo_clean_loss": 0.9004739382463667
      }
    },
    {
      "settings": {
        "algorithm": "dpo",
        "seed": 7,
        "steps": 120,
        "dtype": "torch.float64",
        "device": "cpu",
        "torch": "2.8.0+cu128",
        "learning_rate": 0.08,
        "update_epochs": 1,
        "clip_epsilon": 0.2,
        "target_old_policy_kl": 0.05,
        "kl_beta": 0,
        "dpo_beta": 0.5,
        "group_size": null,
        "online_batch_prompts": 0,
        "offline_preference_pairs": 12,
        "reward_table": [
          [
            1.0,
            0.2,
            -0.5
          ],
          [
            -0.5,
            1.0,
            0.2
          ],
          [
            0.2,
            -0.5,
            1.0
          ],
          [
            1.0,
            -0.5,
            0.2
          ]
        ],
        "reference": "fixed uniform",
        "scope": "one-step tabular contextual bandit; no language model or held-out generalization",
        "preference_flips": 6,
        "flipped_pair_indices": [
          0,
          3,
          4,
          5,
          7,
          9
        ],
        "clean_preference_pairs": [
          [
            0,
            0,
            1
          ],
          [
            0,
            0,
            2
          ],
          [
            0,
            1,
            2
          ],
          [
            1,
            1,
            0
          ],
          [
            1,
            1,
            2
          ],
          [
            1,
            2,
            0
          ],
          [
            2,
            0,
            1
          ],
          [
            2,
            2,
            0
          ],
          [
            2,
            2,
            1
          ],
          [
            3,
            0,
            1
          ],
          [
            3,
            0,
            2
          ],
          [
            3,
            2,
            1
          ]
        ],
        "training_preference_pairs": [
          [
            0,
            1,
            0
          ],
          [
            0,
            0,
            2
          ],
          [
            0,
            1,
            2
          ],
          [
            1,
            0,
            1
          ],
          [
            1,
            2,
            1
          ],
          [
            1,
            0,
            2
          ],
          [
            2,
            0,
            1
          ],
          [
            2,
            0,
            2
          ],
          [
            2,
            2,
            1
          ],
          [
            3,
            1,
            0
          ],
          [
            3,
            0,
            2
          ],
          [
            3,
            2,
            1
          ]
        ],
        "preference_columns": [
          "context",
          "chosen",
          "rejected"
        ]
      },
      "initial": {
        "algorithm": "dpo",
        "step": 0,
        "optimizer_steps": 0,
        "sampled_actions": 0,
        "pair_presentations": 0,
        "expected_reward": 0.23333333333333334,
        "best_action_probability": 0.3333333333333333,
        "entropy": 1.0986122886681096,
        "kl_reference": 0.0,
        "loss": 0.0,
        "value_loss": 0.0,
        "clip_fraction": 0.0,
        "old_policy_kl": 0.0,
        "zero_group_fraction": 0.0,
        "dpo_training_loss": 0.6931471805599453,
        "dpo_clean_loss": 0.6931471805599453
      },
      "final": {
        "algorithm": "dpo",
        "step": 120,
        "optimizer_steps": 120,
        "sampled_actions": 0,
        "pair_presentations": 1440,
        "expected_reward": 0.03686005209997253,
        "best_action_probability": 0.08640864268433518,
        "entropy": 0.30298262823823907,
        "kl_reference": 0.7956296604298707,
        "loss": 0.2130257906192711,
        "value_loss": 0.0,
        "clip_fraction": 0.0,
        "old_policy_kl": 0.0,
        "zero_group_fraction": 0.0,
        "dpo_training_loss": 0.21263264259986622,
        "dpo_clean_loss": 1.4846530560349709
      }
    },
    {
      "settings": {
        "algorithm": "dpo",
        "seed": 7,
        "steps": 120,
        "dtype": "torch.float64",
        "device": "cpu",
        "torch": "2.8.0+cu128",
        "learning_rate": 0.08,
        "update_epochs": 1,
        "clip_epsilon": 0.2,
        "target_old_policy_kl": 0.05,
        "kl_beta": 0,
        "dpo_beta": 0.5,
        "group_size": null,
        "online_batch_prompts": 0,
        "offline_preference_pairs": 12,
        "reward_table": [
          [
            1.0,
            0.2,
            -0.5
          ],
          [
            -0.5,
            1.0,
            0.2
          ],
          [
            0.2,
            -0.5,
            1.0
          ],
          [
            1.0,
            -0.5,
            0.2
          ]
        ],
        "reference": "fixed uniform",
        "scope": "one-step tabular contextual bandit; no language model or held-out generalization",
        "preference_flips": 12,
        "flipped_pair_indices": [
          0,
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11
        ],
        "clean_preference_pairs": [
          [
            0,
            0,
            1
          ],
          [
            0,
            0,
            2
          ],
          [
            0,
            1,
            2
          ],
          [
            1,
            1,
            0
          ],
          [
            1,
            1,
            2
          ],
          [
            1,
            2,
            0
          ],
          [
            2,
            0,
            1
          ],
          [
            2,
            2,
            0
          ],
          [
            2,
            2,
            1
          ],
          [
            3,
            0,
            1
          ],
          [
            3,
            0,
            2
          ],
          [
            3,
            2,
            1
          ]
        ],
        "training_preference_pairs": [
          [
            0,
            1,
            0
          ],
          [
            0,
            2,
            0
          ],
          [
            0,
            2,
            1
          ],
          [
            1,
            0,
            1
          ],
          [
            1,
            2,
            1
          ],
          [
            1,
            0,
            2
          ],
          [
            2,
            1,
            0
          ],
          [
            2,
            0,
            2
          ],
          [
            2,
            1,
            2
          ],
          [
            3,
            1,
            0
          ],
          [
            3,
            2,
            0
          ],
          [
            3,
            1,
            2
          ]
        ],
        "preference_columns": [
          "context",
          "chosen",
          "rejected"
        ]
      },
      "initial": {
        "algorithm": "dpo",
        "step": 0,
        "optimizer_steps": 0,
        "sampled_actions": 0,
        "pair_presentations": 0,
        "expected_reward": 0.23333333333333334,
        "best_action_probability": 0.3333333333333333,
        "entropy": 1.0986122886681096,
        "kl_reference": 0.0,
        "loss": 0.0,
        "value_loss": 0.0,
        "clip_fraction": 0.0,
        "old_policy_kl": 0.0,
        "zero_group_fraction": 0.0,
        "dpo_training_loss": 0.6931471805599453,
        "dpo_clean_loss": 0.6931471805599453
      },
      "final": {
        "algorithm": "dpo",
        "step": 120,
        "optimizer_steps": 120,
        "sampled_actions": 0,
        "pair_presentations": 1440,
        "expected_reward": -0.4956510595049848,
        "best_action_probability": 3.783254010814886e-05,
        "entropy": 0.0377723143394775,
        "kl_reference": 1.0608399743286323,
        "loss": 0.052985320336784214,
        "value_loss": 0.0,
        "clip_fraction": 0.0,
        "old_policy_kl": 0.0,
        "zero_group_fraction": 0.0,
        "dpo_training_loss": 0.05246112303660775,
        "dpo_clean_loss": 3.4445119000589064
      }
    }
  ]
}