{
  "scope": "fixed enumerated batch; no stochastic rollout, KL, MoE or benchmark",
  "learning_rate": 0.5,
  "epsilon": 0.2,
  "responses": [
    [
      0,
      0
    ],
    [
      0,
      1
    ],
    [
      1,
      0
    ],
    [
      1,
      1
    ]
  ],
  "rewards": [
    0.0,
    0.5,
    0.5,
    1.0
  ],
  "advantages": [
    -1.414213522373096,
    0.0,
    0.0,
    1.414213522373096
  ],
  "history": [
    {
      "step": 0,
      "objective": 0.0,
      "exact_expected_reward": 0.5,
      "ratios": [
        1.0,
        1.0,
        1.0,
        1.0
      ],
      "flat_fraction": 0.0
    },
    {
      "step": 1,
      "objective": 0.06233774308912643,
      "exact_expected_reward": 0.544079442108941,
      "ratios": [
        0.911841115782118,
        0.996106425605948,
        0.996106425605948,
        1.088158884217882
      ],
      "flat_fraction": 0.0
    },
    {
      "step": 2,
      "objective": 0.12324304561161159,
      "exact_expected_reward": 0.5871459957509144,
      "ratios": [
        0.825708008498171,
        0.9846940142492623,
        0.9846940142492623,
        1.1742919915018288
      ],
      "flat_fraction": 0.0
    },
    {
      "step": 3,
      "objective": 0.14142135223730956,
      "exact_expected_reward": 0.6279859322979715,
      "ratios": [
        0.7440281354040571,
        0.9666842320712986,
        0.9666842320712986,
        1.255971864595943
      ],
      "flat_fraction": 0.5
    },
    {
      "step": 4,
      "objective": 0.14142135223730956,
      "exact_expected_reward": 0.6279859322979715,
      "ratios": [
        0.7440281354040571,
        0.9666842320712986,
        0.9666842320712986,
        1.255971864595943
      ],
      "flat_fraction": 0.5
    },
    {
      "step": 5,
      "objective": 0.14142135223730956,
      "exact_expected_reward": 0.6279859322979715,
      "ratios": [
        0.7440281354040571,
        0.9666842320712986,
        0.9666842320712986,
        1.255971864595943
      ],
      "flat_fraction": 0.5
    }
  ]
}