{
  "scope": "fixed two-answer bigram gradient fixture; no online sampling",
  "device": "cpu",
  "dtype": "torch.float64",
  "torch": "2.8.0+cu128",
  "vocabulary": {
    "BOS": 0,
    "prompt": 1,
    "good": 2,
    "bad": 3,
    "EOS_and_PAD": 4
  },
  "input_ids": [
    [
      0,
      1,
      2,
      4,
      4
    ],
    [
      0,
      1,
      3,
      3,
      4
    ]
  ],
  "response_mask": [
    [
      false,
      false,
      true,
      true,
      false
    ],
    [
      false,
      false,
      true,
      true,
      true
    ]
  ],
  "dpo_beta": 0.5,
  "grpo_kl_beta": 0.03,
  "clip_epsilon": 0.2,
  "learning_rate": 0.1,
  "reduction": "GRPO: mean tokens per response, then mean responses",
  "dpo": {
    "initial_loss": 0.6931471805599453,
    "final_loss": 0.668435395925845,
    "gradient_norm": 0.5,
    "relative_margin_before": 0.0,
    "relative_margin_after": 0.1000994964803632
  },
  "grpo": {
    "initial_loss": 5.551115123125783e-17,
    "final_loss": -0.01728481776972801,
    "gradient_norm": 0.41499664496635813,
    "relative_margin_before": 0.0,
    "relative_margin_after": 0.08168300685127106
  },
  "alignment": {
    "lengths": [
      2,
      3
    ],
    "sequence_logps": [
      -3.2188758248682006,
      -4.828313737302301
    ],
    "mean_logps": [
      -1.6094379124341003,
      -1.6094379124341003
    ],
    "token_logps": [
      [
        0.0,
        -1.6094379124341003,
        -1.6094379124341003,
        0.0
      ],
      [
        0.0,
        -1.6094379124341003,
        -1.6094379124341003,
        -1.6094379124341003
      ]
    ],
    "advantages": [
      0.9999999800000003,
      -0.9999999800000003
    ],
    "valid": [
      [
        false,
        true,
        true,
        false
      ],
      [
        false,
        true,
        true,
        true
      ]
    ]
  },
  "initial_global_token_loss": 0.19999999600000007
}