{
  "verified_at": "2026-10-15",
  "measurement_date": "2026-10-13",
  "source_project": "tiny_llm",
  "trained_models_added": 6,
  "inference_variants_added": 1,
  "weights_published": false,
  "source_tokens_per_model": 3932160000,
  "optimizer_steps": 15000,
  "training_seed": 2026,
  "training_contracts": {
    "q-rrt-latent-moe": {
      "step": 15000,
      "source_tokens": 3932160000,
      "seed": 2026,
      "batch_size": 64,
      "grad_accum": 2,
      "seq_len": 2048,
      "lr": 0.001,
      "min_lr_frac": 0.1,
      "weight_decay": 0.1,
      "warmup_frac": 0.01,
      "betas": [
        0.9,
        0.95
      ],
      "grad_clip": 1.0,
      "dtype": "bf16"
    },
    "q-rrt-delta": {
      "step": 15000,
      "source_tokens": 3932160000,
      "seed": 2026,
      "batch_size": 64,
      "grad_accum": 2,
      "seq_len": 2048,
      "lr": 0.001,
      "min_lr_frac": 0.1,
      "weight_decay": 0.1,
      "warmup_frac": 0.01,
      "betas": [
        0.9,
        0.95
      ],
      "grad_clip": 1.0,
      "dtype": "bf16"
    },
    "q-rrt-delta-beta2": {
      "step": 15000,
      "source_tokens": 3932160000,
      "seed": 2026,
      "batch_size": 32,
      "grad_accum": 4,
      "seq_len": 2048,
      "lr": 0.001,
      "min_lr_frac": 0.1,
      "weight_decay": 0.1,
      "warmup_frac": 0.01,
      "betas": [
        0.9,
        0.95
      ],
      "grad_clip": 1.0,
      "dtype": "bf16"
    },
    "q-rrt-delta-canon": {
      "step": 15000,
      "source_tokens": 3932160000,
      "seed": 2026,
      "batch_size": 32,
      "grad_accum": 4,
      "seq_len": 2048,
      "lr": 0.001,
      "min_lr_frac": 0.1,
      "weight_decay": 0.1,
      "warmup_frac": 0.01,
      "betas": [
        0.9,
        0.95
      ],
      "grad_clip": 1.0,
      "dtype": "bf16"
    },
    "q-rrt-delta-coarse": {
      "step": 15000,
      "source_tokens": 3932160000,
      "seed": 2026,
      "batch_size": 32,
      "grad_accum": 4,
      "seq_len": 2048,
      "lr": 0.001,
      "min_lr_frac": 0.1,
      "weight_decay": 0.1,
      "warmup_frac": 0.01,
      "betas": [
        0.9,
        0.95
      ],
      "grad_clip": 1.0,
      "dtype": "bf16"
    },
    "q-rrt-delta-coarse-beta2": {
      "step": 15000,
      "source_tokens": 3932160000,
      "seed": 2026,
      "batch_size": 32,
      "grad_accum": 4,
      "seq_len": 2048,
      "lr": 0.001,
      "min_lr_frac": 0.1,
      "weight_decay": 0.1,
      "warmup_frac": 0.01,
      "betas": [
        0.9,
        0.95
      ],
      "grad_clip": 1.0,
      "dtype": "bf16"
    }
  },
  "sources": [
    {
      "path": "docs/all-trained-results-20261013.json",
      "sha256": "035b7dc4663b0d0e4fece7d0669a713844993524f78291f75365a7b94ff9cda1"
    },
    {
      "path": "docs/coarse-one-pass-results-20261013.json",
      "sha256": "660b46cd42ecba129d2724c2f2fa2090c6f4ce4dfb2d6b86322dfaa357cd4750"
    },
    {
      "path": "runs/eval/additional-release-20261004/plan.json",
      "sha256": "2b72d6b57a359364a478b38016526e5f173a5df9a732ff0f6054816a4d88ef16"
    },
    {
      "path": "docs/delta-evaluation-inventory-20261011.json",
      "sha256": "c04eae67287255982c78b2d1be82c63465e388a9bd434a96f7eb2769c4d7f0a2"
    },
    {
      "path": "docs/delta-evaluation-inventory-20261012.json",
      "sha256": "dd06802cf40983bd55b3be6cf07f212331b932fd2e594780402be8c1068eea6f"
    }
  ],
  "verified_artifact_files": 140,
  "controls": [
    {
      "id": "q-moe-98M-A51M-base",
      "repo": "d0rj/q-moe-98M-A51M-base",
      "revision": "61700a031a87850816e263b4006615e27c28967b",
      "label": "Q-MoE 98M / A51M",
      "group": "causal",
      "parameters": 98105088,
      "active_parameters": 50919168,
      "optimized_parameters": null,
      "mean_16_percent": 43.05414834507311,
      "tasks": [
        {
          "id": "hellaswag",
          "label": "HellaSwag",
          "metric": "acc_norm",
          "value": 0.3051185022903804,
          "stderr": 0.004595165551383606,
          "ci95": [
            0.2961885267487427,
            0.31419752044859445
          ],
          "samples": 10042,
          "dataset": "Rowan/hellaswag",
          "split": "validation"
        },
        {
          "id": "arc_easy",
          "label": "ARC-Easy",
          "metric": "acc_norm",
          "value": 0.4385521885521885,
          "stderr": 0.010182010275471127,
          "ci95": [
            0.41871506492003546,
            0.45858768610242223
          ],
          "samples": 2376,
          "dataset": "allenai/ai2_arc",
          "split": "test"
        },
        {
          "id": "arc_challenge",
          "label": "ARC-Challenge",
          "metric": "acc_norm",
          "value": 0.24658703071672355,
          "stderr": 0.012595726268789992,
          "ci95": [
            0.2227647274615329,
            0.2720651277506229
          ],
          "samples": 1172,
          "dataset": "allenai/ai2_arc",
          "split": "test"
        },
        {
          "id": "piqa",
          "label": "PIQA",
          "metric": "acc_norm",
          "value": 0.6104461371055495,
          "stderr": 0.011377657455130346,
          "ci95": [
            0.5879441215348398,
            0.6324874460442482
          ],
          "samples": 1838,
          "dataset": "baber/piqa",
          "split": "validation"
        },
        {
          "id": "winogrande",
          "label": "WinoGrande",
          "metric": "acc",
          "value": 0.5082872928176796,
          "stderr": 0.01405055532282417,
          "ci95": [
            0.48077615465376927,
            0.5357483298554845
          ],
          "samples": 1267,
          "dataset": "allenai/winogrande",
          "split": "validation"
        },
        {
          "id": "openbookqa",
          "label": "OpenBookQA",
          "metric": "acc_norm",
          "value": 0.31,
          "stderr": 0.020704041021724684,
          "ci95": [
            0.2710389112735686,
            0.3518583381018213
          ],
          "samples": 500,
          "dataset": "allenai/openbookqa",
          "split": "test"
        },
        {
          "id": "boolq",
          "label": "BoolQ",
          "metric": "acc",
          "value": 0.5685015290519878,
          "stderr": 0.00866259456902732,
          "ci95": [
            0.5514551457036189,
            0.5853871558051961
          ],
          "samples": 3270,
          "dataset": "aps/super_glue",
          "split": "validation"
        },
        {
          "id": "lambada_openai",
          "label": "LAMBADA OpenAI",
          "metric": "acc",
          "value": 0.22064816611682514,
          "stderr": 0.005777354473289337,
          "ci95": [
            0.20953625932872036,
            0.23217626509346886
          ],
          "samples": 5153,
          "dataset": "EleutherAI/lambada_openai",
          "split": "test"
        },
        {
          "id": "arithmark3",
          "label": "ArithMark-3",
          "metric": "acc_norm",
          "value": 0.369,
          "stderr": 0.015266698139154782,
          "ci95": [
            0.33964716115877136,
            0.399355449564444
          ],
          "samples": 1000,
          "dataset": "AxiomicLabs/Arithmark-3.0",
          "split": "train"
        },
        {
          "id": "balanced_copa",
          "label": "Balanced COPA",
          "metric": "acc",
          "value": 0.543,
          "stderr": 0.0157606915901365,
          "ci95": [
            0.5120192033150067,
            0.5736516954552356
          ],
          "samples": 1000,
          "dataset": "pkavumba/balanced-copa",
          "split": "train"
        },
        {
          "id": "blimp",
          "label": "BLiMP",
          "metric": "acc",
          "value": 0.7723582089552239,
          "stderr": 0.0013958302883342111,
          "ci95": null,
          "samples": 67000,
          "dataset": "nyu-mll/blimp",
          "split": "train"
        },
        {
          "id": "commonsense_qa",
          "label": "CommonsenseQA",
          "metric": "acc",
          "value": 0.19983619983619982,
          "stderr": 0.01144844799672829,
          "ci95": [
            0.17836366711496662,
            0.2231915344000527
          ],
          "samples": 1221,
          "dataset": "tau/commonsense_qa",
          "split": "validation"
        },
        {
          "id": "mmlu_continuation",
          "label": "MMLU continuation",
          "metric": "acc",
          "value": 0.2583677538812135,
          "stderr": 0.003680818968618443,
          "ci95": null,
          "samples": 14042,
          "dataset": "cais/mmlu",
          "split": "test"
        },
        {
          "id": "sciq",
          "label": "SciQ (with support)",
          "metric": "acc_norm",
          "value": 0.642,
          "stderr": 0.015167928865407633,
          "ci95": [
            0.6117948042178514,
            0.6711183963722509
          ],
          "samples": 1000,
          "dataset": "allenai/sciq",
          "split": "test"
        },
        {
          "id": "truthfulqa_mc2",
          "label": "TruthfulQA MC2",
          "metric": "acc",
          "value": 0.4131035830305826,
          "stderr": 0.014947560271896813,
          "ci95": null,
          "samples": 817,
          "dataset": "truthfulqa/truthful_qa",
          "split": "validation"
        },
        {
          "id": "bananamind_base",
          "label": "BananaMind Base 1.1",
          "metric": "raw_accuracy",
          "value": 0.4828571428571429,
          "stderr": 0.026748650848283543,
          "ci95": [
            0.43097642743975617,
            0.5351100790990921
          ],
          "samples": 350,
          "dataset": "BananaMind/BananaMind-Base-Bench-1.1",
          "split": "test"
        }
      ],
      "continuation": {
        "nll": 3.438539972010347,
        "nll_ci95": [
          3.399929995389329,
          3.4766170177263085
        ],
        "ppl": 31.1414575606347,
        "ppl_ci95": [
          29.96200249565909,
          32.35009694595882
        ]
      },
      "continuation_precision": "torch.float32",
      "source_url": "https://huggingface.co/d0rj/q-moe-98M-A51M-base/blob/61700a031a87850816e263b4006615e27c28967b/evaluation/results.json",
      "source_sha256": "45af0c6edb0be62c26282213447dd2945f2481463721b97bce5ed6eac61e0526",
      "card_url": "https://huggingface.co/d0rj/q-moe-98M-A51M-base/blob/61700a031a87850816e263b4006615e27c28967b/README.md",
      "delta_pp": 0.22999196939912991
    },
    {
      "id": "q-latent-moe-52M-A35M-base",
      "repo": "d0rj/q-latent-moe-52M-A35M-base",
      "revision": "dda6dd624d8217576bb315cdb50ad7d6748b392c",
      "label": "LatentMoE 52M / A35M",
      "group": "causal",
      "parameters": 51615488,
      "active_parameters": 35395328,
      "optimized_parameters": null,
      "mean_16_percent": 42.97211735225452,
      "tasks": [
        {
          "id": "hellaswag",
          "label": "HellaSwag",
          "metric": "acc_norm",
          "value": 0.29067914758016333,
          "stderr": 0.004531477407589249,
          "ci95": [
            0.28187943754229217,
            0.29963894324912477
          ],
          "samples": 10042,
          "dataset": "Rowan/hellaswag",
          "split": "validation"
        },
        {
          "id": "arc_easy",
          "label": "ARC-Easy",
          "metric": "acc_norm",
          "value": 0.42676767676767674,
          "stderr": 0.010149141043955583,
          "ci95": [
            0.40701382739079034,
            0.44675794437570043
          ],
          "samples": 2376,
          "dataset": "allenai/ai2_arc",
          "split": "test"
        },
        {
          "id": "arc_challenge",
          "label": "ARC-Challenge",
          "metric": "acc_norm",
          "value": 0.23378839590443687,
          "stderr": 0.01236822537850722,
          "ci95": [
            0.210451158412582,
            0.25886505312342006
          ],
          "samples": 1172,
          "dataset": "allenai/ai2_arc",
          "split": "test"
        },
        {
          "id": "piqa",
          "label": "PIQA",
          "metric": "acc_norm",
          "value": 0.6011969532100109,
          "stderr": 0.011424390545037256,
          "ci95": [
            0.5786229488593012,
            0.623348832272277
          ],
          "samples": 1838,
          "dataset": "baber/piqa",
          "split": "validation"
        },
        {
          "id": "winogrande",
          "label": "WinoGrande",
          "metric": "acc",
          "value": 0.5201262825572218,
          "stderr": 0.014041096664344348,
          "ci95": [
            0.49259780520052165,
            0.5475330857505233
          ],
          "samples": 1267,
          "dataset": "allenai/winogrande",
          "split": "validation"
        },
        {
          "id": "openbookqa",
          "label": "OpenBookQA",
          "metric": "acc_norm",
          "value": 0.314,
          "stderr": 0.020776701920308973,
          "ci95": [
            0.27486785583609774,
            0.35596839881559983
          ],
          "samples": 500,
          "dataset": "allenai/openbookqa",
          "split": "test"
        },
        {
          "id": "boolq",
          "label": "BoolQ",
          "metric": "acc",
          "value": 0.6140672782874618,
          "stderr": 0.008514444495863416,
          "ci95": [
            0.5972572363147873,
            0.630609631822659
          ],
          "samples": 3270,
          "dataset": "aps/super_glue",
          "split": "validation"
        },
        {
          "id": "lambada_openai",
          "label": "LAMBADA OpenAI",
          "metric": "acc",
          "value": 0.20842227828449447,
          "stderr": 0.005658885722145609,
          "ci95": [
            0.197551349622851,
            0.2197276138674855
          ],
          "samples": 5153,
          "dataset": "EleutherAI/lambada_openai",
          "split": "test"
        },
        {
          "id": "arithmark3",
          "label": "ArithMark-3",
          "metric": "acc_norm",
          "value": 0.38,
          "stderr": 0.015356947477797658,
          "ci95": [
            0.3504293063571445,
            0.4104891156793888
          ],
          "samples": 1000,
          "dataset": "AxiomicLabs/Arithmark-3.0",
          "split": "train"
        },
        {
          "id": "balanced_copa",
          "label": "Balanced COPA",
          "metric": "acc",
          "value": 0.533,
          "stderr": 0.015784807891138876,
          "ci95": [
            0.5020104989134991,
            0.5637369350264544
          ],
          "samples": 1000,
          "dataset": "pkavumba/balanced-copa",
          "split": "train"
        },
        {
          "id": "blimp",
          "label": "BLiMP",
          "metric": "acc",
          "value": 0.7622089552238805,
          "stderr": 0.001431525610004643,
          "ci95": null,
          "samples": 67000,
          "dataset": "nyu-mll/blimp",
          "split": "train"
        },
        {
          "id": "commonsense_qa",
          "label": "CommonsenseQA",
          "metric": "acc",
          "value": 0.19901719901719903,
          "stderr": 0.011430809442838318,
          "ci95": [
            0.17758159876867657,
            0.22234073835757068
          ],
          "samples": 1221,
          "dataset": "tau/commonsense_qa",
          "split": "validation"
        },
        {
          "id": "mmlu_continuation",
          "label": "MMLU continuation",
          "metric": "acc",
          "value": 0.2516735507762427,
          "stderr": 0.0036496473554643834,
          "ci95": null,
          "samples": 14042,
          "dataset": "cais/mmlu",
          "split": "test"
        },
        {
          "id": "sciq",
          "label": "SciQ (with support)",
          "metric": "acc_norm",
          "value": 0.66,
          "stderr": 0.014987482264363972,
          "ci95": [
            0.6300773195897643,
            0.6886981176948582
          ],
          "samples": 1000,
          "dataset": "allenai/sciq",
          "split": "test"
        },
        {
          "id": "truthfulqa_mc2",
          "label": "TruthfulQA MC2",
          "metric": "acc",
          "value": 0.42059105875193525,
          "stderr": 0.01524134620662943,
          "ci95": null,
          "samples": 817,
          "dataset": "truthfulqa/truthful_qa",
          "split": "validation"
        },
        {
          "id": "bananamind_base",
          "label": "BananaMind Base 1.1",
          "metric": "raw_accuracy",
          "value": 0.46,
          "stderr": 0.02667860286827002,
          "ci95": [
            0.40850229778075775,
            0.5123662174765549
          ],
          "samples": 350,
          "dataset": "BananaMind/BananaMind-Base-Bench-1.1",
          "split": "test"
        }
      ],
      "continuation": {
        "nll": 3.5736877770768,
        "nll_ci95": [
          3.5348262685475893,
          3.6124454839532727
        ],
        "ppl": 35.647812234350084,
        "ppl_ci95": [
          34.28905710903879,
          37.05656332969549
        ]
      },
      "continuation_precision": "torch.float32",
      "source_url": "https://huggingface.co/d0rj/q-latent-moe-52M-A35M-base/blob/dda6dd624d8217576bb315cdb50ad7d6748b392c/evaluation/results.json",
      "source_sha256": "a9e0a09bf836e4a4fff460041f938a6e5b67b1c6f12013d07c3b737b34086019",
      "card_url": "https://huggingface.co/d0rj/q-latent-moe-52M-A35M-base/blob/dda6dd624d8217576bb315cdb50ad7d6748b392c/README.md",
      "delta_pp": 0.14796097658054208
    },
    {
      "id": "q-51M-base",
      "repo": "d0rj/q-51M-base",
      "revision": "09ba545cfd264e16961ef8df91b3d81a7c16a8a3",
      "label": "Q dense",
      "group": "causal",
      "parameters": 50878208,
      "active_parameters": null,
      "optimized_parameters": null,
      "mean_16_percent": 42.82415637567398,
      "tasks": [
        {
          "id": "hellaswag",
          "label": "hellaswag",
          "metric": "acc_norm",
          "value": 0.29177454690300736,
          "stderr": 0.004536500714147901,
          "ci95": [
            0.28296457902614003,
            0.3007437626650362
          ],
          "samples": 10042,
          "dataset": "Rowan/hellaswag",
          "split": "validation"
        },
        {
          "id": "arc_easy",
          "label": "arc_easy",
          "metric": "acc_norm",
          "value": 0.43308080808080807,
          "stderr": 0.010167478013701743,
          "ci95": [
            0.41328092357316454,
            0.45309672993759426
          ],
          "samples": 2376,
          "dataset": "allenai/ai2_arc",
          "split": "test"
        },
        {
          "id": "arc_challenge",
          "label": "arc_challenge",
          "metric": "acc_norm",
          "value": 0.24232081911262798,
          "stderr": 0.012521593295800071,
          "ci95": [
            0.21865689491479706,
            0.2676684124053075
          ],
          "samples": 1172,
          "dataset": "allenai/ai2_arc",
          "split": "test"
        },
        {
          "id": "piqa",
          "label": "piqa",
          "metric": "acc_norm",
          "value": 0.5990206746463548,
          "stderr": 0.011434766962108364,
          "ci95": [
            0.5764309418644076,
            0.6211973601030507
          ],
          "samples": 1838,
          "dataset": "baber/piqa",
          "split": "validation"
        },
        {
          "id": "winogrande",
          "label": "winogrande",
          "metric": "acc",
          "value": 0.500394632991318,
          "stderr": 0.01405248130604961,
          "ci95": [
            0.47290359623559514,
            0.5278832839791312
          ],
          "samples": 1267,
          "dataset": "allenai/winogrande",
          "split": "validation"
        },
        {
          "id": "openbookqa",
          "label": "openbookqa",
          "metric": "acc_norm",
          "value": 0.282,
          "stderr": 0.020143572847290726,
          "ci95": [
            0.2443364332272433,
            0.32298777921399346
          ],
          "samples": 500,
          "dataset": "allenai/openbookqa",
          "split": "test"
        },
        {
          "id": "boolq",
          "label": "boolq",
          "metric": "acc",
          "value": 0.598776758409786,
          "stderr": 0.008572708337179049,
          "ci95": [
            0.5818706845463824,
            0.615451027004275
          ],
          "samples": 3270,
          "dataset": "aps/super_glue",
          "split": "validation"
        },
        {
          "id": "lambada_openai",
          "label": "lambada_openai",
          "metric": "acc",
          "value": 0.20861633999611875,
          "stderr": 0.005660825573438258,
          "ci95": [
            0.1977414700784266,
            0.2199253277124155
          ],
          "samples": 5153,
          "dataset": "EleutherAI/lambada_openai",
          "split": "test"
        },
        {
          "id": "arithmark3",
          "label": "ArithMark-3",
          "metric": "acc_norm",
          "value": 0.367,
          "stderr": 0.015249378464171697,
          "ci95": [
            0.33768854423226846,
            0.3973293735248892
          ],
          "samples": 1000,
          "dataset": "AxiomicLabs/Arithmark-3.0",
          "split": "train"
        },
        {
          "id": "balanced_copa",
          "label": "Balanced COPA",
          "metric": "acc",
          "value": 0.546,
          "stderr": 0.01575221038877186,
          "ci95": [
            0.5150242420441904,
            0.5766236961751385
          ],
          "samples": 1000,
          "dataset": "pkavumba/balanced-copa",
          "split": "train"
        },
        {
          "id": "blimp",
          "label": "BLiMP",
          "metric": "acc",
          "value": 0.7726567164179104,
          "stderr": 0.0014498672475464,
          "ci95": null,
          "samples": 67000,
          "dataset": "nyu-mll/blimp",
          "split": "train"
        },
        {
          "id": "commonsense_qa",
          "label": "CommonsenseQA",
          "metric": "acc",
          "value": 0.19656019656019655,
          "stderr": 0.011377439773963867,
          "ci95": [
            0.17523627664679478,
            0.21978746731313606
          ],
          "samples": 1221,
          "dataset": "tau/commonsense_qa",
          "split": "validation"
        },
        {
          "id": "mmlu_continuation",
          "label": "MMLU continuation",
          "metric": "acc",
          "value": 0.2543797179888905,
          "stderr": 0.003665805402517398,
          "ci95": null,
          "samples": 14042,
          "dataset": "cais/mmlu",
          "split": "test"
        },
        {
          "id": "sciq",
          "label": "SciQ (with support)",
          "metric": "acc_norm",
          "value": 0.642,
          "stderr": 0.015167928865407633,
          "ci95": [
            0.6117948042178514,
            0.6711183963722509
          ],
          "samples": 1000,
          "dataset": "allenai/sciq",
          "split": "test"
        },
        {
          "id": "truthfulqa_mc2",
          "label": "TruthfulQA MC2",
          "metric": "acc",
          "value": 0.4372838090008176,
          "stderr": 0.01513251695187872,
          "ci95": null,
          "samples": 817,
          "dataset": "truthfulqa/truthful_qa",
          "split": "validation"
        },
        {
          "id": "bananamind_base",
          "label": "BananaMind Base 1.1",
          "metric": "raw_accuracy",
          "value": 0.48,
          "stderr": 0.026742966298044388,
          "ci95": [
            0.4281612478042309,
            0.5322730098244254
          ],
          "samples": 350,
          "dataset": "BananaMind/BananaMind-Base-Bench-1.1",
          "split": "test"
        }
      ],
      "continuation": {
        "nll": 3.560861705095088,
        "nll_ci95": [
          3.521807046434314,
          3.599506885284411
        ],
        "ppl": 35.19351050996088,
        "ppl_ci95": [
          33.84553368820155,
          36.580191764630115
        ]
      },
      "continuation_precision": "CPU FP32",
      "source_url": "https://huggingface.co/d0rj/q-51M-base/blob/09ba545cfd264e16961ef8df91b3d81a7c16a8a3/evaluation/results.json",
      "source_sha256": "f2d0ce9537df546a2d31578fbe9e537ac4fd22ae520aa54037f5c989d0b9e814",
      "card_url": "https://huggingface.co/d0rj/q-51M-base/blob/09ba545cfd264e16961ef8df91b3d81a7c16a8a3/README.md",
      "delta_pp": 0.0
    }
  ],
  "models": [
    {
      "id": "q-rrt-latent-moe",
      "label": "Q-RRT-LatentMoE 52M",
      "published": false,
      "inference_only": false,
      "parameters": 51689232,
      "mean_16_percent": 42.9403999615366,
      "checkpoint_sha256": {
        "config.json": "5eb3f268dc8b1f9ce42c016a66aca564162f1705e356c84d902b1aee8838f168",
        "generation_config.json": "80da72cc13262a67c14c8b96f4f049f6e7890a027d808a02b8f0b0d48704c245",
        "model.safetensors": "41eca8551d3e41e545964d303a8bb17fb492d0ac54c0372960b93424e649f5a1"
      },
      "model_config": {
        "architectures": [
          "QRRTLatentMoEForCausalLM"
        ],
        "attention_dropout": 0.0,
        "attention_scalar_gate": true,
        "bos_token_id": 1,
        "dtype": "float32",
        "eos_token_id": 2,
        "expert_backend": "auto",
        "expert_intermediate_size": 176,
        "experts_per_token": 4,
        "gate_multiplier": 2.0,
        "head_dim": 64,
        "hidden_size": 512,
        "intermediate_size": 1792,
        "latent_size": 256,
        "lora_alpha": 16,
        "lora_rank": 16,
        "max_position_embeddings": 2048,
        "mlp_scalar_gate": true,
        "model_type": "q_rrt_latent_moe",
        "moe_budget": "total",
        "nope_every_n": null,
        "nope_layers": [],
        "num_attention_heads": 8,
        "num_core_layers": 8,
        "num_experts": 16,
        "num_hidden_layers": 18,
        "num_key_value_heads": 2,
        "num_passes": 2,
        "pad_token_id": 0,
        "qk_norm": true,
        "relaxed": true,
        "rms_norm_eps": 1e-05,
        "rope_theta": 10000.0,
        "routed_scaling_factor": 1.0,
        "router_bias_speed": 0.001,
        "router_seq_aux_coef": 0.0001,
        "shared_intermediate_size": 256,
        "swiglu_limit": 10.0,
        "tie_word_embeddings": true,
        "transformers_version": "5.17.0",
        "vocab_size": 32768
      },
      "evaluation": {
        "harness": "0.4.12",
        "seed": 1234,
        "dtype": "BF16",
        "batch_size": 8,
        "max_length": 2048,
        "torch_tf32": false,
        "examples": 112048,
        "recorded_tb_points": 1244
      },
      "tasks": [
        {
          "id": "hellaswag",
          "label": "HellaSwag",
          "metric": "acc_norm",
          "value": 0.2998406691894045,
          "stderr": 0.004572515919210842,
          "ci95": [
            0.2909570750785615,
            0.30887734232902553
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 10042,
          "dataset": "Rowan/hellaswag",
          "config": "default",
          "split": "validation",
          "source": "runs/eval/additional-release-20261004/q-rrt-latent-moe/attempt-001/hellaswag.json",
          "source_sha256": "67dce652daf387a9bf8edad8f19f62c427ed52ad0b115f5ba1ae556bbcb13222"
        },
        {
          "id": "arc_easy",
          "label": "ARC-Easy",
          "metric": "acc_norm",
          "value": 0.4234006734006734,
          "stderr": 0.010138671005288915,
          "ci95": [
            0.40367272547029465,
            0.44337590936591986
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 2376,
          "dataset": "allenai/ai2_arc",
          "config": "ARC-Easy",
          "split": "test",
          "source": "runs/eval/additional-release-20261004/q-rrt-latent-moe/attempt-001/arc_easy.json",
          "source_sha256": "f5646d011e7796f9dc3e148aae58220c971cdb0549eda46e3298d4d695bc1dba"
        },
        {
          "id": "arc_challenge",
          "label": "ARC-Challenge",
          "metric": "acc_norm",
          "value": 0.25426621160409557,
          "stderr": 0.012724999945157845,
          "ci95": [
            0.23016693359420667,
            0.2799711078236413
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1172,
          "dataset": "allenai/ai2_arc",
          "config": "ARC-Challenge",
          "split": "test",
          "source": "runs/eval/additional-release-20261004/q-rrt-latent-moe/attempt-001/arc_challenge.json",
          "source_sha256": "484874f386837d37ae5f0b95d6004432dc2f15f1fe30b3c881a1ae66f4508870"
        },
        {
          "id": "piqa",
          "label": "PIQA",
          "metric": "acc_norm",
          "value": 0.6082698585418934,
          "stderr": 0.011389038597720108,
          "ci95": [
            0.5857501522877211,
            0.6303379361272472
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1838,
          "dataset": "baber/piqa",
          "config": "default",
          "split": "validation",
          "source": "runs/eval/additional-release-20261004/q-rrt-latent-moe/attempt-001/piqa.json",
          "source_sha256": "69df5e3632183d69a772698771b229a313f0fff17cc34939d19dbece18ec0a3d"
        },
        {
          "id": "winogrande",
          "label": "WinoGrande",
          "metric": "acc",
          "value": 0.5311760063141279,
          "stderr": 0.014025142640639542,
          "ci95": [
            0.5036452434296269,
            0.5585182935337564
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1267,
          "dataset": "allenai/winogrande",
          "config": "winogrande_xl",
          "split": "validation",
          "source": "runs/eval/additional-release-20261004/q-rrt-latent-moe/attempt-001/winogrande.json",
          "source_sha256": "6fcf11ba230633eb7b2430041f3317dcac0d239898cb1a33444900126da485b6"
        },
        {
          "id": "openbookqa",
          "label": "OpenBookQA",
          "metric": "acc_norm",
          "value": 0.288,
          "stderr": 0.020271503835075255,
          "ci95": [
            0.250043271115854,
            0.32918944923984417
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 500,
          "dataset": "allenai/openbookqa",
          "config": "main",
          "split": "test",
          "source": "runs/eval/additional-release-20261004/q-rrt-latent-moe/attempt-001/openbookqa.json",
          "source_sha256": "a1113e28e3b6ebc2bf0df8e6a2e437569f7638fcaf8d401cc8c09a65e7be25bc"
        },
        {
          "id": "boolq",
          "label": "BoolQ",
          "metric": "acc",
          "value": 0.5792048929663609,
          "stderr": 0.008634635146574832,
          "ci95": [
            0.5622006444627536,
            0.5960232666568137
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 3270,
          "dataset": "aps/super_glue",
          "config": "boolq",
          "split": "validation",
          "source": "runs/eval/additional-release-20261004/q-rrt-latent-moe/attempt-001/boolq.json",
          "source_sha256": "9e5005b39d5daa7d6755c6706e540d3e79d19f77bf67f19e58ccc61bc0163d6b"
        },
        {
          "id": "lambada_openai",
          "label": "LAMBADA OpenAI",
          "metric": "acc",
          "value": 0.22530564719580826,
          "stderr": 0.005820540380558926,
          "ci95": [
            0.21410574482357514,
            0.2369148028107485
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 5153,
          "dataset": "EleutherAI/lambada_openai",
          "config": "default",
          "split": "test",
          "source": "runs/eval/additional-release-20261004/q-rrt-latent-moe/attempt-001/lambada_openai.json",
          "source_sha256": "dd08c1d34e6de875acac56a7201e3c4bc81ba9f78ff43fece95eb7259f8915e2"
        },
        {
          "id": "arithmark3",
          "label": "ArithMark-3",
          "metric": "acc_norm",
          "value": 0.361,
          "stderr": 0.015195720118175049,
          "ci95": [
            0.3318160018268193,
            0.391247837032165
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1000,
          "dataset": "AxiomicLabs/Arithmark-3.0",
          "config": "default",
          "split": "train",
          "source": "runs/eval/additional-release-20261004/q-rrt-latent-moe/attempt-001/arithmark3.json",
          "source_sha256": "a0063af93fb27f1945953816ff85cb419ef22249bde17c933502301dc6d1d540"
        },
        {
          "id": "balanced_copa",
          "label": "Balanced COPA",
          "metric": "acc",
          "value": 0.541,
          "stderr": 0.015766025737882314,
          "ci95": [
            0.5100164673887266,
            0.5716697384154579
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1000,
          "dataset": "pkavumba/balanced-copa",
          "config": "default",
          "split": "train",
          "source": "runs/eval/additional-release-20261004/q-rrt-latent-moe/attempt-001/balanced_copa.json",
          "source_sha256": "d347d767ce296f72c4a5241ba0b0cd5e26e72e6b0447f73fd08c00581d17ec2f"
        },
        {
          "id": "blimp",
          "label": "BLiMP",
          "metric": "acc",
          "value": 0.7580597014925373,
          "stderr": 0.0014292030539500716,
          "samples": 67000,
          "dataset": "nyu-mll/blimp",
          "config": "67 minimal-pair subsets",
          "split": "train",
          "source": "runs/eval/additional-release-20261004/q-rrt-latent-moe/attempt-001/blimp.json",
          "source_sha256": "d2535fc91f6f03827af0517ed517f7c70b39f9cf688e324e46b3e4129179ee8f"
        },
        {
          "id": "commonsense_qa",
          "label": "CommonsenseQA",
          "metric": "acc",
          "value": 0.1981981981981982,
          "stderr": 0.011413095456219376,
          "ci95": [
            0.17679967712019065,
            0.22148979561728446
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1221,
          "dataset": "tau/commonsense_qa",
          "config": "default",
          "split": "validation",
          "source": "runs/eval/additional-release-20261004/q-rrt-latent-moe/attempt-001/commonsense_qa.json",
          "source_sha256": "767c2205d7c434dae9fd3423d7e54218654fa8ae4e02babec1fdaa660050e49c"
        },
        {
          "id": "mmlu_continuation",
          "label": "MMLU continuation",
          "metric": "acc",
          "value": 0.2542372881355932,
          "stderr": 0.0036624818421183195,
          "samples": 14042,
          "dataset": "cais/mmlu",
          "config": "57 subjects",
          "split": "test",
          "source": "runs/eval/additional-release-20261004/q-rrt-latent-moe/attempt-001/mmlu_continuation.json",
          "source_sha256": "b185fb5e5a68d9e0cc862c32e596334f06ee2189fbcb03a7c55fe5447f3e6bf2"
        },
        {
          "id": "sciq",
          "label": "SciQ (with support)",
          "metric": "acc_norm",
          "value": 0.651,
          "stderr": 0.015080663991563052,
          "ci95": [
            0.6209303029513429,
            0.6799140159860195
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1000,
          "dataset": "allenai/sciq",
          "config": "default",
          "split": "test",
          "source": "runs/eval/additional-release-20261004/q-rrt-latent-moe/attempt-001/sciq.json",
          "source_sha256": "a02475ee5cbc412d031ea1d3156c3bd0e14bef73860233a80950d27ade10cec7"
        },
        {
          "id": "truthfulqa_mc2",
          "label": "TruthfulQA MC2",
          "metric": "acc",
          "value": 0.42321913252144866,
          "stderr": 0.015087295136182781,
          "samples": 817,
          "dataset": "truthfulqa/truthful_qa",
          "config": "multiple_choice",
          "split": "validation",
          "source": "runs/eval/additional-release-20261004/q-rrt-latent-moe/attempt-001/truthfulqa_mc2.json",
          "source_sha256": "0aa7d4ca729dc1fdb160b1b4d66bff2fbd9ffaccbc58d44f41750af7a5f4c69e"
        },
        {
          "id": "bananamind_base",
          "label": "BananaMind Base 1.1",
          "metric": "raw_accuracy",
          "value": 0.4742857142857143,
          "stderr": 0.02672896840841852,
          "ci95": [
            0.4225359497972163,
            0.5265938100110561
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 350,
          "dataset": "BananaMind/BananaMind-Base-Bench-1.1",
          "config": "default",
          "split": "test",
          "source": "runs/eval/additional-release-20261004/q-rrt-latent-moe/attempt-001/bananamind_base.json",
          "source_sha256": "c982eb020a764cb04f84b3e1c15c8de297ee56e09292e1f712b4aa3c7f00896b"
        }
      ],
      "continuation": {
        "source": "runs/eval/additional-release-20261004/q-rrt-latent-moe/continuation.json",
        "source_sha256": "70746d11e820ed25d08345475d37d610e275039b813c0f297246ecaea189531f",
        "blocks": 291,
        "scored_tokens": 148992,
        "corpus_sha256": "aca2f46735043bcfd0a44eca981d04627b9cdf74c4c9a04bf0856d04066f58fc",
        "tokenizer_sha256": "fc71a154536f6c84a5bf729d06324032126603ef5183c0b4979407e4fad68a8d",
        "bf16": {
          "nll": 3.4708731215434385,
          "nll_ci95": [
            3.431748920576679,
            3.5096337493957113
          ],
          "ppl": 32.16481398006538,
          "ppl_ci95": [
            30.93069079461318,
            33.43601957874982
          ]
        },
        "fp32": {
          "nll": 3.470638575013151,
          "nll_ci95": [
            3.4315105441304827,
            3.5093734688775235
          ],
          "ppl": 32.15727071920642,
          "ppl_ci95": [
            30.923318525185763,
            33.427317966726896
          ]
        }
      }
    },
    {
      "id": "q-rrt-delta",
      "label": "Q-RRT-Delta 53M",
      "published": false,
      "inference_only": false,
      "parameters": 53329392,
      "mean_16_percent": 42.766580261717735,
      "checkpoint_sha256": {
        "config.json": "75216a7265ca982a4b8cb0bc3455e6cc5805a8cf12dffa1eb2a5ab3ad05d4338",
        "generation_config.json": "80da72cc13262a67c14c8b96f4f049f6e7890a027d808a02b8f0b0d48704c245",
        "model.safetensors": "c6e810673bf1046a6ed003064aca8ee3d6e6fd89a00732159880e8bd095c4b77"
      },
      "model_config": {
        "architectures": [
          "QRRTDeltaLatentMoEForCausalLM"
        ],
        "attention_dropout": 0.0,
        "attention_scalar_gate": true,
        "bos_token_id": 1,
        "delta_backend": "auto",
        "delta_conv_kernel": 4,
        "dtype": "float32",
        "eos_token_id": 2,
        "expert_backend": "auto",
        "expert_intermediate_size": 176,
        "experts_per_token": 4,
        "gate_multiplier": 2.0,
        "head_dim": 64,
        "hidden_size": 512,
        "intermediate_size": 1792,
        "latent_size": 256,
        "layer_types": [
          "full_attention",
          "linear_attention",
          "linear_attention",
          "linear_attention",
          "full_attention",
          "linear_attention",
          "linear_attention",
          "linear_attention",
          "full_attention",
          "linear_attention",
          "linear_attention",
          "linear_attention",
          "full_attention",
          "linear_attention",
          "linear_attention",
          "linear_attention",
          "full_attention",
          "full_attention"
        ],
        "lora_alpha": 16,
        "lora_rank": 16,
        "max_position_embeddings": 2048,
        "mlp_scalar_gate": true,
        "model_type": "q_rrt_delta_latent_moe",
        "moe_budget": "total",
        "nope_every_n": null,
        "nope_layers": [],
        "num_attention_heads": 8,
        "num_core_layers": 8,
        "num_experts": 16,
        "num_hidden_layers": 18,
        "num_key_value_heads": 2,
        "num_passes": 2,
        "pad_token_id": 0,
        "qk_norm": true,
        "relaxed": true,
        "rms_norm_eps": 1e-05,
        "rope_theta": 10000.0,
        "routed_scaling_factor": 1.0,
        "router_bias_speed": 0.001,
        "router_seq_aux_coef": 0.0001,
        "shared_intermediate_size": 256,
        "swiglu_limit": 10.0,
        "tie_word_embeddings": true,
        "transformers_version": "5.17.0",
        "vocab_size": 32768
      },
      "evaluation": {
        "harness": "0.4.12",
        "seed": 1234,
        "dtype": "BF16",
        "batch_size": 8,
        "max_length": 2048,
        "torch_tf32": false,
        "examples": 112048,
        "recorded_tb_points": 1244
      },
      "tasks": [
        {
          "id": "hellaswag",
          "label": "HellaSwag",
          "metric": "acc_norm",
          "value": 0.30083648675562635,
          "stderr": 0.00457684440743003,
          "ci95": [
            0.29194403376553785,
            0.30988125718728504
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 10042,
          "dataset": "Rowan/hellaswag",
          "config": "default",
          "split": "validation",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta/attempt-001/hellaswag.json",
          "source_sha256": "d8e5f9be7cbc4227c5c9ed9aceb403a89f4084a6ba2b5d6c90145ed1c18320d0"
        },
        {
          "id": "arc_easy",
          "label": "ARC-Easy",
          "metric": "acc_norm",
          "value": 0.43686868686868685,
          "stderr": 0.0101776729281577,
          "ci95": [
            0.41704275926445944,
            0.4568984232928601
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 2376,
          "dataset": "allenai/ai2_arc",
          "config": "ARC-Easy",
          "split": "test",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta/attempt-001/arc_easy.json",
          "source_sha256": "14727fdfe0ec3069339e35528ba76790d04c418caad86b7a44166c90355338e1"
        },
        {
          "id": "arc_challenge",
          "label": "ARC-Challenge",
          "metric": "acc_norm",
          "value": 0.23976109215017063,
          "stderr": 0.012476304127453939,
          "ci95": [
            0.2161937705339983,
            0.2650288080508755
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1172,
          "dataset": "allenai/ai2_arc",
          "config": "ARC-Challenge",
          "split": "test",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta/attempt-001/arc_challenge.json",
          "source_sha256": "a717f44e8cb9c2652bd640f71fafa299fa6dabec4fad58897dc315acd92baba2"
        },
        {
          "id": "piqa",
          "label": "PIQA",
          "metric": "acc_norm",
          "value": 0.5903155603917302,
          "stderr": 0.011473932007187755,
          "ci95": [
            0.5676674852752623,
            0.612586900035716
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1838,
          "dataset": "baber/piqa",
          "config": "default",
          "split": "validation",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta/attempt-001/piqa.json",
          "source_sha256": "b4f0bf809d790985ec740cf32683c3537bb274f009f2436e75c2da5ca535c90b"
        },
        {
          "id": "winogrande",
          "label": "WinoGrande",
          "metric": "acc",
          "value": 0.5098658247829518,
          "stderr": 0.014049749833367674,
          "ci95": [
            0.4823514860396483,
            0.5373205193285109
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1267,
          "dataset": "allenai/winogrande",
          "config": "winogrande_xl",
          "split": "validation",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta/attempt-001/winogrande.json",
          "source_sha256": "4f644777b71bfc23c327d02a787667228b9b37279214577be36b5a8a718b3a5f"
        },
        {
          "id": "openbookqa",
          "label": "OpenBookQA",
          "metric": "acc_norm",
          "value": 0.288,
          "stderr": 0.020271503835075255,
          "ci95": [
            0.250043271115854,
            0.32918944923984417
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 500,
          "dataset": "allenai/openbookqa",
          "config": "main",
          "split": "test",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta/attempt-001/openbookqa.json",
          "source_sha256": "8adb9c4a7ede12963efd4a9a8837bfbc554528ebb2f03f4506bea9672bcc7788"
        },
        {
          "id": "boolq",
          "label": "BoolQ",
          "metric": "acc",
          "value": 0.572782874617737,
          "stderr": 0.008651907722486125,
          "ci95": [
            0.5557523731574014,
            0.5896425721957146
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 3270,
          "dataset": "aps/super_glue",
          "config": "boolq",
          "split": "validation",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta/attempt-001/boolq.json",
          "source_sha256": "9f1b687d97c51229b5f8083e5d8c2c29dfd77082838fb2225082f4a38b148072"
        },
        {
          "id": "lambada_openai",
          "label": "LAMBADA OpenAI",
          "metric": "acc",
          "value": 0.21385600620997477,
          "stderr": 0.0057124689069429505,
          "ci95": [
            0.20287615579625212,
            0.22526216810824118
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 5153,
          "dataset": "EleutherAI/lambada_openai",
          "config": "default",
          "split": "test",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta/attempt-001/lambada_openai.json",
          "source_sha256": "5f3d4108df36abce143015a4ec65837d19a257dd70d6eddf133613bcb936d5e8"
        },
        {
          "id": "arithmark3",
          "label": "ArithMark-3",
          "metric": "acc_norm",
          "value": 0.349,
          "stderr": 0.015080663991563052,
          "ci95": [
            0.3200859840139804,
            0.3790696970486572
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1000,
          "dataset": "AxiomicLabs/Arithmark-3.0",
          "config": "default",
          "split": "train",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta/attempt-001/arithmark3.json",
          "source_sha256": "5815a525814d142e48e9c7d0ee9aa4c023a7125d7353a61c1a7fc62541d2b1e2"
        },
        {
          "id": "balanced_copa",
          "label": "Balanced COPA",
          "metric": "acc",
          "value": 0.544,
          "stderr": 0.01575792855397918,
          "ci95": [
            0.5130207581807846,
            0.5746424870724867
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1000,
          "dataset": "pkavumba/balanced-copa",
          "config": "default",
          "split": "train",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta/attempt-001/balanced_copa.json",
          "source_sha256": "083e640e91a5981a81463073da9373cce31c3e549a939aa1ea2520da20ce5bc7"
        },
        {
          "id": "blimp",
          "label": "BLiMP",
          "metric": "acc",
          "value": 0.7848358208955224,
          "stderr": 0.0014126939956689614,
          "samples": 67000,
          "dataset": "nyu-mll/blimp",
          "config": "67 minimal-pair subsets",
          "split": "train",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta/attempt-001/blimp.json",
          "source_sha256": "13b0fae1adf630df387e4349d6964226e280902dd027dd29e2814c9d2d76c9e5"
        },
        {
          "id": "commonsense_qa",
          "label": "CommonsenseQA",
          "metric": "acc",
          "value": 0.19737919737919737,
          "stderr": 0.011395305685091258,
          "ci95": [
            0.17601790285026156,
            0.22063870549844142
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1221,
          "dataset": "tau/commonsense_qa",
          "config": "default",
          "split": "validation",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta/attempt-001/commonsense_qa.json",
          "source_sha256": "cb7ff79dfe2dd923fdb83feb1030c366cd2cabb48a169a90679796c40f274734"
        },
        {
          "id": "mmlu_continuation",
          "label": "MMLU continuation",
          "metric": "acc",
          "value": 0.25395242842899873,
          "stderr": 0.0036595004816842858,
          "samples": 14042,
          "dataset": "cais/mmlu",
          "config": "57 subjects",
          "split": "test",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta/attempt-001/mmlu_continuation.json",
          "source_sha256": "e5fb2b5dae0bd694726830f92ed78ef19722fc9ba10bc9b065b44230008cbec7"
        },
        {
          "id": "sciq",
          "label": "SciQ (with support)",
          "metric": "acc_norm",
          "value": 0.65,
          "stderr": 0.015090650341444127,
          "ci95": [
            0.6199146825068091,
            0.6789372899475243
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1000,
          "dataset": "allenai/sciq",
          "config": "default",
          "split": "test",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta/attempt-001/sciq.json",
          "source_sha256": "36f1460ec27b9dafa729fd66365195a88fcd02f159839eb3d8a5adb737f4fb6e"
        },
        {
          "id": "truthfulqa_mc2",
          "label": "TruthfulQA MC2",
          "metric": "acc",
          "value": 0.44262743482281286,
          "stderr": 0.01519474820616521,
          "samples": 817,
          "dataset": "truthfulqa/truthful_qa",
          "config": "multiple_choice",
          "split": "validation",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta/attempt-001/truthfulqa_mc2.json",
          "source_sha256": "37783413b44d12c565d586200de5f54a17b7d7a3aa57fafdea1944287c0344bc"
        },
        {
          "id": "bananamind_base",
          "label": "BananaMind Base 1.1",
          "metric": "raw_accuracy",
          "value": 0.4685714285714286,
          "stderr": 0.026711460730894265,
          "ci95": [
            0.41691740916070796,
            0.5209078528271806
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 350,
          "dataset": "BananaMind/BananaMind-Base-Bench-1.1",
          "config": "default",
          "split": "test",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta/attempt-001/bananamind_base.json",
          "source_sha256": "37fdbf818c867cba7d4aeb45f762b9067b2d4a45d82ed9321a0ee9bf3d0293cc"
        }
      ],
      "continuation": {
        "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta/continuation.json",
        "source_sha256": "ed18ee67346f964f675fcfa408d879336672083e4ff38102d4e14a4b6fb02071",
        "blocks": 291,
        "scored_tokens": 148992,
        "corpus_sha256": "aca2f46735043bcfd0a44eca981d04627b9cdf74c4c9a04bf0856d04066f58fc",
        "tokenizer_sha256": "fc71a154536f6c84a5bf729d06324032126603ef5183c0b4979407e4fad68a8d",
        "bf16": {
          "nll": 3.4578770853809475,
          "nll_ci95": [
            3.418976838932824,
            3.4966521702476383
          ],
          "ppl": 31.749503435224472,
          "ppl_ci95": [
            30.538153581225583,
            33.00477243483789
          ]
        },
        "fp32": {
          "nll": 3.4576403512987484,
          "nll_ci95": [
            3.4187069021344594,
            3.4963111575936123
          ],
          "ppl": 31.741988135267373,
          "ppl_ci95": [
            30.52991132231444,
            32.993519308632635
          ]
        }
      }
    },
    {
      "id": "q-rrt-delta-beta2",
      "label": "Q-RRT-Delta beta2 53M",
      "published": false,
      "inference_only": false,
      "parameters": 53329392,
      "mean_16_percent": 42.86895351968607,
      "checkpoint_sha256": {
        "config.json": "4bee2a8ddb8be8bd7914bfc472bbf535fc1d24a59cbe5205454fc1318daaf51d",
        "generation_config.json": "80da72cc13262a67c14c8b96f4f049f6e7890a027d808a02b8f0b0d48704c245",
        "model.safetensors": "2946ad7e13f82c4eeca690d2ede8278e8e39bda67aa3aa2176a0880daa8b6aae"
      },
      "model_config": {
        "architectures": [
          "QRRTDeltaBeta2LatentMoEForCausalLM"
        ],
        "attention_dropout": 0.0,
        "attention_scalar_gate": true,
        "bos_token_id": 1,
        "delta_backend": "auto",
        "delta_beta_multiplier": 2.0,
        "delta_conv_kernel": 4,
        "dtype": "float32",
        "eos_token_id": 2,
        "expert_backend": "auto",
        "expert_intermediate_size": 176,
        "experts_per_token": 4,
        "gate_multiplier": 2.0,
        "head_dim": 64,
        "hidden_size": 512,
        "intermediate_size": 1792,
        "latent_size": 256,
        "layer_types": [
          "full_attention",
          "linear_attention",
          "linear_attention",
          "linear_attention",
          "full_attention",
          "linear_attention",
          "linear_attention",
          "linear_attention",
          "full_attention",
          "linear_attention",
          "linear_attention",
          "linear_attention",
          "full_attention",
          "linear_attention",
          "linear_attention",
          "linear_attention",
          "full_attention",
          "full_attention"
        ],
        "lora_alpha": 16,
        "lora_rank": 16,
        "max_position_embeddings": 2048,
        "mlp_scalar_gate": true,
        "model_type": "q_rrt_delta_beta2_latent_moe",
        "moe_budget": "total",
        "nope_every_n": null,
        "nope_layers": [],
        "num_attention_heads": 8,
        "num_core_layers": 8,
        "num_experts": 16,
        "num_hidden_layers": 18,
        "num_key_value_heads": 2,
        "num_passes": 2,
        "pad_token_id": 0,
        "qk_norm": true,
        "relaxed": true,
        "rms_norm_eps": 1e-05,
        "rope_theta": 10000.0,
        "routed_scaling_factor": 1.0,
        "router_bias_speed": 0.001,
        "router_seq_aux_coef": 0.0001,
        "shared_intermediate_size": 256,
        "swiglu_limit": 10.0,
        "tie_word_embeddings": true,
        "transformers_version": "5.17.0",
        "vocab_size": 32768
      },
      "evaluation": {
        "harness": "0.4.12",
        "seed": 1234,
        "dtype": "BF16",
        "batch_size": 8,
        "max_length": 2048,
        "torch_tf32": false,
        "examples": 112048,
        "recorded_tb_points": 1244
      },
      "tasks": [
        {
          "id": "hellaswag",
          "label": "HellaSwag",
          "metric": "acc_norm",
          "value": 0.29954192391953793,
          "stderr": 0.004571212360565399,
          "ci95": [
            0.29066099728896555,
            0.3085761580550507
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 10042,
          "dataset": "Rowan/hellaswag",
          "config": "default",
          "split": "validation",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-beta2/attempt-001/hellaswag.json",
          "source_sha256": "4f0f047889d769aa6b094d101e8f71ee8eb1cdc86f4aad76d6ea5fb21ec58dc9"
        },
        {
          "id": "arc_easy",
          "label": "ARC-Easy",
          "metric": "acc_norm",
          "value": 0.43813131313131315,
          "stderr": 0.010180937100600149,
          "ci95": [
            0.41829696668747757,
            0.45816539221869573
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 2376,
          "dataset": "allenai/ai2_arc",
          "config": "ARC-Easy",
          "split": "test",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-beta2/attempt-001/arc_easy.json",
          "source_sha256": "3b571f55d72082d3193fa95692c641e34c7e1e0b83ed8e0177e7f2d8e127e9cc"
        },
        {
          "id": "arc_challenge",
          "label": "ARC-Challenge",
          "metric": "acc_norm",
          "value": 0.2363481228668942,
          "stderr": 0.012414960524301884,
          "ci95": [
            0.21291146902948319,
            0.2615074712417496
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1172,
          "dataset": "allenai/ai2_arc",
          "config": "ARC-Challenge",
          "split": "test",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-beta2/attempt-001/arc_challenge.json",
          "source_sha256": "a169097a8cbdad487d8993f061348e28baeb3c3dad105d0da92d622a5efc390c"
        },
        {
          "id": "piqa",
          "label": "PIQA",
          "metric": "acc_norm",
          "value": 0.6131664853101197,
          "stderr": 0.011363095931902692,
          "ci95": [
            0.5906872375306749,
            0.6351736790035631
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1838,
          "dataset": "baber/piqa",
          "config": "default",
          "split": "validation",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-beta2/attempt-001/piqa.json",
          "source_sha256": "584a72319b05ab6ba8e9b42eec6462514e1809ad63ad91639e09c3b0754fea2a"
        },
        {
          "id": "winogrande",
          "label": "WinoGrande",
          "metric": "acc",
          "value": 0.4996053670086819,
          "stderr": 0.01405248130604961,
          "ci95": [
            0.4721167160208688,
            0.5270964037644049
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1267,
          "dataset": "allenai/winogrande",
          "config": "winogrande_xl",
          "split": "validation",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-beta2/attempt-001/winogrande.json",
          "source_sha256": "03012b925ba65d216df9a24538b6590855cb0ad2dd147dc756979773cbf03788"
        },
        {
          "id": "openbookqa",
          "label": "OpenBookQA",
          "metric": "acc_norm",
          "value": 0.264,
          "stderr": 0.01973288558592215,
          "ci95": [
            0.22726783141305193,
            0.3043308572848008
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 500,
          "dataset": "allenai/openbookqa",
          "config": "main",
          "split": "test",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-beta2/attempt-001/openbookqa.json",
          "source_sha256": "3ab4d78711c4c5fc28ccfebdb192567a895ea79b29f57fa1a90c6f2744b7df67"
        },
        {
          "id": "boolq",
          "label": "BoolQ",
          "metric": "acc",
          "value": 0.5418960244648318,
          "stderr": 0.00871430091403349,
          "ci95": [
            0.5247797114627903,
            0.5589140175850116
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 3270,
          "dataset": "aps/super_glue",
          "config": "boolq",
          "split": "validation",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-beta2/attempt-001/boolq.json",
          "source_sha256": "715ef497dbdf3a9e444d63d5208ec0e124d5893626ac5e724dda35eba62c98d3"
        },
        {
          "id": "lambada_openai",
          "label": "LAMBADA OpenAI",
          "metric": "acc",
          "value": 0.23791965845138754,
          "stderr": 0.005932361614015241,
          "ci95": [
            0.226491493128481,
            0.24973828403870663
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 5153,
          "dataset": "EleutherAI/lambada_openai",
          "config": "default",
          "split": "test",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-beta2/attempt-001/lambada_openai.json",
          "source_sha256": "2407f9400e95f5b5307beab9d70cdeaf8b4dd2dd0856b48bf965e25b06b2de5c"
        },
        {
          "id": "arithmark3",
          "label": "ArithMark-3",
          "metric": "acc_norm",
          "value": 0.371,
          "stderr": 0.015283736211823096,
          "ci95": [
            0.341606326344111,
            0.40138097734516226
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1000,
          "dataset": "AxiomicLabs/Arithmark-3.0",
          "config": "default",
          "split": "train",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-beta2/attempt-001/arithmark3.json",
          "source_sha256": "ced917334b3e45d828286f56d74b8e5fff924596cffc655716d2294065315937"
        },
        {
          "id": "balanced_copa",
          "label": "Balanced COPA",
          "metric": "acc",
          "value": 0.553,
          "stderr": 0.015730176046009098,
          "ci95": [
            0.5220403713020199,
            0.5835539922985113
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1000,
          "dataset": "pkavumba/balanced-copa",
          "config": "default",
          "split": "train",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-beta2/attempt-001/balanced_copa.json",
          "source_sha256": "f2d71bc5afa4cf87fd4c0d04f4fca2e8921dc7cf8c559e447fa633705e444f59"
        },
        {
          "id": "blimp",
          "label": "BLiMP",
          "metric": "acc",
          "value": 0.797134328358209,
          "stderr": 0.0013779276489519732,
          "samples": 67000,
          "dataset": "nyu-mll/blimp",
          "config": "67 minimal-pair subsets",
          "split": "train",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-beta2/attempt-001/blimp.json",
          "source_sha256": "f77b153fbe1d77ee7566d58cfe789590ac08c92dc056a1b78c07f19f304ffd02"
        },
        {
          "id": "commonsense_qa",
          "label": "CommonsenseQA",
          "metric": "acc",
          "value": 0.19901719901719903,
          "stderr": 0.011430809442838318,
          "ci95": [
            0.17758159876867657,
            0.22234073835757068
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1221,
          "dataset": "tau/commonsense_qa",
          "config": "default",
          "split": "validation",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-beta2/attempt-001/commonsense_qa.json",
          "source_sha256": "09533875271eb44049ab971e69653e011935974862cd9b1bfb414f9d54f3e757"
        },
        {
          "id": "mmlu_continuation",
          "label": "MMLU continuation",
          "metric": "acc",
          "value": 0.2553767269619712,
          "stderr": 0.003667765071810069,
          "samples": 14042,
          "dataset": "cais/mmlu",
          "config": "57 subjects",
          "split": "test",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-beta2/attempt-001/mmlu_continuation.json",
          "source_sha256": "1c1ff8317339438822ec1dfbabe1842d9628eba5b18b49976c13a8890dcada91"
        },
        {
          "id": "sciq",
          "label": "SciQ (with support)",
          "metric": "acc_norm",
          "value": 0.648,
          "stderr": 0.015110404505648562,
          "ci95": [
            0.6178838670299425,
            0.6769834124583332
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1000,
          "dataset": "allenai/sciq",
          "config": "default",
          "split": "test",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-beta2/attempt-001/sciq.json",
          "source_sha256": "494ae8d5d76490e1ac6537f1f0fb9c121f705c994015dbb9f33e007d55b08654"
        },
        {
          "id": "truthfulqa_mc2",
          "label": "TruthfulQA MC2",
          "metric": "acc",
          "value": 0.43060969937391114,
          "stderr": 0.015199078525717995,
          "samples": 817,
          "dataset": "truthfulqa/truthful_qa",
          "config": "multiple_choice",
          "split": "validation",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-beta2/attempt-001/truthfulqa_mc2.json",
          "source_sha256": "70a73abdb327157146c5ed61a2a6649e085b7645ed1155afc53618eae9807c6c"
        },
        {
          "id": "bananamind_base",
          "label": "BananaMind Base 1.1",
          "metric": "raw_accuracy",
          "value": 0.4742857142857143,
          "stderr": 0.02672896840841852,
          "ci95": [
            0.4225359497972163,
            0.5265938100110561
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 350,
          "dataset": "BananaMind/BananaMind-Base-Bench-1.1",
          "config": "default",
          "split": "test",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-beta2/attempt-001/bananamind_base.json",
          "source_sha256": "68ea9ab3f0d3c0ee1064d2f5808794316b7d5779eea5ef7e41431a16ab8f0163"
        }
      ],
      "continuation": {
        "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-beta2/continuation.json",
        "source_sha256": "8bea3907f1244ded13e7f7dc7a1cba66b3a1d2cc1c6dfc3b7546a311fe94a71f",
        "blocks": 291,
        "scored_tokens": 148992,
        "corpus_sha256": "aca2f46735043bcfd0a44eca981d04627b9cdf74c4c9a04bf0856d04066f58fc",
        "tokenizer_sha256": "fc71a154536f6c84a5bf729d06324032126603ef5183c0b4979407e4fad68a8d",
        "bf16": {
          "nll": 3.482951736122472,
          "nll_ci95": [
            3.443992311663644,
            3.5213862030981335
          ],
          "ppl": 32.55567615593404,
          "ppl_ci95": [
            31.311715093718906,
            33.83129301764523
          ]
        },
        "fp32": {
          "nll": 3.48239857716249,
          "nll_ci95": [
            3.443376019259089,
            3.5208638490065676
          ],
          "ppl": 32.537672671823444,
          "ppl_ci95": [
            31.292423866662013,
            33.813625718009334
          ]
        }
      }
    },
    {
      "id": "q-rrt-delta-canon",
      "label": "Q-RRT-Delta Canon 53M",
      "published": false,
      "inference_only": false,
      "parameters": 53349872,
      "mean_16_percent": 42.73234274910314,
      "checkpoint_sha256": {
        "config.json": "62d00693e173a0948db38c4a41b5d4417696a1687fa6254d1300aff4fd779494",
        "generation_config.json": "80da72cc13262a67c14c8b96f4f049f6e7890a027d808a02b8f0b0d48704c245",
        "model.safetensors": "3a4d0d9ee6abc23cd6e5783a65555276c5dc6b42971e47715f40d90c87500e08"
      },
      "model_config": {
        "architectures": [
          "QRRTDeltaCanonLatentMoEForCausalLM"
        ],
        "attention_dropout": 0.0,
        "attention_scalar_gate": true,
        "bos_token_id": 1,
        "canon_kernel_size": 4,
        "delta_backend": "auto",
        "delta_conv_kernel": 4,
        "dtype": "float32",
        "eos_token_id": 2,
        "expert_backend": "auto",
        "expert_intermediate_size": 176,
        "experts_per_token": 4,
        "gate_multiplier": 2.0,
        "head_dim": 64,
        "hidden_size": 512,
        "intermediate_size": 1792,
        "latent_size": 256,
        "layer_types": [
          "full_attention",
          "linear_attention",
          "linear_attention",
          "linear_attention",
          "full_attention",
          "linear_attention",
          "linear_attention",
          "linear_attention",
          "full_attention",
          "linear_attention",
          "linear_attention",
          "linear_attention",
          "full_attention",
          "linear_attention",
          "linear_attention",
          "linear_attention",
          "full_attention",
          "full_attention"
        ],
        "lora_alpha": 16,
        "lora_rank": 16,
        "max_position_embeddings": 2048,
        "mlp_scalar_gate": true,
        "model_type": "q_rrt_delta_canon_latent_moe",
        "moe_budget": "total",
        "nope_every_n": null,
        "nope_layers": [],
        "num_attention_heads": 8,
        "num_core_layers": 8,
        "num_experts": 16,
        "num_hidden_layers": 18,
        "num_key_value_heads": 2,
        "num_passes": 2,
        "pad_token_id": 0,
        "qk_norm": true,
        "relaxed": true,
        "rms_norm_eps": 1e-05,
        "rope_theta": 10000.0,
        "routed_scaling_factor": 1.0,
        "router_bias_speed": 0.001,
        "router_seq_aux_coef": 0.0001,
        "shared_intermediate_size": 256,
        "swiglu_limit": 10.0,
        "tie_word_embeddings": true,
        "transformers_version": "5.17.0",
        "vocab_size": 32768
      },
      "evaluation": {
        "harness": "0.4.12",
        "seed": 1234,
        "dtype": "BF16",
        "batch_size": 8,
        "max_length": 2048,
        "torch_tf32": false,
        "examples": 112048,
        "recorded_tb_points": 1244
      },
      "tasks": [
        {
          "id": "hellaswag",
          "label": "HellaSwag",
          "metric": "acc_norm",
          "value": 0.29665405297749453,
          "stderr": 0.00455849155067332,
          "ci95": [
            0.2877991465091661,
            0.3056644755536661
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 10042,
          "dataset": "Rowan/hellaswag",
          "config": "default",
          "split": "validation",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-canon/attempt-001/hellaswag.json",
          "source_sha256": "fb53037d097abb42f6b72fa5e4e19348e57ce2d7e4b94defd2ec8973b404fa67"
        },
        {
          "id": "arc_easy",
          "label": "ARC-Easy",
          "metric": "acc_norm",
          "value": 0.43392255892255893,
          "stderr": 0.010169795770462075,
          "ci95": [
            0.4141167849838814,
            0.45394165275944653
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 2376,
          "dataset": "allenai/ai2_arc",
          "config": "ARC-Easy",
          "split": "test",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-canon/attempt-001/arc_easy.json",
          "source_sha256": "dc6440253c7dc20713ba88a063a8e4c58461fddca1f23cebd32aae34e0e26f84"
        },
        {
          "id": "arc_challenge",
          "label": "ARC-Challenge",
          "metric": "acc_norm",
          "value": 0.24658703071672355,
          "stderr": 0.012595726268789992,
          "ci95": [
            0.2227647274615329,
            0.2720651277506229
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1172,
          "dataset": "allenai/ai2_arc",
          "config": "ARC-Challenge",
          "split": "test",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-canon/attempt-001/arc_challenge.json",
          "source_sha256": "fd8ebf7ec9815cdd1ed7ea70654abf052837cc868c2ca75a28f880ddbd85e337"
        },
        {
          "id": "piqa",
          "label": "PIQA",
          "metric": "acc_norm",
          "value": 0.6115342763873776,
          "stderr": 0.011371877593210428,
          "ci95": [
            0.5890412805423275,
            0.6335620266188204
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1838,
          "dataset": "baber/piqa",
          "config": "default",
          "split": "validation",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-canon/attempt-001/piqa.json",
          "source_sha256": "dfbaffba2a0336e94d26bbf0c6df445b77c61dd58c4ee2e157109588ef238bed"
        },
        {
          "id": "winogrande",
          "label": "WinoGrande",
          "metric": "acc",
          "value": 0.5043409629044988,
          "stderr": 0.014051956064076884,
          "ci95": [
            0.47683902169584386,
            0.5318166606661463
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1267,
          "dataset": "allenai/winogrande",
          "config": "winogrande_xl",
          "split": "validation",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-canon/attempt-001/winogrande.json",
          "source_sha256": "51fd332e47dadf5245adf34b0ab4e16b6d6bda854448ccf8df435e069b0d78de"
        },
        {
          "id": "openbookqa",
          "label": "OpenBookQA",
          "metric": "acc_norm",
          "value": 0.284,
          "stderr": 0.020186703693570774,
          "ci95": [
            0.2462377719556586,
            0.32505594312373204
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 500,
          "dataset": "allenai/openbookqa",
          "config": "main",
          "split": "test",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-canon/attempt-001/openbookqa.json",
          "source_sha256": "aea1a60af092c5e179fc092dc0296617438bb4f880a2a32272a2623b13c2c927"
        },
        {
          "id": "boolq",
          "label": "BoolQ",
          "metric": "acc",
          "value": 0.5602446483180428,
          "stderr": 0.008681343983423894,
          "ci95": [
            0.5431712758823889,
            0.5771766410695599
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 3270,
          "dataset": "aps/super_glue",
          "config": "boolq",
          "split": "validation",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-canon/attempt-001/boolq.json",
          "source_sha256": "cd9d77fabe3c01c35f78419741b544674b4751e4f377cde68b4a1d2321e02975"
        },
        {
          "id": "lambada_openai",
          "label": "LAMBADA OpenAI",
          "metric": "acc",
          "value": 0.23365030079565302,
          "stderr": 0.0058953384201480205,
          "ci95": [
            0.22229778139124626,
            0.2453996411648182
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 5153,
          "dataset": "EleutherAI/lambada_openai",
          "config": "default",
          "split": "test",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-canon/attempt-001/lambada_openai.json",
          "source_sha256": "77e5c672b1803795f241f933734bfb5753d3bba1c3d831b10a5bea1ee0156207"
        },
        {
          "id": "arithmark3",
          "label": "ArithMark-3",
          "metric": "acc_norm",
          "value": 0.361,
          "stderr": 0.015195720118175049,
          "ci95": [
            0.3318160018268193,
            0.391247837032165
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1000,
          "dataset": "AxiomicLabs/Arithmark-3.0",
          "config": "default",
          "split": "train",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-canon/attempt-001/arithmark3.json",
          "source_sha256": "63fe8ebe1348d14839914fbfca2fe2de89526b03cf422d3b1b952047fefa087a"
        },
        {
          "id": "balanced_copa",
          "label": "Balanced COPA",
          "metric": "acc",
          "value": 0.551,
          "stderr": 0.015736792768752044,
          "ci95": [
            0.5200351373690727,
            0.5815745332654009
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1000,
          "dataset": "pkavumba/balanced-copa",
          "config": "default",
          "split": "train",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-canon/attempt-001/balanced_copa.json",
          "source_sha256": "8b321365f988fbe9663d114ab26345916feb85c4c007dde5344118033c31708a"
        },
        {
          "id": "blimp",
          "label": "BLiMP",
          "metric": "acc",
          "value": 0.7792985074626866,
          "stderr": 0.0014132842996867148,
          "samples": 67000,
          "dataset": "nyu-mll/blimp",
          "config": "67 minimal-pair subsets",
          "split": "train",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-canon/attempt-001/blimp.json",
          "source_sha256": "f4f5a562b094d0af67eee76bf28512e5c17f23d62f03c8be80e90d0a40dd0f8c"
        },
        {
          "id": "commonsense_qa",
          "label": "CommonsenseQA",
          "metric": "acc",
          "value": 0.19737919737919737,
          "stderr": 0.011395305685091258,
          "ci95": [
            0.17601790285026156,
            0.22063870549844142
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1221,
          "dataset": "tau/commonsense_qa",
          "config": "default",
          "split": "validation",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-canon/attempt-001/commonsense_qa.json",
          "source_sha256": "686bc2dce0bb4a302239f41fbe6c73bc06e18ba142a18648aeecf13c95a39b78"
        },
        {
          "id": "mmlu_continuation",
          "label": "MMLU continuation",
          "metric": "acc",
          "value": 0.2570146702748896,
          "stderr": 0.0036731589814631396,
          "samples": 14042,
          "dataset": "cais/mmlu",
          "config": "57 subjects",
          "split": "test",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-canon/attempt-001/mmlu_continuation.json",
          "source_sha256": "fb40516f4b5ec8873740f3acb784f38061a728c344ef07569c3cd6068c506b22"
        },
        {
          "id": "sciq",
          "label": "SciQ (with support)",
          "metric": "acc_norm",
          "value": 0.64,
          "stderr": 0.015186527932039985,
          "ci95": [
            0.609766237395941,
            0.6691622702281036
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1000,
          "dataset": "allenai/sciq",
          "config": "default",
          "split": "test",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-canon/attempt-001/sciq.json",
          "source_sha256": "9e827a2b46126578d279f74873c1e443374289295b37a32ea1b2329a1cdee8e8"
        },
        {
          "id": "truthfulqa_mc2",
          "label": "TruthfulQA MC2",
          "metric": "acc",
          "value": 0.42912006228880795,
          "stderr": 0.015089364876979552,
          "samples": 817,
          "dataset": "truthfulqa/truthful_qa",
          "config": "multiple_choice",
          "split": "validation",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-canon/attempt-001/truthfulqa_mc2.json",
          "source_sha256": "2344db33f3bc47169088fa85464488c6b739bc61c1da92bd7c910b0621f98ac4"
        },
        {
          "id": "bananamind_base",
          "label": "BananaMind Base 1.1",
          "metric": "raw_accuracy",
          "value": 0.4514285714285714,
          "stderr": 0.026637802830142845,
          "ci95": [
            0.40010247646563957,
            0.5038092920610971
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 350,
          "dataset": "BananaMind/BananaMind-Base-Bench-1.1",
          "config": "default",
          "split": "test",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-canon/attempt-001/bananamind_base.json",
          "source_sha256": "b0f1e8ad593b47aa94f7f52adecab34a86120085aa979f3a83713012ab233b57"
        }
      ],
      "continuation": {
        "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-canon/continuation.json",
        "source_sha256": "ef762b12be5daf930f54467c70161953047918c517f47f59c950401a8fa19f92",
        "blocks": 291,
        "scored_tokens": 148992,
        "corpus_sha256": "aca2f46735043bcfd0a44eca981d04627b9cdf74c4c9a04bf0856d04066f58fc",
        "tokenizer_sha256": "fc71a154536f6c84a5bf729d06324032126603ef5183c0b4979407e4fad68a8d",
        "bf16": {
          "nll": 3.4365377852187535,
          "nll_ci95": [
            3.397641767054489,
            3.475043039305513
          ],
          "ppl": 31.07916892318772,
          "ppl_ci95": [
            29.8935209731408,
            32.299218642634855
          ]
        },
        "fp32": {
          "nll": 3.4365859187345733,
          "nll_ci95": [
            3.3977137755897036,
            3.4750634951075328
          ],
          "ppl": 31.080664908859987,
          "ppl_ci95": [
            29.89567363930272,
            32.29987935581449
          ]
        }
      }
    },
    {
      "id": "q-rrt-delta-coarse",
      "label": "Q-RRT-Delta coarse 53M",
      "published": false,
      "inference_only": false,
      "parameters": 53296624,
      "mean_16_percent": 42.99815683840643,
      "checkpoint_sha256": {
        "config.json": "ac782615e208b9b3fdf4f4d62e52dd77012abe86b476a567e8986ee74c3d58c4",
        "generation_config.json": "80da72cc13262a67c14c8b96f4f049f6e7890a027d808a02b8f0b0d48704c245",
        "model.safetensors": "1590dfffdf72356529eda5c1e2422c0be6b675c167c2e239c35a8509eb0fe7f2"
      },
      "model_config": {
        "architectures": [
          "QRRTDeltaCoarseLatentMoEForCausalLM"
        ],
        "attention_dropout": 0.0,
        "attention_scalar_gate": true,
        "bos_token_id": 1,
        "delta_backend": "auto",
        "delta_conv_kernel": 4,
        "dtype": "float32",
        "eos_token_id": 2,
        "expert_backend": "auto",
        "expert_intermediate_size": 352,
        "experts_per_token": 2,
        "gate_multiplier": 2.0,
        "head_dim": 64,
        "hidden_size": 512,
        "intermediate_size": 1792,
        "latent_size": 256,
        "layer_types": [
          "full_attention",
          "linear_attention",
          "linear_attention",
          "linear_attention",
          "full_attention",
          "linear_attention",
          "linear_attention",
          "linear_attention",
          "full_attention",
          "linear_attention",
          "linear_attention",
          "linear_attention",
          "full_attention",
          "linear_attention",
          "linear_attention",
          "linear_attention",
          "full_attention",
          "full_attention"
        ],
        "lora_alpha": 16,
        "lora_rank": 16,
        "max_position_embeddings": 2048,
        "mlp_scalar_gate": true,
        "model_type": "q_rrt_delta_coarse_latent_moe",
        "moe_budget": "total",
        "nope_every_n": null,
        "nope_layers": [],
        "num_attention_heads": 8,
        "num_core_layers": 8,
        "num_experts": 8,
        "num_hidden_layers": 18,
        "num_key_value_heads": 2,
        "num_passes": 2,
        "pad_token_id": 0,
        "qk_norm": true,
        "relaxed": true,
        "rms_norm_eps": 1e-05,
        "rope_theta": 10000.0,
        "routed_scaling_factor": 1.0,
        "router_bias_speed": 0.001,
        "router_seq_aux_coef": 0.0001,
        "shared_intermediate_size": 256,
        "swiglu_limit": 10.0,
        "tie_word_embeddings": true,
        "transformers_version": "5.17.0",
        "vocab_size": 32768
      },
      "evaluation": {
        "harness": "0.4.12",
        "seed": 1234,
        "dtype": "BF16",
        "batch_size": 8,
        "max_length": 2048,
        "torch_tf32": false,
        "examples": 112048,
        "recorded_tb_points": 1244
      },
      "tasks": [
        {
          "id": "hellaswag",
          "label": "HellaSwag",
          "metric": "acc_norm",
          "value": 0.30322644891455885,
          "stderr": 0.004587128273935255,
          "ci95": [
            0.29431293929106317,
            0.3122904481703258
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 10042,
          "dataset": "Rowan/hellaswag",
          "config": "default",
          "split": "validation",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-coarse/attempt-001/hellaswag.json",
          "source_sha256": "b655bf7a155bbc0fdafdcbdc79bb22396c49e27372c5e5c1099130b4f103a039"
        },
        {
          "id": "arc_easy",
          "label": "ARC-Easy",
          "metric": "acc_norm",
          "value": 0.4356060606060606,
          "stderr": 0.01017434173366523,
          "ci95": [
            0.41578868285159826,
            0.4556313233568678
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 2376,
          "dataset": "allenai/ai2_arc",
          "config": "ARC-Easy",
          "split": "test",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-coarse/attempt-001/arc_easy.json",
          "source_sha256": "87635683108158d4b36557c1431207a24d3877e2b8a335b122aa3f125e8504f7"
        },
        {
          "id": "arc_challenge",
          "label": "ARC-Challenge",
          "metric": "acc_norm",
          "value": 0.23293515358361774,
          "stderr": 0.012352507042617355,
          "ci95": [
            0.2096313271937155,
            0.25798397476387624
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1172,
          "dataset": "allenai/ai2_arc",
          "config": "ARC-Challenge",
          "split": "test",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-coarse/attempt-001/arc_challenge.json",
          "source_sha256": "878db29901133a0732b29dc83d88bd0660843824626863a41eff09e6d993cdb6"
        },
        {
          "id": "piqa",
          "label": "PIQA",
          "metric": "acc_norm",
          "value": 0.6109902067464635,
          "stderr": 0.0113747749744473,
          "ci95": [
            0.5884926864882883,
            0.6330247508818297
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1838,
          "dataset": "baber/piqa",
          "config": "default",
          "split": "validation",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-coarse/attempt-001/piqa.json",
          "source_sha256": "f4e9e265a2eae9f9bf76add35014bdadbaf2604fd339a5d2740df648aa93b584"
        },
        {
          "id": "winogrande",
          "label": "WinoGrande",
          "metric": "acc",
          "value": 0.49171270718232046,
          "stderr": 0.01405055532282417,
          "ci95": [
            0.4642516701445155,
            0.5192238453462308
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1267,
          "dataset": "allenai/winogrande",
          "config": "winogrande_xl",
          "split": "validation",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-coarse/attempt-001/winogrande.json",
          "source_sha256": "4ac988297a4b3d4b6fab620943634df613d2c6b1487378cdc87ce0464d90a7c3"
        },
        {
          "id": "openbookqa",
          "label": "OpenBookQA",
          "metric": "acc_norm",
          "value": 0.306,
          "stderr": 0.020629569998345414,
          "ci95": [
            0.26721346373113636,
            0.3477447803679459
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 500,
          "dataset": "allenai/openbookqa",
          "config": "main",
          "split": "test",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-coarse/attempt-001/openbookqa.json",
          "source_sha256": "11bdd192d08e929ca53e8946a390b77cab285c7b703d42d3d28f4a32a58026a2"
        },
        {
          "id": "boolq",
          "label": "BoolQ",
          "metric": "acc",
          "value": 0.5565749235474006,
          "stderr": 0.00868889366131818,
          "ci95": [
            0.5394910884102307,
            0.5735259909608886
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 3270,
          "dataset": "aps/super_glue",
          "config": "boolq",
          "split": "validation",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-coarse/attempt-001/boolq.json",
          "source_sha256": "8378e06b353797cb36fc86bc3d6b77ccd7b9a08a21fd07868fd8d622807caa31"
        },
        {
          "id": "lambada_openai",
          "label": "LAMBADA OpenAI",
          "metric": "acc",
          "value": 0.23112749854453715,
          "stderr": 0.005873068236013304,
          "ci95": [
            0.2198204476847019,
            0.24283512896478973
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 5153,
          "dataset": "EleutherAI/lambada_openai",
          "config": "default",
          "split": "test",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-coarse/attempt-001/lambada_openai.json",
          "source_sha256": "4b4fc76cb2d96afb49a8531e9f14913c7a53840863417fb2e665d30966a37d99"
        },
        {
          "id": "arithmark3",
          "label": "ArithMark-3",
          "metric": "acc_norm",
          "value": 0.37,
          "stderr": 0.015275252316519466,
          "ci95": [
            0.3406266753339855,
            0.4003682818722588
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1000,
          "dataset": "AxiomicLabs/Arithmark-3.0",
          "config": "default",
          "split": "train",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-coarse/attempt-001/arithmark3.json",
          "source_sha256": "0d84545a3602cd600ddaa700ca8e79217ec644c05fe4349932719ce772fd8395"
        },
        {
          "id": "balanced_copa",
          "label": "Balanced COPA",
          "metric": "acc",
          "value": 0.542,
          "stderr": 0.015763390640483554,
          "ci95": [
            0.5110177730722398,
            0.5726607792149736
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1000,
          "dataset": "pkavumba/balanced-copa",
          "config": "default",
          "split": "train",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-coarse/attempt-001/balanced_copa.json",
          "source_sha256": "5d1e61e4a84f81cba4f80a1608ebecda052521a7d732501d89072bbeac8e963e"
        },
        {
          "id": "blimp",
          "label": "BLiMP",
          "metric": "acc",
          "value": 0.7846567164179105,
          "stderr": 0.0014120869428611452,
          "samples": 67000,
          "dataset": "nyu-mll/blimp",
          "config": "67 minimal-pair subsets",
          "split": "train",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-coarse/attempt-001/blimp.json",
          "source_sha256": "798ba4b5398af1f0b0884f8a6055bad0198ab6c7e840b59d4f2f79042b047dac"
        },
        {
          "id": "commonsense_qa",
          "label": "CommonsenseQA",
          "metric": "acc",
          "value": 0.19656019656019655,
          "stderr": 0.011377439773963867,
          "ci95": [
            0.17523627664679478,
            0.21978746731313606
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1221,
          "dataset": "tau/commonsense_qa",
          "config": "default",
          "split": "validation",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-coarse/attempt-001/commonsense_qa.json",
          "source_sha256": "31d07bb7850b76ab83b61236685a20ad088bde891ae8afdfa5f9822e09411961"
        },
        {
          "id": "mmlu_continuation",
          "label": "MMLU continuation",
          "metric": "acc",
          "value": 0.25566158666856575,
          "stderr": 0.0036682773405850525,
          "samples": 14042,
          "dataset": "cais/mmlu",
          "config": "57 subjects",
          "split": "test",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-coarse/attempt-001/mmlu_continuation.json",
          "source_sha256": "824dc44964b43d598cfe6508dd8c5b1a2ac1fdc551ffefa1667060b5148aacd7"
        },
        {
          "id": "sciq",
          "label": "SciQ (with support)",
          "metric": "acc_norm",
          "value": 0.627,
          "stderr": 0.015300493622922927,
          "ci95": [
            0.5965940413088956,
            0.6564339620357734
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1000,
          "dataset": "allenai/sciq",
          "config": "default",
          "split": "test",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-coarse/attempt-001/sciq.json",
          "source_sha256": "9d7cb25a8e7e0ce51da79d0c5819b5312a0c86ba6160a387e36837206ea3052e"
        },
        {
          "id": "truthfulqa_mc2",
          "label": "TruthfulQA MC2",
          "metric": "acc",
          "value": 0.44422502394482555,
          "stderr": 0.0150716007524175,
          "samples": 817,
          "dataset": "truthfulqa/truthful_qa",
          "config": "multiple_choice",
          "split": "validation",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-coarse/attempt-001/truthfulqa_mc2.json",
          "source_sha256": "99f11bc5c2eb069227596a03abbc0b666fde9e0899677a0dc07541e49524e86e"
        },
        {
          "id": "bananamind_base",
          "label": "BananaMind Base 1.1",
          "metric": "raw_accuracy",
          "value": 0.49142857142857144,
          "stderr": 0.02676045336347026,
          "ci95": [
            0.4394320766047808,
            0.5436111766646433
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 350,
          "dataset": "BananaMind/BananaMind-Base-Bench-1.1",
          "config": "default",
          "split": "test",
          "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-coarse/attempt-001/bananamind_base.json",
          "source_sha256": "364f3d6569b766460e116b75488f9fdb24a6276723771fabc09cd3e45b01a215"
        }
      ],
      "continuation": {
        "source": "runs/eval/delta-leaderboard-20261011/q-rrt-delta-coarse/continuation.json",
        "source_sha256": "fdabf7742e7b913b29c86eb6bbcfda517f038be47f0ba5a9b556c8eff4958105",
        "blocks": 291,
        "scored_tokens": 148992,
        "corpus_sha256": "aca2f46735043bcfd0a44eca981d04627b9cdf74c4c9a04bf0856d04066f58fc",
        "tokenizer_sha256": "fc71a154536f6c84a5bf729d06324032126603ef5183c0b4979407e4fad68a8d",
        "bf16": {
          "nll": 3.469682036396564,
          "nll_ci95": [
            3.431143294925133,
            3.5077612225952017
          ],
          "ppl": 32.1265257546766,
          "ppl_ci95": [
            30.91196404611993,
            33.373468318718906
          ]
        },
        "fp32": {
          "nll": 3.469663361093842,
          "nll_ci95": [
            3.4310606733425377,
            3.5077276950439638
          ],
          "ppl": 32.12592578768503,
          "ppl_ci95": [
            30.909410156233985,
            33.37234940680716
          ]
        }
      }
    },
    {
      "id": "q-rrt-delta-coarse-beta2",
      "label": "Q-RRT-Delta coarse+beta2 53M",
      "published": false,
      "inference_only": false,
      "parameters": 53296624,
      "mean_16_percent": 42.917308041870875,
      "checkpoint_sha256": {
        "config.json": "e214cd53295af0fb8eb290f97523be7a42bbebd0780ef367ceeb45462c6afef5",
        "generation_config.json": "80da72cc13262a67c14c8b96f4f049f6e7890a027d808a02b8f0b0d48704c245",
        "model.safetensors": "a63d8319a557e210637c9c5aa062d5128ec42dec29bd87fdf3287251def7b2d5"
      },
      "model_config": {
        "architectures": [
          "QRRTDeltaCoarseBeta2LatentMoEForCausalLM"
        ],
        "attention_dropout": 0.0,
        "attention_scalar_gate": true,
        "bos_token_id": 1,
        "delta_backend": "auto",
        "delta_beta_multiplier": 2.0,
        "delta_conv_kernel": 4,
        "dtype": "float32",
        "eos_token_id": 2,
        "expert_backend": "auto",
        "expert_intermediate_size": 352,
        "experts_per_token": 2,
        "gate_multiplier": 2.0,
        "head_dim": 64,
        "hidden_size": 512,
        "intermediate_size": 1792,
        "latent_size": 256,
        "layer_types": [
          "full_attention",
          "linear_attention",
          "linear_attention",
          "linear_attention",
          "full_attention",
          "linear_attention",
          "linear_attention",
          "linear_attention",
          "full_attention",
          "linear_attention",
          "linear_attention",
          "linear_attention",
          "full_attention",
          "linear_attention",
          "linear_attention",
          "linear_attention",
          "full_attention",
          "full_attention"
        ],
        "lora_alpha": 16,
        "lora_rank": 16,
        "max_position_embeddings": 2048,
        "mlp_scalar_gate": true,
        "model_type": "q_rrt_delta_coarse_beta2_latent_moe",
        "moe_budget": "total",
        "nope_every_n": null,
        "nope_layers": [],
        "num_attention_heads": 8,
        "num_core_layers": 8,
        "num_experts": 8,
        "num_hidden_layers": 18,
        "num_key_value_heads": 2,
        "num_passes": 2,
        "pad_token_id": 0,
        "qk_norm": true,
        "relaxed": true,
        "rms_norm_eps": 1e-05,
        "rope_theta": 10000.0,
        "routed_scaling_factor": 1.0,
        "router_bias_speed": 0.001,
        "router_seq_aux_coef": 0.0001,
        "shared_intermediate_size": 256,
        "swiglu_limit": 10.0,
        "tie_word_embeddings": true,
        "transformers_version": "5.17.0",
        "vocab_size": 32768
      },
      "evaluation": {
        "harness": "0.4.12",
        "seed": 1234,
        "dtype": "BF16",
        "batch_size": 8,
        "max_length": 2048,
        "torch_tf32": false,
        "examples": 112048,
        "recorded_tb_points": 1244
      },
      "tasks": [
        {
          "id": "hellaswag",
          "label": "HellaSwag",
          "metric": "acc_norm",
          "value": 0.29864568810993825,
          "stderr": 0.004567287775700271,
          "ci95": [
            0.2897727911611859,
            0.307672577992118
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 10042,
          "dataset": "Rowan/hellaswag",
          "config": "default",
          "split": "validation",
          "source": "runs/eval/coarse-beta2-leaderboard-20261012/q-rrt-delta-coarse-beta2/attempt-001/hellaswag.json",
          "source_sha256": "9b25eb7fa2d8c9c32559fd90e57dd34c9090febdf2b232cd77bbc379eb5f75f6"
        },
        {
          "id": "arc_easy",
          "label": "ARC-Easy",
          "metric": "acc_norm",
          "value": 0.43686868686868685,
          "stderr": 0.0101776729281577,
          "ci95": [
            0.41704275926445944,
            0.4568984232928601
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 2376,
          "dataset": "allenai/ai2_arc",
          "config": "ARC-Easy",
          "split": "test",
          "source": "runs/eval/coarse-beta2-leaderboard-20261012/q-rrt-delta-coarse-beta2/attempt-001/arc_easy.json",
          "source_sha256": "5c7f3d2d1f28eced4034e4c631109ef2980410ea738934b1563f3a70e7790330"
        },
        {
          "id": "arc_challenge",
          "label": "ARC-Challenge",
          "metric": "acc_norm",
          "value": 0.2354948805460751,
          "stderr": 0.01239945185500478,
          "ci95": [
            0.21209122984248996,
            0.2606268008503326
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1172,
          "dataset": "allenai/ai2_arc",
          "config": "ARC-Challenge",
          "split": "test",
          "source": "runs/eval/coarse-beta2-leaderboard-20261012/q-rrt-delta-coarse-beta2/attempt-001/arc_challenge.json",
          "source_sha256": "f60153c0bec18f611e8ef214e351d7289922d2e6ed9934178953860a91c70a8b"
        },
        {
          "id": "piqa",
          "label": "PIQA",
          "metric": "acc_norm",
          "value": 0.6022850924918389,
          "stderr": 0.011419114133117261,
          "ci95": [
            0.5797191246219102,
            0.624424396091728
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1838,
          "dataset": "baber/piqa",
          "config": "default",
          "split": "validation",
          "source": "runs/eval/coarse-beta2-leaderboard-20261012/q-rrt-delta-coarse-beta2/attempt-001/piqa.json",
          "source_sha256": "35016bb2af8931fbd2422d019727c46b9840f1083d5c3bca9ab75181e8ba5aa7"
        },
        {
          "id": "winogrande",
          "label": "WinoGrande",
          "metric": "acc",
          "value": 0.5114443567482242,
          "stderr": 0.014048804199859391,
          "ci95": [
            0.48392709074992285,
            0.5388924354771418
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1267,
          "dataset": "allenai/winogrande",
          "config": "winogrande_xl",
          "split": "validation",
          "source": "runs/eval/coarse-beta2-leaderboard-20261012/q-rrt-delta-coarse-beta2/attempt-001/winogrande.json",
          "source_sha256": "f54380f85e34ea5a376e05b1387d446394db04804cd5e1abb6003afe4aacd34a"
        },
        {
          "id": "openbookqa",
          "label": "OpenBookQA",
          "metric": "acc_norm",
          "value": 0.284,
          "stderr": 0.020186703693570774,
          "ci95": [
            0.2462377719556586,
            0.32505594312373204
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 500,
          "dataset": "allenai/openbookqa",
          "config": "main",
          "split": "test",
          "source": "runs/eval/coarse-beta2-leaderboard-20261012/q-rrt-delta-coarse-beta2/attempt-001/openbookqa.json",
          "source_sha256": "f37bcd896a631ba026b4150120080a6b3c3a930c6a3a16c9400e6c7f21076a12"
        },
        {
          "id": "boolq",
          "label": "BoolQ",
          "metric": "acc",
          "value": 0.5678899082568807,
          "stderr": 0.008664067354619397,
          "ci95": [
            0.5508413615169776,
            0.5847791337283657
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 3270,
          "dataset": "aps/super_glue",
          "config": "boolq",
          "split": "validation",
          "source": "runs/eval/coarse-beta2-leaderboard-20261012/q-rrt-delta-coarse-beta2/attempt-001/boolq.json",
          "source_sha256": "71372bc16152c5bb5484e9a48bac6a136423e0568f1469c80bf6d402a7245d48"
        },
        {
          "id": "lambada_openai",
          "label": "LAMBADA OpenAI",
          "metric": "acc",
          "value": 0.22045410440520086,
          "stderr": 0.005775532232204848,
          "ci95": [
            0.20934590876574993,
            0.23197878135593375
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 5153,
          "dataset": "EleutherAI/lambada_openai",
          "config": "default",
          "split": "test",
          "source": "runs/eval/coarse-beta2-leaderboard-20261012/q-rrt-delta-coarse-beta2/attempt-001/lambada_openai.json",
          "source_sha256": "55f1c37fa4c4d130d5b6b6b39dc5987e8b847a34569aa4498478018c413cda1b"
        },
        {
          "id": "arithmark3",
          "label": "ArithMark-3",
          "metric": "acc_norm",
          "value": 0.351,
          "stderr": 0.015100563798316506,
          "ci95": [
            0.322039577987741,
            0.3811007960409543
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1000,
          "dataset": "AxiomicLabs/Arithmark-3.0",
          "config": "default",
          "split": "train",
          "source": "runs/eval/coarse-beta2-leaderboard-20261012/q-rrt-delta-coarse-beta2/attempt-001/arithmark3.json",
          "source_sha256": "17f7b48d68e5b13f179e16ee454c80c5e2c151e9fd35dbde78f3431fdb5fe030"
        },
        {
          "id": "balanced_copa",
          "label": "Balanced COPA",
          "metric": "acc",
          "value": 0.545,
          "stderr": 0.015755101498347232,
          "ci95": [
            0.5140224377348876,
            0.5756331540014126
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1000,
          "dataset": "pkavumba/balanced-copa",
          "config": "default",
          "split": "train",
          "source": "runs/eval/coarse-beta2-leaderboard-20261012/q-rrt-delta-coarse-beta2/attempt-001/balanced_copa.json",
          "source_sha256": "3c38467341ed7a51757be9f0a164fb507f98ac00c746c84dde8c2153c5a69440"
        },
        {
          "id": "blimp",
          "label": "BLiMP",
          "metric": "acc",
          "value": 0.7800149253731343,
          "stderr": 0.001391384036961351,
          "samples": 67000,
          "dataset": "nyu-mll/blimp",
          "config": "67 minimal-pair subsets",
          "split": "train",
          "source": "runs/eval/coarse-beta2-leaderboard-20261012/q-rrt-delta-coarse-beta2/attempt-001/blimp.json",
          "source_sha256": "8e840545166ed804b23af02d18b5998b3f03df1aa6bcd574a97164c9474f6812"
        },
        {
          "id": "commonsense_qa",
          "label": "CommonsenseQA",
          "metric": "acc",
          "value": 0.19737919737919737,
          "stderr": 0.011395305685091258,
          "ci95": [
            0.17601790285026156,
            0.22063870549844142
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1221,
          "dataset": "tau/commonsense_qa",
          "config": "default",
          "split": "validation",
          "source": "runs/eval/coarse-beta2-leaderboard-20261012/q-rrt-delta-coarse-beta2/attempt-001/commonsense_qa.json",
          "source_sha256": "46cc675cb6acd4727e2b934023e321459e2d78c4c90a15d92f582967446462ca"
        },
        {
          "id": "mmlu_continuation",
          "label": "MMLU continuation",
          "metric": "acc",
          "value": 0.25616009115510613,
          "stderr": 0.0036705076433819774,
          "samples": 14042,
          "dataset": "cais/mmlu",
          "config": "57 subjects",
          "split": "test",
          "source": "runs/eval/coarse-beta2-leaderboard-20261012/q-rrt-delta-coarse-beta2/attempt-001/mmlu_continuation.json",
          "source_sha256": "4c3134557711462d41c98f4edc25a3c98bc559dc2f828a271a201c2bc59fddcf"
        },
        {
          "id": "sciq",
          "label": "SciQ (with support)",
          "metric": "acc_norm",
          "value": 0.641,
          "stderr": 0.01517726422479853,
          "ci95": [
            0.6107804510612619,
            0.6701404030458117
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1000,
          "dataset": "allenai/sciq",
          "config": "default",
          "split": "test",
          "source": "runs/eval/coarse-beta2-leaderboard-20261012/q-rrt-delta-coarse-beta2/attempt-001/sciq.json",
          "source_sha256": "6c0143f165a041f7f0e932b342cfc7d1997222124343d9166b49013759fe3b4d"
        },
        {
          "id": "truthfulqa_mc2",
          "label": "TruthfulQA MC2",
          "metric": "acc",
          "value": 0.43627521250791507,
          "stderr": 0.015194560730509662,
          "samples": 817,
          "dataset": "truthfulqa/truthful_qa",
          "config": "multiple_choice",
          "split": "validation",
          "source": "runs/eval/coarse-beta2-leaderboard-20261012/q-rrt-delta-coarse-beta2/attempt-001/truthfulqa_mc2.json",
          "source_sha256": "f775239a45947fd6e92d5bca8d889f9a49d89e06affcdb4f2e41adaefb1e8e4d"
        },
        {
          "id": "bananamind_base",
          "label": "BananaMind Base 1.1",
          "metric": "raw_accuracy",
          "value": 0.5028571428571429,
          "stderr": 0.026763949405468775,
          "ci95": [
            0.4507298432323698,
            0.5549224056778221
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 350,
          "dataset": "BananaMind/BananaMind-Base-Bench-1.1",
          "config": "default",
          "split": "test",
          "source": "runs/eval/coarse-beta2-leaderboard-20261012/q-rrt-delta-coarse-beta2/attempt-001/bananamind_base.json",
          "source_sha256": "ff12be46d043974a89c6b7f50326b6cee7b886ed5ad5826eb80d23030cc8f549"
        }
      ],
      "continuation": {
        "source": "runs/eval/coarse-beta2-leaderboard-20261012/q-rrt-delta-coarse-beta2/continuation.json",
        "source_sha256": "d18089e3900d8320cac1cc09af2bd32522a9fa8a52fb85892ae61729b69ad9d8",
        "blocks": 291,
        "scored_tokens": 148992,
        "corpus_sha256": "aca2f46735043bcfd0a44eca981d04627b9cdf74c4c9a04bf0856d04066f58fc",
        "tokenizer_sha256": "fc71a154536f6c84a5bf729d06324032126603ef5183c0b4979407e4fad68a8d",
        "bf16": {
          "nll": 3.45535594818928,
          "nll_ci95": [
            3.4175091336478074,
            3.4927638639699143
          ],
          "ppl": 31.669559398583537,
          "ppl_ci95": [
            30.493365447749937,
            32.876688946274676
          ]
        },
        "fp32": {
          "nll": 3.4545493134108605,
          "nll_ci95": [
            3.416741665265814,
            3.4919637256266736
          ],
          "ppl": 31.644023930838177,
          "ppl_ci95": [
            30.469971732025055,
            32.850393568222984
          ]
        }
      }
    },
    {
      "id": "q-rrt-delta-coarse-one-pass",
      "label": "Q-RRT-Delta coarse · 1 pass",
      "published": false,
      "inference_only": true,
      "parameters": 53296624,
      "mean_16_percent": 38.78503712965899,
      "checkpoint_sha256": {
        "config.json": "ac782615e208b9b3fdf4f4d62e52dd77012abe86b476a567e8986ee74c3d58c4",
        "generation_config.json": "80da72cc13262a67c14c8b96f4f049f6e7890a027d808a02b8f0b0d48704c245",
        "model.safetensors": "1590dfffdf72356529eda5c1e2422c0be6b675c167c2e239c35a8509eb0fe7f2"
      },
      "model_config": {
        "architectures": [
          "QRRTDeltaCoarseLatentMoEForCausalLM"
        ],
        "attention_dropout": 0.0,
        "attention_scalar_gate": true,
        "bos_token_id": 1,
        "delta_backend": "auto",
        "delta_conv_kernel": 4,
        "dtype": "float32",
        "eos_token_id": 2,
        "expert_backend": "auto",
        "expert_intermediate_size": 352,
        "experts_per_token": 2,
        "gate_multiplier": 2.0,
        "head_dim": 64,
        "hidden_size": 512,
        "intermediate_size": 1792,
        "latent_size": 256,
        "layer_types": [
          "full_attention",
          "linear_attention",
          "linear_attention",
          "linear_attention",
          "full_attention",
          "linear_attention",
          "linear_attention",
          "linear_attention",
          "full_attention",
          "linear_attention",
          "linear_attention",
          "linear_attention",
          "full_attention",
          "linear_attention",
          "linear_attention",
          "linear_attention",
          "full_attention",
          "full_attention"
        ],
        "lora_alpha": 16,
        "lora_rank": 16,
        "max_position_embeddings": 2048,
        "mlp_scalar_gate": true,
        "model_type": "q_rrt_delta_coarse_latent_moe",
        "moe_budget": "total",
        "nope_every_n": null,
        "nope_layers": [],
        "num_attention_heads": 8,
        "num_core_layers": 8,
        "num_experts": 8,
        "num_hidden_layers": 18,
        "num_key_value_heads": 2,
        "num_passes": 2,
        "pad_token_id": 0,
        "qk_norm": true,
        "relaxed": true,
        "rms_norm_eps": 1e-05,
        "rope_theta": 10000.0,
        "routed_scaling_factor": 1.0,
        "router_bias_speed": 0.001,
        "router_seq_aux_coef": 0.0001,
        "shared_intermediate_size": 256,
        "swiglu_limit": 10.0,
        "tie_word_embeddings": true,
        "transformers_version": "5.17.0",
        "vocab_size": 32768
      },
      "evaluation": {
        "harness": "0.4.12",
        "seed": 1234,
        "dtype": "BF16",
        "batch_size": 8,
        "max_length": 2048,
        "torch_tf32": false,
        "examples": 112048,
        "recorded_tb_points": 1244
      },
      "tasks": [
        {
          "id": "hellaswag",
          "label": "HellaSwag",
          "metric": "acc_norm",
          "value": 0.28161720772754434,
          "stderr": 0.0044886843979798155,
          "ci95": [
            0.2729047798007004,
            0.2904966517290699
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 10042,
          "dataset": "Rowan/hellaswag",
          "config": "default",
          "split": "validation",
          "source": "runs/eval/coarse-one-pass-20261013/q-rrt-delta-coarse-one-pass/attempt-001/hellaswag.json",
          "source_sha256": "2eda6fbcf7811b37b7d150fe205a920d180e36045e4c4e519943d36eefbb2de9"
        },
        {
          "id": "arc_easy",
          "label": "ARC-Easy",
          "metric": "acc_norm",
          "value": 0.3421717171717172,
          "stderr": 0.009735236771958604,
          "ci95": [
            0.3233634807519649,
            0.3614894756413342
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 2376,
          "dataset": "allenai/ai2_arc",
          "config": "ARC-Easy",
          "split": "test",
          "source": "runs/eval/coarse-one-pass-20261013/q-rrt-delta-coarse-one-pass/attempt-001/arc_easy.json",
          "source_sha256": "8197873318a204c649da3d4a236d4a2ac16eb834bb50198772f2c87fef7f6458"
        },
        {
          "id": "arc_challenge",
          "label": "ARC-Challenge",
          "metric": "acc_norm",
          "value": 0.22781569965870307,
          "stderr": 0.012256708602326964,
          "ci95": [
            0.20471523586388393,
            0.25269460862324633
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1172,
          "dataset": "allenai/ai2_arc",
          "config": "ARC-Challenge",
          "split": "test",
          "source": "runs/eval/coarse-one-pass-20261013/q-rrt-delta-coarse-one-pass/attempt-001/arc_challenge.json",
          "source_sha256": "22ceba58fa7ebaf9bf543d066d1a48af1d698bd558ca7d91d5ed5a9d7f24e24e"
        },
        {
          "id": "piqa",
          "label": "PIQA",
          "metric": "acc_norm",
          "value": 0.5538628944504896,
          "stderr": 0.011597936590301249,
          "ci95": [
            0.5310486343716763,
            0.5764524749402928
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1838,
          "dataset": "baber/piqa",
          "config": "default",
          "split": "validation",
          "source": "runs/eval/coarse-one-pass-20261013/q-rrt-delta-coarse-one-pass/attempt-001/piqa.json",
          "source_sha256": "1072532d3f768db2b2603c817815627a0146cdba3ac04d9443a813e42fca150e"
        },
        {
          "id": "winogrande",
          "label": "WinoGrande",
          "metric": "acc",
          "value": 0.4956590370955012,
          "stderr": 0.014051956064076884,
          "ci95": [
            0.46818333933385375,
            0.5231609783041562
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1267,
          "dataset": "allenai/winogrande",
          "config": "winogrande_xl",
          "split": "validation",
          "source": "runs/eval/coarse-one-pass-20261013/q-rrt-delta-coarse-one-pass/attempt-001/winogrande.json",
          "source_sha256": "8dbd79d4469417cd204bc03d3a0c0b6e240c194c6d21b4e6ecefce1243ca976d"
        },
        {
          "id": "openbookqa",
          "label": "OpenBookQA",
          "metric": "acc_norm",
          "value": 0.272,
          "stderr": 0.019920483209566107,
          "ci95": [
            0.2348440922052204,
            0.31263260704524753
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 500,
          "dataset": "allenai/openbookqa",
          "config": "main",
          "split": "test",
          "source": "runs/eval/coarse-one-pass-20261013/q-rrt-delta-coarse-one-pass/attempt-001/openbookqa.json",
          "source_sha256": "96b1a77b281b1eb9efc442a1050e7e8bebed380e75ad1e5a5320ba598dc0c691"
        },
        {
          "id": "boolq",
          "label": "BoolQ",
          "metric": "acc",
          "value": 0.6076452599388379,
          "stderr": 0.00853998383816761,
          "ci95": [
            0.5907927944746808,
            0.6242451078963142
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 3270,
          "dataset": "aps/super_glue",
          "config": "boolq",
          "split": "validation",
          "source": "runs/eval/coarse-one-pass-20261013/q-rrt-delta-coarse-one-pass/attempt-001/boolq.json",
          "source_sha256": "1943a58a43e5c4e6727fc745079642851811e2edb1778d7e756d6a63722074e6"
        },
        {
          "id": "lambada_openai",
          "label": "LAMBADA OpenAI",
          "metric": "acc",
          "value": 0.04851542790607413,
          "stderr": 0.002993318165770289,
          "ci95": [
            0.04297807211258836,
            0.05472542776113404
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 5153,
          "dataset": "EleutherAI/lambada_openai",
          "config": "default",
          "split": "test",
          "source": "runs/eval/coarse-one-pass-20261013/q-rrt-delta-coarse-one-pass/attempt-001/lambada_openai.json",
          "source_sha256": "fef548a65de5d928879133cead9250dc72a120a53c9938398fc7fe6bd01658e3"
        },
        {
          "id": "arithmark3",
          "label": "ArithMark-3",
          "metric": "acc_norm",
          "value": 0.338,
          "stderr": 0.014965960710224385,
          "ci95": [
            0.3093514451469222,
            0.36788842460239773
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1000,
          "dataset": "AxiomicLabs/Arithmark-3.0",
          "config": "default",
          "split": "train",
          "source": "runs/eval/coarse-one-pass-20261013/q-rrt-delta-coarse-one-pass/attempt-001/arithmark3.json",
          "source_sha256": "ffa53af9e29f1c00bc3ecd02717dd91fc689e18db435bc40af2c9aaffe0f4978"
        },
        {
          "id": "balanced_copa",
          "label": "Balanced COPA",
          "metric": "acc",
          "value": 0.52,
          "stderr": 0.015806639423035177,
          "ci95": [
            0.4890177246575191,
            0.5508292050030589
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1000,
          "dataset": "pkavumba/balanced-copa",
          "config": "default",
          "split": "train",
          "source": "runs/eval/coarse-one-pass-20261013/q-rrt-delta-coarse-one-pass/attempt-001/balanced_copa.json",
          "source_sha256": "dbe3959921f06f39a0d226c11c16c88557bf20e69b5629440d7e95642b2c600a"
        },
        {
          "id": "blimp",
          "label": "BLiMP",
          "metric": "acc",
          "value": 0.701910447761194,
          "stderr": 0.0015912370236339558,
          "samples": 67000,
          "dataset": "nyu-mll/blimp",
          "config": "67 minimal-pair subsets",
          "split": "train",
          "source": "runs/eval/coarse-one-pass-20261013/q-rrt-delta-coarse-one-pass/attempt-001/blimp.json",
          "source_sha256": "39acfc42ecca91e6f17d03f5ddc41cf2a69f41c9ada1e7735cc8dc1ac64f2ff5"
        },
        {
          "id": "commonsense_qa",
          "label": "CommonsenseQA",
          "metric": "acc",
          "value": 0.19656019656019655,
          "stderr": 0.011377439773963867,
          "ci95": [
            0.17523627664679478,
            0.21978746731313606
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1221,
          "dataset": "tau/commonsense_qa",
          "config": "default",
          "split": "validation",
          "source": "runs/eval/coarse-one-pass-20261013/q-rrt-delta-coarse-one-pass/attempt-001/commonsense_qa.json",
          "source_sha256": "ceb7d0e65e0c3753d2848bf50500506e82ccb33d065e71e4ae75cb884ca324a1"
        },
        {
          "id": "mmlu_continuation",
          "label": "MMLU continuation",
          "metric": "acc",
          "value": 0.2419883207520296,
          "stderr": 0.0036071980560207976,
          "samples": 14042,
          "dataset": "cais/mmlu",
          "config": "57 subjects",
          "split": "test",
          "source": "runs/eval/coarse-one-pass-20261013/q-rrt-delta-coarse-one-pass/attempt-001/mmlu_continuation.json",
          "source_sha256": "fbd009eb8002f8657eb4c365800b6a8fa57c0f793fd209939cc6a749dc80b1cb"
        },
        {
          "id": "sciq",
          "label": "SciQ (with support)",
          "metric": "acc_norm",
          "value": 0.431,
          "stderr": 0.01566794448817344,
          "ci95": [
            0.40062844376747236,
            0.46189964890353424
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 1000,
          "dataset": "allenai/sciq",
          "config": "default",
          "split": "test",
          "source": "runs/eval/coarse-one-pass-20261013/q-rrt-delta-coarse-one-pass/attempt-001/sciq.json",
          "source_sha256": "ee109aa49114511e28171abbe4cbda3e2c1d0a55c65bec8290e8c95a91e2c3ad"
        },
        {
          "id": "truthfulqa_mc2",
          "label": "TruthfulQA MC2",
          "metric": "acc",
          "value": 0.48114544600886505,
          "stderr": 0.015988974637983544,
          "samples": 817,
          "dataset": "truthfulqa/truthful_qa",
          "config": "multiple_choice",
          "split": "validation",
          "source": "runs/eval/coarse-one-pass-20261013/q-rrt-delta-coarse-one-pass/attempt-001/truthfulqa_mc2.json",
          "source_sha256": "63bf4283d71210ae01dd960a64e19e745546e51aa94e31e227c6bb743598f935"
        },
        {
          "id": "bananamind_base",
          "label": "BananaMind Base 1.1",
          "metric": "raw_accuracy",
          "value": 0.4657142857142857,
          "stderr": 0.026701388618254156,
          "ci95": [
            0.4141106768619489,
            0.5180623362157476
          ],
          "ci_method": "Wilson 95%; item independence approximation",
          "samples": 350,
          "dataset": "BananaMind/BananaMind-Base-Bench-1.1",
          "config": "default",
          "split": "test",
          "source": "runs/eval/coarse-one-pass-20261013/q-rrt-delta-coarse-one-pass/attempt-001/bananamind_base.json",
          "source_sha256": "2e7aa99b97c64cbfb1adcca4c28cf5a75ae00ffe6ebaa1caeaf1cf8d9d8a1b2b"
        }
      ],
      "continuation": {
        "source": "runs/eval/coarse-one-pass-20261013/q-rrt-delta-coarse-one-pass/continuation.json",
        "source_sha256": "c45d155684672625f88aed613d080513242360b19035f399aa7f860311aaa915",
        "blocks": 291,
        "scored_tokens": 148992,
        "corpus_sha256": "aca2f46735043bcfd0a44eca981d04627b9cdf74c4c9a04bf0856d04066f58fc",
        "tokenizer_sha256": "fc71a154536f6c84a5bf729d06324032126603ef5183c0b4979407e4fad68a8d",
        "bf16": {
          "nll": 5.107050066551392,
          "nll_ci95": [
            5.066642320319959,
            5.1472322537317305
          ],
          "ppl": 165.18235848476326,
          "ppl_ci95": [
            158.64076720209462,
            171.95490354286812
          ]
        },
        "fp32": {
          "nll": 5.108858147847283,
          "nll_ci95": [
            5.068375137540483,
            5.149071032279955
          ],
          "ppl": 165.48129178396593,
          "ppl_ci95": [
            158.91590096479615,
            172.27138140794892
          ]
        }
      }
    }
  ],
  "contrasts": [
    {
      "id": "rrt",
      "baseline": "q-latent-moe-52M-A35M-base",
      "variant": "q-rrt-latent-moe",
      "mean_delta_pp": -0.031717390717922456,
      "ppl_delta_percent": -9.77058067795953
    },
    {
      "id": "delta",
      "baseline": "q-rrt-latent-moe",
      "variant": "q-rrt-delta",
      "mean_delta_pp": -0.17381969981886414,
      "ppl_delta_percent": -1.2911952330839083
    },
    {
      "id": "beta2",
      "baseline": "q-rrt-delta",
      "variant": "q-rrt-delta-beta2",
      "mean_delta_pp": 0.1023732579683383,
      "ppl_delta_percent": 2.539166391544745
    },
    {
      "id": "canon",
      "baseline": "q-rrt-delta",
      "variant": "q-rrt-delta-canon",
      "mean_delta_pp": -0.03423751261459529,
      "ppl_delta_percent": -2.1113228224320824
    },
    {
      "id": "coarse",
      "baseline": "q-rrt-delta",
      "variant": "q-rrt-delta-coarse",
      "mean_delta_pp": 0.23157657668869547,
      "ppl_delta_percent": 1.187490444445305
    },
    {
      "id": "coarse-beta2",
      "baseline": "q-rrt-delta-coarse",
      "variant": "q-rrt-delta-coarse-beta2",
      "mean_delta_pp": -0.08084879653555532,
      "ppl_delta_percent": -1.4223958095641365
    },
    {
      "id": "one-pass",
      "baseline": "q-rrt-delta-coarse",
      "variant": "q-rrt-delta-coarse-one-pass",
      "mean_delta_pp": -4.213119708747442,
      "ppl_delta_percent": 414.1619101490237
    }
  ],
  "interaction_point_pp": -0.18322205450389362,
  "one_pass_execution": {
    "trained_passes": 2,
    "inference_passes": 1,
    "block_applications_before": 18,
    "block_applications_after": 10,
    "delta_applications_before": 12,
    "delta_applications_after": 6,
    "gqa_applications_before": 6,
    "gqa_applications_after": 4,
    "stored_parameters": 53296624
  },
  "delta_runtime": {
    "torch": "2.11.0+cu128",
    "fla": "0.5.2",
    "triton": "3.6.0",
    "pytorch_tf32_flags": false,
    "fla_solve_tril_dot_precision": "constexpr[tf32]",
    "note": "PyTorch matmul/cuDNN TF32 flags are disabled by the evaluator; FLA retains its shipped Triton internal precision settings."
  },
  "limits": "Historical core manifests predate the explicit TF32 flag; new runs record TF32 disabled. Diffusion PLL and LAMBADA reconstruction form a separate diagnostic group. cRia is benchmark-adapted; external models have different tokenizers, parameter counts and training budgets. Means are descriptive, not evidence of statistically significant rank differences."
}