{
  "verified_at": "2026-10-06",
  "collection": "https://huggingface.co/collections/d0rj/tiny-llm-ablation-6aafca336122dd2c2868f923",
  "source_tokens_per_model": 3932160000,
  "optimizer_steps": 15000,
  "training_seed": 2026,
  "evaluation_harness": "lm-eval 0.4.12",
  "mean_definition": "Unweighted mean of exactly one primary metric per task, 16 tasks; exclude PPL, SE, CI and subset scores. Descriptive; one training seed.",
  "continuation_definition": "WikiText-2 raw test, 291 nonoverlapping blocks, 512 prefix + 512 scored suffix tokens; 148992 scored tokens; 335 tail tokens omitted. PPL is conditional token PPL, not rolling or word PPL. Diffusion uses BF16 single-mask PLL and is a separate diagnostic group.",
  "models": [
    {
      "repo": "d0rj/q-moe-98M-A51M-base",
      "revision": "61700a031a87850816e263b4006615e27c28967b",
      "label": "Q-MoE 98M / A51M",
      "group": "causal",
      "parameters": 98105088,
      "active_parameters": 50919168,
      "optimized_parameters": null,
      "mean_16_percent": 43.05414834507311,
      "tasks": [
        {
          "id": "hellaswag",
          "label": "HellaSwag",
          "metric": "acc_norm",
          "value": 0.3051185022903804,
          "stderr": 0.004595165551383606,
          "ci95": [
            0.2961885267487427,
            0.31419752044859445
          ],
          "samples": 10042,
          "dataset": "Rowan/hellaswag",
          "split": "validation"
        },
        {
          "id": "arc_easy",
          "label": "ARC-Easy",
          "metric": "acc_norm",
          "value": 0.4385521885521885,
          "stderr": 0.010182010275471127,
          "ci95": [
            0.41871506492003546,
            0.45858768610242223
          ],
          "samples": 2376,
          "dataset": "allenai/ai2_arc",
          "split": "test"
        },
        {
          "id": "arc_challenge",
          "label": "ARC-Challenge",
          "metric": "acc_norm",
          "value": 0.24658703071672355,
          "stderr": 0.012595726268789992,
          "ci95": [
            0.2227647274615329,
            0.2720651277506229
          ],
          "samples": 1172,
          "dataset": "allenai/ai2_arc",
          "split": "test"
        },
        {
          "id": "piqa",
          "label": "PIQA",
          "metric": "acc_norm",
          "value": 0.6104461371055495,
          "stderr": 0.011377657455130346,
          "ci95": [
            0.5879441215348398,
            0.6324874460442482
          ],
          "samples": 1838,
          "dataset": "baber/piqa",
          "split": "validation"
        },
        {
          "id": "winogrande",
          "label": "WinoGrande",
          "metric": "acc",
          "value": 0.5082872928176796,
          "stderr": 0.01405055532282417,
          "ci95": [
            0.48077615465376927,
            0.5357483298554845
          ],
          "samples": 1267,
          "dataset": "allenai/winogrande",
          "split": "validation"
        },
        {
          "id": "openbookqa",
          "label": "OpenBookQA",
          "metric": "acc_norm",
          "value": 0.31,
          "stderr": 0.020704041021724684,
          "ci95": [
            0.2710389112735686,
            0.3518583381018213
          ],
          "samples": 500,
          "dataset": "allenai/openbookqa",
          "split": "test"
        },
        {
          "id": "boolq",
          "label": "BoolQ",
          "metric": "acc",
          "value": 0.5685015290519878,
          "stderr": 0.00866259456902732,
          "ci95": [
            0.5514551457036189,
            0.5853871558051961
          ],
          "samples": 3270,
          "dataset": "aps/super_glue",
          "split": "validation"
        },
        {
          "id": "lambada_openai",
          "label": "LAMBADA OpenAI",
          "metric": "acc",
          "value": 0.22064816611682514,
          "stderr": 0.005777354473289337,
          "ci95": [
            0.20953625932872036,
            0.23217626509346886
          ],
          "samples": 5153,
          "dataset": "EleutherAI/lambada_openai",
          "split": "test"
        },
        {
          "id": "arithmark3",
          "label": "ArithMark-3",
          "metric": "acc_norm",
          "value": 0.369,
          "stderr": 0.015266698139154782,
          "ci95": [
            0.33964716115877136,
            0.399355449564444
          ],
          "samples": 1000,
          "dataset": "AxiomicLabs/Arithmark-3.0",
          "split": "train"
        },
        {
          "id": "balanced_copa",
          "label": "Balanced COPA",
          "metric": "acc",
          "value": 0.543,
          "stderr": 0.0157606915901365,
          "ci95": [
            0.5120192033150067,
            0.5736516954552356
          ],
          "samples": 1000,
          "dataset": "pkavumba/balanced-copa",
          "split": "train"
        },
        {
          "id": "blimp",
          "label": "BLiMP",
          "metric": "acc",
          "value": 0.7723582089552239,
          "stderr": 0.0013958302883342111,
          "ci95": null,
          "samples": 67000,
          "dataset": "nyu-mll/blimp",
          "split": "train"
        },
        {
          "id": "commonsense_qa",
          "label": "CommonsenseQA",
          "metric": "acc",
          "value": 0.19983619983619982,
          "stderr": 0.01144844799672829,
          "ci95": [
            0.17836366711496662,
            0.2231915344000527
          ],
          "samples": 1221,
          "dataset": "tau/commonsense_qa",
          "split": "validation"
        },
        {
          "id": "mmlu_continuation",
          "label": "MMLU continuation",
          "metric": "acc",
          "value": 0.2583677538812135,
          "stderr": 0.003680818968618443,
          "ci95": null,
          "samples": 14042,
          "dataset": "cais/mmlu",
          "split": "test"
        },
        {
          "id": "sciq",
          "label": "SciQ (with support)",
          "metric": "acc_norm",
          "value": 0.642,
          "stderr": 0.015167928865407633,
          "ci95": [
            0.6117948042178514,
            0.6711183963722509
          ],
          "samples": 1000,
          "dataset": "allenai/sciq",
          "split": "test"
        },
        {
          "id": "truthfulqa_mc2",
          "label": "TruthfulQA MC2",
          "metric": "acc",
          "value": 0.4131035830305826,
          "stderr": 0.014947560271896813,
          "ci95": null,
          "samples": 817,
          "dataset": "truthfulqa/truthful_qa",
          "split": "validation"
        },
        {
          "id": "bananamind_base",
          "label": "BananaMind Base 1.1",
          "metric": "raw_accuracy",
          "value": 0.4828571428571429,
          "stderr": 0.026748650848283543,
          "ci95": [
            0.43097642743975617,
            0.5351100790990921
          ],
          "samples": 350,
          "dataset": "BananaMind/BananaMind-Base-Bench-1.1",
          "split": "test"
        }
      ],
      "continuation": {
        "nll": 3.438539972010347,
        "nll_ci95": [
          3.399929995389329,
          3.4766170177263085
        ],
        "ppl": 31.1414575606347,
        "ppl_ci95": [
          29.96200249565909,
          32.35009694595882
        ]
      },
      "continuation_precision": "torch.float32",
      "source_url": "https://huggingface.co/d0rj/q-moe-98M-A51M-base/blob/61700a031a87850816e263b4006615e27c28967b/evaluation/results.json",
      "source_sha256": "45af0c6edb0be62c26282213447dd2945f2481463721b97bce5ed6eac61e0526",
      "card_url": "https://huggingface.co/d0rj/q-moe-98M-A51M-base/blob/61700a031a87850816e263b4006615e27c28967b/README.md",
      "delta_pp": 0.22999196939912991
    },
    {
      "repo": "d0rj/q-latent-moe-52M-A35M-base",
      "revision": "dda6dd624d8217576bb315cdb50ad7d6748b392c",
      "label": "LatentMoE 52M / A35M",
      "group": "causal",
      "parameters": 51615488,
      "active_parameters": 35395328,
      "optimized_parameters": null,
      "mean_16_percent": 42.97211735225452,
      "tasks": [
        {
          "id": "hellaswag",
          "label": "HellaSwag",
          "metric": "acc_norm",
          "value": 0.29067914758016333,
          "stderr": 0.004531477407589249,
          "ci95": [
            0.28187943754229217,
            0.29963894324912477
          ],
          "samples": 10042,
          "dataset": "Rowan/hellaswag",
          "split": "validation"
        },
        {
          "id": "arc_easy",
          "label": "ARC-Easy",
          "metric": "acc_norm",
          "value": 0.42676767676767674,
          "stderr": 0.010149141043955583,
          "ci95": [
            0.40701382739079034,
            0.44675794437570043
          ],
          "samples": 2376,
          "dataset": "allenai/ai2_arc",
          "split": "test"
        },
        {
          "id": "arc_challenge",
          "label": "ARC-Challenge",
          "metric": "acc_norm",
          "value": 0.23378839590443687,
          "stderr": 0.01236822537850722,
          "ci95": [
            0.210451158412582,
            0.25886505312342006
          ],
          "samples": 1172,
          "dataset": "allenai/ai2_arc",
          "split": "test"
        },
        {
          "id": "piqa",
          "label": "PIQA",
          "metric": "acc_norm",
          "value": 0.6011969532100109,
          "stderr": 0.011424390545037256,
          "ci95": [
            0.5786229488593012,
            0.623348832272277
          ],
          "samples": 1838,
          "dataset": "baber/piqa",
          "split": "validation"
        },
        {
          "id": "winogrande",
          "label": "WinoGrande",
          "metric": "acc",
          "value": 0.5201262825572218,
          "stderr": 0.014041096664344348,
          "ci95": [
            0.49259780520052165,
            0.5475330857505233
          ],
          "samples": 1267,
          "dataset": "allenai/winogrande",
          "split": "validation"
        },
        {
          "id": "openbookqa",
          "label": "OpenBookQA",
          "metric": "acc_norm",
          "value": 0.314,
          "stderr": 0.020776701920308973,
          "ci95": [
            0.27486785583609774,
            0.35596839881559983
          ],
          "samples": 500,
          "dataset": "allenai/openbookqa",
          "split": "test"
        },
        {
          "id": "boolq",
          "label": "BoolQ",
          "metric": "acc",
          "value": 0.6140672782874618,
          "stderr": 0.008514444495863416,
          "ci95": [
            0.5972572363147873,
            0.630609631822659
          ],
          "samples": 3270,
          "dataset": "aps/super_glue",
          "split": "validation"
        },
        {
          "id": "lambada_openai",
          "label": "LAMBADA OpenAI",
          "metric": "acc",
          "value": 0.20842227828449447,
          "stderr": 0.005658885722145609,
          "ci95": [
            0.197551349622851,
            0.2197276138674855
          ],
          "samples": 5153,
          "dataset": "EleutherAI/lambada_openai",
          "split": "test"
        },
        {
          "id": "arithmark3",
          "label": "ArithMark-3",
          "metric": "acc_norm",
          "value": 0.38,
          "stderr": 0.015356947477797658,
          "ci95": [
            0.3504293063571445,
            0.4104891156793888
          ],
          "samples": 1000,
          "dataset": "AxiomicLabs/Arithmark-3.0",
          "split": "train"
        },
        {
          "id": "balanced_copa",
          "label": "Balanced COPA",
          "metric": "acc",
          "value": 0.533,
          "stderr": 0.015784807891138876,
          "ci95": [
            0.5020104989134991,
            0.5637369350264544
          ],
          "samples": 1000,
          "dataset": "pkavumba/balanced-copa",
          "split": "train"
        },
        {
          "id": "blimp",
          "label": "BLiMP",
          "metric": "acc",
          "value": 0.7622089552238805,
          "stderr": 0.001431525610004643,
          "ci95": null,
          "samples": 67000,
          "dataset": "nyu-mll/blimp",
          "split": "train"
        },
        {
          "id": "commonsense_qa",
          "label": "CommonsenseQA",
          "metric": "acc",
          "value": 0.19901719901719903,
          "stderr": 0.011430809442838318,
          "ci95": [
            0.17758159876867657,
            0.22234073835757068
          ],
          "samples": 1221,
          "dataset": "tau/commonsense_qa",
          "split": "validation"
        },
        {
          "id": "mmlu_continuation",
          "label": "MMLU continuation",
          "metric": "acc",
          "value": 0.2516735507762427,
          "stderr": 0.0036496473554643834,
          "ci95": null,
          "samples": 14042,
          "dataset": "cais/mmlu",
          "split": "test"
        },
        {
          "id": "sciq",
          "label": "SciQ (with support)",
          "metric": "acc_norm",
          "value": 0.66,
          "stderr": 0.014987482264363972,
          "ci95": [
            0.6300773195897643,
            0.6886981176948582
          ],
          "samples": 1000,
          "dataset": "allenai/sciq",
          "split": "test"
        },
        {
          "id": "truthfulqa_mc2",
          "label": "TruthfulQA MC2",
          "metric": "acc",
          "value": 0.42059105875193525,
          "stderr": 0.01524134620662943,
          "ci95": null,
          "samples": 817,
          "dataset": "truthfulqa/truthful_qa",
          "split": "validation"
        },
        {
          "id": "bananamind_base",
          "label": "BananaMind Base 1.1",
          "metric": "raw_accuracy",
          "value": 0.46,
          "stderr": 0.02667860286827002,
          "ci95": [
            0.40850229778075775,
            0.5123662174765549
          ],
          "samples": 350,
          "dataset": "BananaMind/BananaMind-Base-Bench-1.1",
          "split": "test"
        }
      ],
      "continuation": {
        "nll": 3.5736877770768,
        "nll_ci95": [
          3.5348262685475893,
          3.6124454839532727
        ],
        "ppl": 35.647812234350084,
        "ppl_ci95": [
          34.28905710903879,
          37.05656332969549
        ]
      },
      "continuation_precision": "torch.float32",
      "source_url": "https://huggingface.co/d0rj/q-latent-moe-52M-A35M-base/blob/dda6dd624d8217576bb315cdb50ad7d6748b392c/evaluation/results.json",
      "source_sha256": "a9e0a09bf836e4a4fff460041f938a6e5b67b1c6f12013d07c3b737b34086019",
      "card_url": "https://huggingface.co/d0rj/q-latent-moe-52M-A35M-base/blob/dda6dd624d8217576bb315cdb50ad7d6748b392c/README.md",
      "delta_pp": 0.14796097658054208
    },
    {
      "repo": "d0rj/q-51M-base",
      "revision": "09ba545cfd264e16961ef8df91b3d81a7c16a8a3",
      "label": "Q dense",
      "group": "causal",
      "parameters": 50878208,
      "active_parameters": null,
      "optimized_parameters": null,
      "mean_16_percent": 42.82415637567398,
      "tasks": [
        {
          "id": "hellaswag",
          "label": "hellaswag",
          "metric": "acc_norm",
          "value": 0.29177454690300736,
          "stderr": 0.004536500714147901,
          "ci95": [
            0.28296457902614003,
            0.3007437626650362
          ],
          "samples": 10042,
          "dataset": "Rowan/hellaswag",
          "split": "validation"
        },
        {
          "id": "arc_easy",
          "label": "arc_easy",
          "metric": "acc_norm",
          "value": 0.43308080808080807,
          "stderr": 0.010167478013701743,
          "ci95": [
            0.41328092357316454,
            0.45309672993759426
          ],
          "samples": 2376,
          "dataset": "allenai/ai2_arc",
          "split": "test"
        },
        {
          "id": "arc_challenge",
          "label": "arc_challenge",
          "metric": "acc_norm",
          "value": 0.24232081911262798,
          "stderr": 0.012521593295800071,
          "ci95": [
            0.21865689491479706,
            0.2676684124053075
          ],
          "samples": 1172,
          "dataset": "allenai/ai2_arc",
          "split": "test"
        },
        {
          "id": "piqa",
          "label": "piqa",
          "metric": "acc_norm",
          "value": 0.5990206746463548,
          "stderr": 0.011434766962108364,
          "ci95": [
            0.5764309418644076,
            0.6211973601030507
          ],
          "samples": 1838,
          "dataset": "baber/piqa",
          "split": "validation"
        },
        {
          "id": "winogrande",
          "label": "winogrande",
          "metric": "acc",
          "value": 0.500394632991318,
          "stderr": 0.01405248130604961,
          "ci95": [
            0.47290359623559514,
            0.5278832839791312
          ],
          "samples": 1267,
          "dataset": "allenai/winogrande",
          "split": "validation"
        },
        {
          "id": "openbookqa",
          "label": "openbookqa",
          "metric": "acc_norm",
          "value": 0.282,
          "stderr": 0.020143572847290726,
          "ci95": [
            0.2443364332272433,
            0.32298777921399346
          ],
          "samples": 500,
          "dataset": "allenai/openbookqa",
          "split": "test"
        },
        {
          "id": "boolq",
          "label": "boolq",
          "metric": "acc",
          "value": 0.598776758409786,
          "stderr": 0.008572708337179049,
          "ci95": [
            0.5818706845463824,
            0.615451027004275
          ],
          "samples": 3270,
          "dataset": "aps/super_glue",
          "split": "validation"
        },
        {
          "id": "lambada_openai",
          "label": "lambada_openai",
          "metric": "acc",
          "value": 0.20861633999611875,
          "stderr": 0.005660825573438258,
          "ci95": [
            0.1977414700784266,
            0.2199253277124155
          ],
          "samples": 5153,
          "dataset": "EleutherAI/lambada_openai",
          "split": "test"
        },
        {
          "id": "arithmark3",
          "label": "ArithMark-3",
          "metric": "acc_norm",
          "value": 0.367,
          "stderr": 0.015249378464171697,
          "ci95": [
            0.33768854423226846,
            0.3973293735248892
          ],
          "samples": 1000,
          "dataset": "AxiomicLabs/Arithmark-3.0",
          "split": "train"
        },
        {
          "id": "balanced_copa",
          "label": "Balanced COPA",
          "metric": "acc",
          "value": 0.546,
          "stderr": 0.01575221038877186,
          "ci95": [
            0.5150242420441904,
            0.5766236961751385
          ],
          "samples": 1000,
          "dataset": "pkavumba/balanced-copa",
          "split": "train"
        },
        {
          "id": "blimp",
          "label": "BLiMP",
          "metric": "acc",
          "value": 0.7726567164179104,
          "stderr": 0.0014498672475464,
          "ci95": null,
          "samples": 67000,
          "dataset": "nyu-mll/blimp",
          "split": "train"
        },
        {
          "id": "commonsense_qa",
          "label": "CommonsenseQA",
          "metric": "acc",
          "value": 0.19656019656019655,
          "stderr": 0.011377439773963867,
          "ci95": [
            0.17523627664679478,
            0.21978746731313606
          ],
          "samples": 1221,
          "dataset": "tau/commonsense_qa",
          "split": "validation"
        },
        {
          "id": "mmlu_continuation",
          "label": "MMLU continuation",
          "metric": "acc",
          "value": 0.2543797179888905,
          "stderr": 0.003665805402517398,
          "ci95": null,
          "samples": 14042,
          "dataset": "cais/mmlu",
          "split": "test"
        },
        {
          "id": "sciq",
          "label": "SciQ (with support)",
          "metric": "acc_norm",
          "value": 0.642,
          "stderr": 0.015167928865407633,
          "ci95": [
            0.6117948042178514,
            0.6711183963722509
          ],
          "samples": 1000,
          "dataset": "allenai/sciq",
          "split": "test"
        },
        {
          "id": "truthfulqa_mc2",
          "label": "TruthfulQA MC2",
          "metric": "acc",
          "value": 0.4372838090008176,
          "stderr": 0.01513251695187872,
          "ci95": null,
          "samples": 817,
          "dataset": "truthfulqa/truthful_qa",
          "split": "validation"
        },
        {
          "id": "bananamind_base",
          "label": "BananaMind Base 1.1",
          "metric": "raw_accuracy",
          "value": 0.48,
          "stderr": 0.026742966298044388,
          "ci95": [
            0.4281612478042309,
            0.5322730098244254
          ],
          "samples": 350,
          "dataset": "BananaMind/BananaMind-Base-Bench-1.1",
          "split": "test"
        }
      ],
      "continuation": {
        "nll": 3.560861705095088,
        "nll_ci95": [
          3.521807046434314,
          3.599506885284411
        ],
        "ppl": 35.19351050996088,
        "ppl_ci95": [
          33.84553368820155,
          36.580191764630115
        ]
      },
      "continuation_precision": "CPU FP32",
      "source_url": "https://huggingface.co/d0rj/q-51M-base/blob/09ba545cfd264e16961ef8df91b3d81a7c16a8a3/evaluation/results.json",
      "source_sha256": "f2d0ce9537df546a2d31578fbe9e537ac4fd22ae520aa54037f5c989d0b9e814",
      "card_url": "https://huggingface.co/d0rj/q-51M-base/blob/09ba545cfd264e16961ef8df91b3d81a7c16a8a3/README.md",
      "delta_pp": 0.0
    },
    {
      "repo": "d0rj/q-latent-moe-93M-A52M-base",
      "revision": "60f7cfb412daa372eeea6586d97ebdfe8cfd3849",
      "label": "LatentMoE 93M / A52M",
      "group": "causal",
      "parameters": 92903168,
      "active_parameters": 51615488,
      "optimized_parameters": null,
      "mean_16_percent": 42.77249955714257,
      "tasks": [
        {
          "id": "hellaswag",
          "label": "HellaSwag",
          "metric": "acc_norm",
          "value": 0.30033857797251545,
          "stderr": 0.004574683373821224,
          "ci95": [
            0.2914505481341901,
            0.3093793060460149
          ],
          "samples": 10042,
          "dataset": "Rowan/hellaswag",
          "split": "validation"
        },
        {
          "id": "arc_easy",
          "label": "ARC-Easy",
          "metric": "acc_norm",
          "value": 0.4393939393939394,
          "stderr": 0.010184134315437384,
          "ci95": [
            0.41955130499488646,
            0.45943223026014035
          ],
          "samples": 2376,
          "dataset": "allenai/ai2_arc",
          "split": "test"
        },
        {
          "id": "arc_challenge",
          "label": "ARC-Challenge",
          "metric": "acc_norm",
          "value": 0.24061433447098976,
          "stderr": 0.012491468532390682,
          "ci95": [
            0.21701467963390295,
            0.26590880852938104
          ],
          "samples": 1172,
          "dataset": "allenai/ai2_arc",
          "split": "test"
        },
        {
          "id": "piqa",
          "label": "PIQA",
          "metric": "acc_norm",
          "value": 0.6088139281828074,
          "stderr": 0.011386215606728629,
          "ci95": [
            0.5862986010580833,
            0.6308753571479149
          ],
          "samples": 1838,
          "dataset": "baber/piqa",
          "split": "validation"
        },
        {
          "id": "winogrande",
          "label": "WinoGrande",
          "metric": "acc",
          "value": 0.49329123914759276,
          "stderr": 0.014051220692330247,
          "ci95": [
            0.46582413289507446,
            0.5207989034545772
          ],
          "samples": 1267,
          "dataset": "allenai/winogrande",
          "split": "validation"
        },
        {
          "id": "openbookqa",
          "label": "OpenBookQA",
          "metric": "acc_norm",
          "value": 0.31,
          "stderr": 0.020704041021724684,
          "ci95": [
            0.2710389112735686,
            0.3518583381018213
          ],
          "samples": 500,
          "dataset": "allenai/openbookqa",
          "split": "test"
        },
        {
          "id": "boolq",
          "label": "BoolQ",
          "metric": "acc",
          "value": 0.5051987767584097,
          "stderr": 0.008744582253526267,
          "ci95": [
            0.48806628683812947,
            0.5223190664013787
          ],
          "samples": 3270,
          "dataset": "aps/super_glue",
          "split": "validation"
        },
        {
          "id": "lambada_openai",
          "label": "LAMBADA OpenAI",
          "metric": "acc",
          "value": 0.22627595575392975,
          "stderr": 0.005829406265404571,
          "ci95": [
            0.21505797763701381,
            0.23790174149983787
          ],
          "samples": 5153,
          "dataset": "EleutherAI/lambada_openai",
          "split": "test"
        },
        {
          "id": "arithmark3",
          "label": "ArithMark-3",
          "metric": "acc_norm",
          "value": 0.34,
          "stderr": 0.014987482264363972,
          "ci95": [
            0.31130188230514194,
            0.36992268041023574
          ],
          "samples": 1000,
          "dataset": "AxiomicLabs/Arithmark-3.0",
          "split": "train"
        },
        {
          "id": "balanced_copa",
          "label": "Balanced COPA",
          "metric": "acc",
          "value": 0.56,
          "stderr": 0.015704987954361718,
          "ci95": [
            0.5290626428712739,
            0.5904781461104596
          ],
          "samples": 1000,
          "dataset": "pkavumba/balanced-copa",
          "split": "train"
        },
        {
          "id": "blimp",
          "label": "BLiMP",
          "metric": "acc",
          "value": 0.7584179104477612,
          "stderr": 0.0014536843096797146,
          "ci95": null,
          "samples": 67000,
          "dataset": "nyu-mll/blimp",
          "split": "train"
        },
        {
          "id": "commonsense_qa",
          "label": "CommonsenseQA",
          "metric": "acc",
          "value": 0.1981981981981982,
          "stderr": 0.011413095456219376,
          "ci95": [
            0.17679967712019065,
            0.22148979561728446
          ],
          "samples": 1221,
          "dataset": "tau/commonsense_qa",
          "split": "validation"
        },
        {
          "id": "mmlu_continuation",
          "label": "MMLU continuation",
          "metric": "acc",
          "value": 0.2608602763139154,
          "stderr": 0.0036933962400115122,
          "ci95": null,
          "samples": 14042,
          "dataset": "cais/mmlu",
          "split": "test"
        },
        {
          "id": "sciq",
          "label": "SciQ (with support)",
          "metric": "acc_norm",
          "value": 0.672,
          "stderr": 0.01485384248727031,
          "ci95": [
            0.6422916335335495,
            0.7003919615474198
          ],
          "samples": 1000,
          "dataset": "allenai/sciq",
          "split": "test"
        },
        {
          "id": "truthfulqa_mc2",
          "label": "TruthfulQA MC2",
          "metric": "acc",
          "value": 0.4301967925027518,
          "stderr": 0.014973807398908884,
          "ci95": null,
          "samples": 817,
          "dataset": "truthfulqa/truthful_qa",
          "split": "validation"
        },
        {
          "id": "bananamind_base",
          "label": "BananaMind Base 1.1",
          "metric": "raw_accuracy",
          "value": 0.5,
          "stderr": 0.02676438637860946,
          "ci95": [
            0.44790287743932533,
            0.5520971225606747
          ],
          "samples": 350,
          "dataset": "BananaMind/BananaMind-Base-Bench-1.1",
          "split": "test"
        }
      ],
      "continuation": {
        "nll": 3.4343477270447513,
        "nll_ci95": [
          3.39560829505478,
          3.4722306688626605
        ],
        "ppl": 31.011178214225882,
        "ppl_ci95": [
          29.832795098377364,
          32.20850888930017
        ]
      },
      "continuation_precision": "torch.float32",
      "source_url": "https://huggingface.co/d0rj/q-latent-moe-93M-A52M-base/blob/60f7cfb412daa372eeea6586d97ebdfe8cfd3849/evaluation/results.json",
      "source_sha256": "e5e417d4eb0de53fe3a2c9e736f159244d5b7cdf3065f90eb2cd6221f60dfb26",
      "card_url": "https://huggingface.co/d0rj/q-latent-moe-93M-A52M-base/blob/60f7cfb412daa372eeea6586d97ebdfe8cfd3849/README.md",
      "delta_pp": -0.051656818531412796
    },
    {
      "repo": "d0rj/looped-51M-base",
      "revision": "95e4d745ce7e8ca0368ef389512d09683f482477",
      "label": "Looped (6 × 10)",
      "group": "causal",
      "parameters": 50869760,
      "active_parameters": null,
      "optimized_parameters": null,
      "mean_16_percent": 42.74882181495543,
      "tasks": [
        {
          "id": "hellaswag",
          "label": "hellaswag",
          "metric": "acc_norm",
          "value": 0.2961561441943836,
          "stderr": 0.004556276293751663,
          "ci95": [
            0.2873057670755037,
            0.30516241821471046
          ],
          "samples": 10042,
          "dataset": "Rowan/hellaswag",
          "split": "validation"
        },
        {
          "id": "arc_easy",
          "label": "arc_easy",
          "metric": "acc_norm",
          "value": 0.44276094276094274,
          "stderr": 0.010192333348394433,
          "ci95": [
            0.4228968461002096,
            0.46280982608509347
          ],
          "samples": 2376,
          "dataset": "allenai/ai2_arc",
          "split": "test"
        },
        {
          "id": "arc_challenge",
          "label": "arc_challenge",
          "metric": "acc_norm",
          "value": 0.22098976109215018,
          "stderr": 0.012124929206818241,
          "ci95": [
            0.1981683230517509,
            0.2456342448080974
          ],
          "samples": 1172,
          "dataset": "allenai/ai2_arc",
          "split": "test"
        },
        {
          "id": "piqa",
          "label": "piqa",
          "metric": "acc_norm",
          "value": 0.602829162132753,
          "stderr": 0.011416453840790413,
          "ci95": [
            0.5802672556387278,
            0.6249621348659402
          ],
          "samples": 1838,
          "dataset": "baber/piqa",
          "split": "validation"
        },
        {
          "id": "winogrande",
          "label": "winogrande",
          "metric": "acc",
          "value": 0.5011838989739542,
          "stderr": 0.014052446290528871,
          "ci95": [
            0.47369054474157507,
            0.5286700959026042
          ],
          "samples": 1267,
          "dataset": "allenai/winogrande",
          "split": "validation"
        },
        {
          "id": "openbookqa",
          "label": "openbookqa",
          "metric": "acc_norm",
          "value": 0.29,
          "stderr": 0.020313179231745138,
          "ci95": [
            0.2519474198773656,
            0.33125480311648636
          ],
          "samples": 500,
          "dataset": "allenai/openbookqa",
          "split": "test"
        },
        {
          "id": "boolq",
          "label": "boolq",
          "metric": "acc",
          "value": 0.6159021406727829,
          "stderr": 0.008506861063860427,
          "ci95": [
            0.5991047800560695,
            0.6324275068717916
          ],
          "samples": 3270,
          "dataset": "aps/super_glue",
          "split": "validation"
        },
        {
          "id": "lambada_openai",
          "label": "lambada_openai",
          "metric": "acc",
          "value": 0.20900446341936735,
          "stderr": 0.0056646994119225694,
          "ci95": [
            0.19812172246246124,
            0.22032074392939205
          ],
          "samples": 5153,
          "dataset": "EleutherAI/lambada_openai",
          "split": "test"
        },
        {
          "id": "arithmark3",
          "label": "ArithMark-3",
          "metric": "acc_norm",
          "value": 0.357,
          "stderr": 0.01515852172148659,
          "ci95": [
            0.3279037500492223,
            0.3871907028776464
          ],
          "samples": 1000,
          "dataset": "AxiomicLabs/Arithmark-3.0",
          "split": "train"
        },
        {
          "id": "balanced_copa",
          "label": "Balanced COPA",
          "metric": "acc",
          "value": 0.548,
          "stderr": 0.01574623586588058,
          "ci95": [
            0.5170282252037207,
            0.5786044059816661
          ],
          "samples": 1000,
          "dataset": "pkavumba/balanced-copa",
          "split": "train"
        },
        {
          "id": "blimp",
          "label": "BLiMP",
          "metric": "acc",
          "value": 0.7584328358208955,
          "stderr": 0.0014455497248339602,
          "ci95": null,
          "samples": 67000,
          "dataset": "nyu-mll/blimp",
          "split": "train"
        },
        {
          "id": "commonsense_qa",
          "label": "CommonsenseQA",
          "metric": "acc",
          "value": 0.19656019656019655,
          "stderr": 0.011377439773963867,
          "ci95": [
            0.17523627664679478,
            0.21978746731313606
          ],
          "samples": 1221,
          "dataset": "tau/commonsense_qa",
          "split": "validation"
        },
        {
          "id": "mmlu_continuation",
          "label": "MMLU continuation",
          "metric": "acc",
          "value": 0.25672981056829514,
          "stderr": 0.0036704736315457996,
          "ci95": null,
          "samples": 14042,
          "dataset": "cais/mmlu",
          "split": "test"
        },
        {
          "id": "sciq",
          "label": "SciQ (with support)",
          "metric": "acc_norm",
          "value": 0.64,
          "stderr": 0.015186527932039985,
          "ci95": [
            0.609766237395941,
            0.6691622702281036
          ],
          "samples": 1000,
          "dataset": "allenai/sciq",
          "split": "test"
        },
        {
          "id": "truthfulqa_mc2",
          "label": "TruthfulQA MC2",
          "metric": "acc",
          "value": 0.4242621341971478,
          "stderr": 0.015001938042777255,
          "ci95": null,
          "samples": 817,
          "dataset": "truthfulqa/truthful_qa",
          "split": "validation"
        },
        {
          "id": "bananamind_base",
          "label": "BananaMind Base 1.1",
          "metric": "raw_accuracy",
          "value": 0.48,
          "stderr": 0.026742966298044388,
          "ci95": [
            0.4281612478042309,
            0.5322730098244254
          ],
          "samples": 350,
          "dataset": "BananaMind/BananaMind-Base-Bench-1.1",
          "split": "test"
        }
      ],
      "continuation": {
        "nll": 3.503892080890354,
        "nll_ci95": [
          3.465063270322236,
          3.5422630942154587
        ],
        "ppl": 33.244591123454065,
        "ppl_ci95": [
          31.97848299804872,
          34.54500940573081
        ]
      },
      "continuation_precision": "torch.float32",
      "source_url": "https://huggingface.co/d0rj/looped-51M-base/blob/95e4d745ce7e8ca0368ef389512d09683f482477/evaluation/results.json",
      "source_sha256": "f53fec00b0835fdfbb4e89c349317acaab67ca5e39092737935a1d84dfa4cbee",
      "card_url": "https://huggingface.co/d0rj/looped-51M-base/blob/95e4d745ce7e8ca0368ef389512d09683f482477/README.md",
      "delta_pp": -0.07533456071855227
    },
    {
      "repo": "d0rj/q-moe-51M-A33M-base",
      "revision": "86755ea315ab9c31ab95a6780419b0ec1ce16975",
      "label": "Q-MoE 51M / A33M",
      "group": "causal",
      "parameters": 50919168,
      "active_parameters": 33224448,
      "optimized_parameters": null,
      "mean_16_percent": 42.19222452916442,
      "tasks": [
        {
          "id": "hellaswag",
          "label": "HellaSwag",
          "metric": "acc_norm",
          "value": 0.2923720374427405,
          "stderr": 0.004539227260397284,
          "ci95": [
            0.2835565007016171,
            0.30134636511670077
          ],
          "samples": 10042,
          "dataset": "Rowan/hellaswag",
          "split": "validation"
        },
        {
          "id": "arc_easy",
          "label": "ARC-Easy",
          "metric": "acc_norm",
          "value": 0.4208754208754209,
          "stderr": 0.010130502164066243,
          "ci95": [
            0.40116751732413125,
            0.4408387648143761
          ],
          "samples": 2376,
          "dataset": "allenai/ai2_arc",
          "split": "test"
        },
        {
          "id": "arc_challenge",
          "label": "ARC-Challenge",
          "metric": "acc_norm",
          "value": 0.2363481228668942,
          "stderr": 0.012414960524301884,
          "ci95": [
            0.21291146902948319,
            0.2615074712417496
          ],
          "samples": 1172,
          "dataset": "allenai/ai2_arc",
          "split": "test"
        },
        {
          "id": "piqa",
          "label": "PIQA",
          "metric": "acc_norm",
          "value": 0.602829162132753,
          "stderr": 0.011416453840790413,
          "ci95": [
            0.5802672556387278,
            0.6249621348659402
          ],
          "samples": 1838,
          "dataset": "baber/piqa",
          "split": "validation"
        },
        {
          "id": "winogrande",
          "label": "WinoGrande",
          "metric": "acc",
          "value": 0.5138121546961326,
          "stderr": 0.014047122916440271,
          "ci95": [
            0.4862910104211901,
            0.5412497970942327
          ],
          "samples": 1267,
          "dataset": "allenai/winogrande",
          "split": "validation"
        },
        {
          "id": "openbookqa",
          "label": "OpenBookQA",
          "metric": "acc_norm",
          "value": 0.292,
          "stderr": 0.020354375480530096,
          "ci95": [
            0.2538524938356277,
            0.3333192317963781
          ],
          "samples": 500,
          "dataset": "allenai/openbookqa",
          "split": "test"
        },
        {
          "id": "boolq",
          "label": "BoolQ",
          "metric": "acc",
          "value": 0.5697247706422018,
          "stderr": 0.008659608602932242,
          "ci95": [
            0.5526827930954011,
            0.5866031209403569
          ],
          "samples": 3270,
          "dataset": "aps/super_glue",
          "split": "validation"
        },
        {
          "id": "lambada_openai",
          "label": "LAMBADA OpenAI",
          "metric": "acc",
          "value": 0.18901610712206482,
          "stderr": 0.005454658857815098,
          "ci95": [
            0.17855934076251695,
            0.1999361926772596
          ],
          "samples": 5153,
          "dataset": "EleutherAI/lambada_openai",
          "split": "test"
        },
        {
          "id": "arithmark3",
          "label": "ArithMark-3",
          "metric": "acc_norm",
          "value": 0.35,
          "stderr": 0.015090650341444125,
          "ci95": [
            0.32106271005247566,
            0.38008531749319086
          ],
          "samples": 1000,
          "dataset": "AxiomicLabs/Arithmark-3.0",
          "split": "train"
        },
        {
          "id": "balanced_copa",
          "label": "Balanced COPA",
          "metric": "acc",
          "value": 0.528,
          "stderr": 0.015794475789511517,
          "ci95": [
            0.4970108021953935,
            0.5587748993294154
          ],
          "samples": 1000,
          "dataset": "pkavumba/balanced-copa",
          "split": "train"
        },
        {
          "id": "blimp",
          "label": "BLiMP",
          "metric": "acc",
          "value": 0.7597611940298507,
          "stderr": 0.0014750717900401267,
          "ci95": null,
          "samples": 67000,
          "dataset": "nyu-mll/blimp",
          "split": "train"
        },
        {
          "id": "commonsense_qa",
          "label": "CommonsenseQA",
          "metric": "acc",
          "value": 0.19656019656019655,
          "stderr": 0.011377439773963867,
          "ci95": [
            0.17523627664679478,
            0.21978746731313606
          ],
          "samples": 1221,
          "dataset": "tau/commonsense_qa",
          "split": "validation"
        },
        {
          "id": "mmlu_continuation",
          "label": "MMLU continuation",
          "metric": "acc",
          "value": 0.25566158666856575,
          "stderr": 0.0036707677399799013,
          "ci95": null,
          "samples": 14042,
          "dataset": "cais/mmlu",
          "split": "test"
        },
        {
          "id": "sciq",
          "label": "SciQ (with support)",
          "metric": "acc_norm",
          "value": 0.626,
          "stderr": 0.015308767369006505,
          "ci95": [
            0.5955817549893309,
            0.6554539018723092
          ],
          "samples": 1000,
          "dataset": "allenai/sciq",
          "split": "test"
        },
        {
          "id": "truthfulqa_mc2",
          "label": "TruthfulQA MC2",
          "metric": "acc",
          "value": 0.43208088591520116,
          "stderr": 0.015302002887905923,
          "ci95": null,
          "samples": 817,
          "dataset": "truthfulqa/truthful_qa",
          "split": "validation"
        },
        {
          "id": "bananamind_base",
          "label": "BananaMind Base 1.1",
          "metric": "raw_accuracy",
          "value": 0.4857142857142857,
          "stderr": 0.026753459908924214,
          "ci95": [
            0.4337932926903378,
            0.5379454627587024
          ],
          "samples": 350,
          "dataset": "BananaMind/BananaMind-Base-Bench-1.1",
          "split": "test"
        }
      ],
      "continuation": {
        "nll": 3.571762175904107,
        "nll_ci95": [
          3.5330876772141537,
          3.6101139180643864
        ],
        "ppl": 35.57923481287936,
        "ppl_ci95": [
          34.22949424426097,
          36.970264155876535
        ]
      },
      "continuation_precision": "torch.float32",
      "source_url": "https://huggingface.co/d0rj/q-moe-51M-A33M-base/blob/86755ea315ab9c31ab95a6780419b0ec1ce16975/evaluation/results.json",
      "source_sha256": "2c936a3c88d879f52fe16e490a34b629bb558932c4cc6044efecc74db521bf27",
      "card_url": "https://huggingface.co/d0rj/q-moe-51M-A33M-base/blob/86755ea315ab9c31ab95a6780419b0ec1ce16975/README.md",
      "delta_pp": -0.6319318465095591
    },
    {
      "repo": "d0rj/q-t5-tied-58M-base",
      "revision": "5dcd2373889c2de4289ea6edd44bbc1b8bedfa45",
      "label": "Q-T5 tied (10 + 10)",
      "group": "seq2seq",
      "parameters": 57707008,
      "active_parameters": null,
      "optimized_parameters": null,
      "mean_16_percent": 42.10554816950241,
      "tasks": [
        {
          "id": "hellaswag",
          "label": "HellaSwag",
          "metric": "acc_norm",
          "value": 0.280920135431189,
          "stderr": 0.004485300194072218,
          "ci95": [
            0.27221460254342755,
            0.28979321750467757
          ],
          "samples": 10042,
          "dataset": "Rowan/hellaswag",
          "split": "validation"
        },
        {
          "id": "arc_easy",
          "label": "ARC-Easy",
          "metric": "acc_norm",
          "value": 0.3771043771043771,
          "stderr": 0.009945041946366447,
          "ci95": [
            0.3578296534394978,
            0.3967758486054178
          ],
          "samples": 2376,
          "dataset": "allenai/ai2_arc",
          "split": "test"
        },
        {
          "id": "arc_challenge",
          "label": "ARC-Challenge",
          "metric": "acc_norm",
          "value": 0.2295221843003413,
          "stderr": 0.012288926760890708,
          "ci95": [
            0.20635337780287624,
            0.25445828584107455
          ],
          "samples": 1172,
          "dataset": "allenai/ai2_arc",
          "split": "test"
        },
        {
          "id": "piqa",
          "label": "PIQA",
          "metric": "acc_norm",
          "value": 0.5718171926006529,
          "stderr": 0.011544859155318712,
          "ci95": [
            0.5490691605103258,
            0.594265651905633
          ],
          "samples": 1838,
          "dataset": "baber/piqa",
          "split": "validation"
        },
        {
          "id": "winogrande",
          "label": "WinoGrande",
          "metric": "acc",
          "value": 0.510655090765588,
          "stderr": 0.014049294536290372,
          "ci95": [
            0.48313925422598825,
            0.5381065115716237
          ],
          "samples": 1267,
          "dataset": "allenai/winogrande",
          "split": "validation"
        },
        {
          "id": "openbookqa",
          "label": "OpenBookQA",
          "metric": "acc_norm",
          "value": 0.272,
          "stderr": 0.019920483209566107,
          "ci95": [
            0.2348440922052204,
            0.31263260704524753
          ],
          "samples": 500,
          "dataset": "allenai/openbookqa",
          "split": "test"
        },
        {
          "id": "boolq",
          "label": "BoolQ",
          "metric": "acc",
          "value": 0.591743119266055,
          "stderr": 0.008596583869583145,
          "ci95": [
            0.5747985938632083,
            0.6084723456575261
          ],
          "samples": 3270,
          "dataset": "aps/super_glue",
          "split": "validation"
        },
        {
          "id": "lambada_openai",
          "label": "LAMBADA OpenAI",
          "metric": "acc",
          "value": 0.25984863186493307,
          "stderr": 0.006109878348081022,
          "ci95": [
            0.24805667095981895,
            0.2719983821645015
          ],
          "samples": 5153,
          "dataset": "EleutherAI/lambada_openai",
          "split": "test"
        },
        {
          "id": "arithmark3",
          "label": "ArithMark-3",
          "metric": "acc_norm",
          "value": 0.363,
          "stderr": 0.015213890444671358,
          "ci95": [
            0.3337729626449257,
            0.39327556918011636
          ],
          "samples": 1000,
          "dataset": "AxiomicLabs/Arithmark-3.0",
          "split": "train"
        },
        {
          "id": "balanced_copa",
          "label": "Balanced COPA",
          "metric": "acc",
          "value": 0.542,
          "stderr": 0.015763390640483554,
          "ci95": [
            0.5110177730722398,
            0.5726607792149736
          ],
          "samples": 1000,
          "dataset": "pkavumba/balanced-copa",
          "split": "train"
        },
        {
          "id": "blimp",
          "label": "BLiMP",
          "metric": "acc",
          "value": 0.7244925373134328,
          "stderr": 0.0015133319750304283,
          "ci95": null,
          "samples": 67000,
          "dataset": "nyu-mll/blimp",
          "split": "train"
        },
        {
          "id": "commonsense_qa",
          "label": "CommonsenseQA",
          "metric": "acc",
          "value": 0.19656019656019655,
          "stderr": 0.011377439773963867,
          "ci95": [
            0.17523627664679478,
            0.21978746731313606
          ],
          "samples": 1221,
          "dataset": "tau/commonsense_qa",
          "split": "validation"
        },
        {
          "id": "mmlu_continuation",
          "label": "MMLU continuation",
          "metric": "acc",
          "value": 0.24910981341689217,
          "stderr": 0.003638719420160661,
          "ci95": null,
          "samples": 14042,
          "dataset": "cais/mmlu",
          "split": "test"
        },
        {
          "id": "sciq",
          "label": "SciQ (with support)",
          "metric": "acc_norm",
          "value": 0.651,
          "stderr": 0.015080663991563052,
          "ci95": [
            0.6209303029513429,
            0.6799140159860195
          ],
          "samples": 1000,
          "dataset": "allenai/sciq",
          "split": "test"
        },
        {
          "id": "truthfulqa_mc2",
          "label": "TruthfulQA MC2",
          "metric": "acc",
          "value": 0.4542572856395859,
          "stderr": 0.015651216709838878,
          "ci95": null,
          "samples": 817,
          "dataset": "truthfulqa/truthful_qa",
          "split": "validation"
        },
        {
          "id": "bananamind_base",
          "label": "BananaMind Base 1.1",
          "metric": "raw_accuracy",
          "value": 0.46285714285714286,
          "stderr": 0.02669043635870257,
          "ci95": [
            0.41130563904726447,
            0.5152151251202401
          ],
          "samples": 350,
          "dataset": "BananaMind/BananaMind-Base-Bench-1.1",
          "split": "test"
        }
      ],
      "continuation": {
        "nll": 3.731859866696125,
        "nll_ci95": [
          3.6906630000297964,
          3.7720494392811226
        ],
        "ppl": 41.756697880126325,
        "ppl_ci95": [
          40.07140549519545,
          43.469060809515916
        ]
      },
      "continuation_precision": "torch.float32",
      "source_url": "https://huggingface.co/d0rj/q-t5-tied-58M-base/blob/5dcd2373889c2de4289ea6edd44bbc1b8bedfa45/evaluation/results.json",
      "source_sha256": "70f8d4cdeae310b57474d1d13facbcf8aac80194bc631b3ffdd0ca70f3449969",
      "card_url": "https://huggingface.co/d0rj/q-t5-tied-58M-base/blob/5dcd2373889c2de4289ea6edd44bbc1b8bedfa45/README.md",
      "delta_pp": -0.7186082061715666
    },
    {
      "repo": "d0rj/q-mod-51M-base",
      "revision": "622b284e1e1c5a8f00a46eba9a667935db88c66b",
      "label": "Q-MoD",
      "group": "causal",
      "parameters": 50919168,
      "active_parameters": null,
      "optimized_parameters": null,
      "mean_16_percent": 41.461276742189675,
      "tasks": [
        {
          "id": "hellaswag",
          "label": "HellaSwag",
          "metric": "acc_norm",
          "value": 0.2874925313682533,
          "stderr": 0.004516681953878919,
          "ci95": [
            0.27872301915216174,
            0.29642456629450026
          ],
          "samples": 10042,
          "dataset": "Rowan/hellaswag",
          "split": "validation"
        },
        {
          "id": "arc_easy",
          "label": "ARC-Easy",
          "metric": "acc_norm",
          "value": 0.41919191919191917,
          "stderr": 0.010124905282491126,
          "ci95": [
            0.3994976736933545,
            0.43914703998001464
          ],
          "samples": 2376,
          "dataset": "allenai/ai2_arc",
          "split": "test"
        },
        {
          "id": "arc_challenge",
          "label": "ARC-Challenge",
          "metric": "acc_norm",
          "value": 0.23122866894197952,
          "stderr": 0.01232085883477234,
          "ci95": [
            0.2079920763450836,
            0.2562214064556877
          ],
          "samples": 1172,
          "dataset": "allenai/ai2_arc",
          "split": "test"
        },
        {
          "id": "piqa",
          "label": "PIQA",
          "metric": "acc_norm",
          "value": 0.5875952121871599,
          "stderr": 0.011485407152743092,
          "ci95": [
            0.5649303971157125,
            0.6098946392401159
          ],
          "samples": 1838,
          "dataset": "baber/piqa",
          "split": "validation"
        },
        {
          "id": "winogrande",
          "label": "WinoGrande",
          "metric": "acc",
          "value": 0.5138121546961326,
          "stderr": 0.014047122916440271,
          "ci95": [
            0.4862910104211901,
            0.5412497970942327
          ],
          "samples": 1267,
          "dataset": "allenai/winogrande",
          "split": "validation"
        },
        {
          "id": "openbookqa",
          "label": "OpenBookQA",
          "metric": "acc_norm",
          "value": 0.306,
          "stderr": 0.020629569998345414,
          "ci95": [
            0.26721346373113636,
            0.3477447803679459
          ],
          "samples": 500,
          "dataset": "allenai/openbookqa",
          "split": "test"
        },
        {
          "id": "boolq",
          "label": "BoolQ",
          "metric": "acc",
          "value": 0.5577981651376147,
          "stderr": 0.008686430526114761,
          "ci95": [
            0.5407177130610681,
            0.5747429788369947
          ],
          "samples": 3270,
          "dataset": "aps/super_glue",
          "split": "validation"
        },
        {
          "id": "lambada_openai",
          "label": "LAMBADA OpenAI",
          "metric": "acc",
          "value": 0.16689307199689501,
          "stderr": 0.005194952730633079,
          "ci95": [
            0.1569610474392711,
            0.17732137574286708
          ],
          "samples": 5153,
          "dataset": "EleutherAI/lambada_openai",
          "split": "test"
        },
        {
          "id": "arithmark3",
          "label": "ArithMark-3",
          "metric": "acc_norm",
          "value": 0.341,
          "stderr": 0.01499813134840284,
          "ci95": [
            0.31227731829126676,
            0.37093959090713985
          ],
          "samples": 1000,
          "dataset": "AxiomicLabs/Arithmark-3.0",
          "split": "train"
        },
        {
          "id": "balanced_copa",
          "label": "Balanced COPA",
          "metric": "acc",
          "value": 0.533,
          "stderr": 0.015784807891138876,
          "ci95": [
            0.5020104989134991,
            0.5637369350264544
          ],
          "samples": 1000,
          "dataset": "pkavumba/balanced-copa",
          "split": "train"
        },
        {
          "id": "blimp",
          "label": "BLiMP",
          "metric": "acc",
          "value": 0.7317910447761194,
          "stderr": 0.001517376416839114,
          "ci95": null,
          "samples": 67000,
          "dataset": "nyu-mll/blimp",
          "split": "train"
        },
        {
          "id": "commonsense_qa",
          "label": "CommonsenseQA",
          "metric": "acc",
          "value": 0.19492219492219492,
          "stderr": 0.011341478090883516,
          "ci95": [
            0.17367347122724505,
            0.21808454395514162
          ],
          "samples": 1221,
          "dataset": "tau/commonsense_qa",
          "split": "validation"
        },
        {
          "id": "mmlu_continuation",
          "label": "MMLU continuation",
          "metric": "acc",
          "value": 0.2521720552627831,
          "stderr": 0.0036528205915989775,
          "ci95": null,
          "samples": 14042,
          "dataset": "cais/mmlu",
          "split": "test"
        },
        {
          "id": "sciq",
          "label": "SciQ (with support)",
          "metric": "acc_norm",
          "value": 0.612,
          "stderr": 0.015417317979911216,
          "ci95": [
            0.5814239245764211,
            0.6417188815228144
          ],
          "samples": 1000,
          "dataset": "allenai/sciq",
          "split": "test"
        },
        {
          "id": "truthfulqa_mc2",
          "label": "TruthfulQA MC2",
          "metric": "acc",
          "value": 0.4417644031264386,
          "stderr": 0.01519901667714673,
          "ci95": null,
          "samples": 817,
          "dataset": "truthfulqa/truthful_qa",
          "split": "validation"
        },
        {
          "id": "bananamind_base",
          "label": "BananaMind Base 1.1",
          "metric": "raw_accuracy",
          "value": 0.45714285714285713,
          "stderr": 0.02666588697376804,
          "ci95": [
            0.40570065529637145,
            0.5095156110507492
          ],
          "samples": 350,
          "dataset": "BananaMind/BananaMind-Base-Bench-1.1",
          "split": "test"
        }
      ],
      "continuation": {
        "nll": 3.6888507895453277,
        "nll_ci95": [
          3.649182625079073,
          3.728063138858559
        ],
        "ppl": 39.99885343368865,
        "ppl_ci95": [
          38.44323067090763,
          41.598459646718396
        ]
      },
      "continuation_precision": "torch.float32",
      "source_url": "https://huggingface.co/d0rj/q-mod-51M-base/blob/622b284e1e1c5a8f00a46eba9a667935db88c66b/evaluation/results.json",
      "source_sha256": "dcda27cae5a1e4437f1d4d634fe57cf6ad609603e33d524fe209792157254309",
      "card_url": "https://huggingface.co/d0rj/q-mod-51M-base/blob/622b284e1e1c5a8f00a46eba9a667935db88c66b/README.md",
      "delta_pp": -1.3628796334843045
    },
    {
      "repo": "d0rj/zarya-51M-base",
      "revision": "b737e407881d7c47aaeaa7c8a63e0756b13a47fd",
      "label": "Zarya (AR + MDM)",
      "group": "causal",
      "parameters": 50852096,
      "active_parameters": null,
      "optimized_parameters": null,
      "mean_16_percent": 41.37540290345162,
      "tasks": [
        {
          "id": "hellaswag",
          "label": "HellaSwag",
          "metric": "acc_norm",
          "value": 0.27325234017128064,
          "stderr": 0.004447185883327611,
          "ci95": [
            0.26462439196595045,
            0.2820537017838384
          ],
          "samples": 10042,
          "dataset": "Rowan/hellaswag",
          "split": "validation"
        },
        {
          "id": "arc_easy",
          "label": "ARC-Easy",
          "metric": "acc_norm",
          "value": 0.39941077441077444,
          "stderr": 0.010050018228741988,
          "ci95": [
            0.37989484476305374,
            0.4192514394449423
          ],
          "samples": 2376,
          "dataset": "allenai/ai2_arc",
          "split": "test"
        },
        {
          "id": "arc_challenge",
          "label": "ARC-Challenge",
          "metric": "acc_norm",
          "value": 0.23464163822525597,
          "stderr": 0.01238387356076873,
          "ci95": [
            0.21127112613365806,
            0.2597459949807542
          ],
          "samples": 1172,
          "dataset": "allenai/ai2_arc",
          "split": "test"
        },
        {
          "id": "piqa",
          "label": "PIQA",
          "metric": "acc_norm",
          "value": 0.5772578890097932,
          "stderr": 0.01152572136157213,
          "ci95": [
            0.5545358905047227,
            0.5996576198215359
          ],
          "samples": 1838,
          "dataset": "baber/piqa",
          "split": "validation"
        },
        {
          "id": "winogrande",
          "label": "WinoGrande",
          "metric": "acc",
          "value": 0.5248618784530387,
          "stderr": 0.01403510288362781,
          "ci95": [
            0.4973307762059857,
            0.5522426773217756
          ],
          "samples": 1267,
          "dataset": "allenai/winogrande",
          "split": "validation"
        },
        {
          "id": "openbookqa",
          "label": "OpenBookQA",
          "metric": "acc_norm",
          "value": 0.266,
          "stderr": 0.019780559675655396,
          "ci95": [
            0.2291604018204711,
            0.30640778951553543
          ],
          "samples": 500,
          "dataset": "allenai/openbookqa",
          "split": "test"
        },
        {
          "id": "boolq",
          "label": "BoolQ",
          "metric": "acc",
          "value": 0.5923547400611621,
          "stderr": 0.008594580270731697,
          "ci95": [
            0.5754134163866872,
            0.6090793293975186
          ],
          "samples": 3270,
          "dataset": "aps/super_glue",
          "split": "validation"
        },
        {
          "id": "lambada_openai",
          "label": "LAMBADA OpenAI",
          "metric": "acc",
          "value": 0.18455268775470599,
          "stderr": 0.005404682831181825,
          "ci95": [
            0.1741970550653037,
            0.19537828946284402
          ],
          "samples": 5153,
          "dataset": "EleutherAI/lambada_openai",
          "split": "test"
        },
        {
          "id": "arithmark3",
          "label": "ArithMark-3",
          "metric": "acc_norm",
          "value": 0.36,
          "stderr": 0.015186527932039983,
          "ci95": [
            0.3308377297718964,
            0.39023376260405895
          ],
          "samples": 1000,
          "dataset": "AxiomicLabs/Arithmark-3.0",
          "split": "train"
        },
        {
          "id": "balanced_copa",
          "label": "Balanced COPA",
          "metric": "acc",
          "value": 0.519,
          "stderr": 0.015807874268505835,
          "ci95": [
            0.48801914625314236,
            0.5498354369244066
          ],
          "samples": 1000,
          "dataset": "pkavumba/balanced-copa",
          "split": "train"
        },
        {
          "id": "blimp",
          "label": "BLiMP",
          "metric": "acc",
          "value": 0.7196567164179104,
          "stderr": 0.0015155272267772084,
          "ci95": null,
          "samples": 67000,
          "dataset": "nyu-mll/blimp",
          "split": "train"
        },
        {
          "id": "commonsense_qa",
          "label": "CommonsenseQA",
          "metric": "acc",
          "value": 0.19656019656019655,
          "stderr": 0.011377439773963867,
          "ci95": [
            0.17523627664679478,
            0.21978746731313606
          ],
          "samples": 1221,
          "dataset": "tau/commonsense_qa",
          "split": "validation"
        },
        {
          "id": "mmlu_continuation",
          "label": "MMLU continuation",
          "metric": "acc",
          "value": 0.24996439253667568,
          "stderr": 0.003644401052128753,
          "ci95": null,
          "samples": 14042,
          "dataset": "cais/mmlu",
          "split": "test"
        },
        {
          "id": "sciq",
          "label": "SciQ (with support)",
          "metric": "acc_norm",
          "value": 0.624,
          "stderr": 0.0153251055088981,
          "ci95": [
            0.5935575899582508,
            0.6534933739373314
          ],
          "samples": 1000,
          "dataset": "allenai/sciq",
          "split": "test"
        },
        {
          "id": "truthfulqa_mc2",
          "label": "TruthfulQA MC2",
          "metric": "acc",
          "value": 0.45855121095146534,
          "stderr": 0.015508783749876553,
          "ci95": null,
          "samples": 817,
          "dataset": "truthfulqa/truthful_qa",
          "split": "validation"
        },
        {
          "id": "bananamind_base",
          "label": "BananaMind Base 1.1",
          "metric": "raw_accuracy",
          "value": 0.44,
          "stderr": 0.026570984023499722,
          "ci95": [
            0.38892662161199665,
            0.4923761512739723
          ],
          "samples": 350,
          "dataset": "BananaMind/BananaMind-Base-Bench-1.1",
          "split": "test"
        }
      ],
      "continuation": {
        "nll": 3.8300181172557712,
        "nll_ci95": [
          3.788649591301725,
          3.8708685474911912
        ],
        "ppl": 46.063372768560846,
        "ppl_ci95": [
          44.19667638300468,
          47.984044408470616
        ]
      },
      "continuation_precision": "torch.float32",
      "source_url": "https://huggingface.co/d0rj/zarya-51M-base/blob/b737e407881d7c47aaeaa7c8a63e0756b13a47fd/evaluation/results.json",
      "source_sha256": "9b7fa4aeb710f02aa6a99c0d8b71f811e249c5fb874b36db6467dacd8ff5ed47",
      "card_url": "https://huggingface.co/d0rj/zarya-51M-base/blob/b737e407881d7c47aaeaa7c8a63e0756b13a47fd/README.md",
      "delta_pp": -1.4487534722223572
    },
    {
      "repo": "d0rj/q-t5-tied-37M-base",
      "revision": "53b26902d1681cd6aa8bdd491a04390745b1801a",
      "label": "Q-T5 tied (5 + 5)",
      "group": "seq2seq",
      "parameters": 37374208,
      "active_parameters": null,
      "optimized_parameters": null,
      "mean_16_percent": 40.727991183438775,
      "tasks": [
        {
          "id": "hellaswag",
          "label": "HellaSwag",
          "metric": "acc_norm",
          "value": 0.27504481179047996,
          "stderr": 0.004456242601950599,
          "ci95": [
            0.2663984392478132,
            0.2838632268834002
          ],
          "samples": 10042,
          "dataset": "Rowan/hellaswag",
          "split": "validation"
        },
        {
          "id": "arc_easy",
          "label": "ARC-Easy",
          "metric": "acc_norm",
          "value": 0.3686868686868687,
          "stderr": 0.009899640855681058,
          "ci95": [
            0.3495144777754739,
            0.38828318194375094
          ],
          "samples": 2376,
          "dataset": "allenai/ai2_arc",
          "split": "test"
        },
        {
          "id": "arc_challenge",
          "label": "ARC-Challenge",
          "metric": "acc_norm",
          "value": 0.2354948805460751,
          "stderr": 0.01239945185500478,
          "ci95": [
            0.21209122984248996,
            0.2606268008503326
          ],
          "samples": 1172,
          "dataset": "allenai/ai2_arc",
          "split": "test"
        },
        {
          "id": "piqa",
          "label": "PIQA",
          "metric": "acc_norm",
          "value": 0.5707290533188248,
          "stderr": 0.01154851547125808,
          "ci95": [
            0.5479761489777162,
            0.5931869238561827
          ],
          "samples": 1838,
          "dataset": "baber/piqa",
          "split": "validation"
        },
        {
          "id": "winogrande",
          "label": "WinoGrande",
          "metric": "acc",
          "value": 0.4972375690607735,
          "stderr": 0.014052271211616401,
          "ci95": [
            0.4697564851267556,
            0.5247353533701599
          ],
          "samples": 1267,
          "dataset": "allenai/winogrande",
          "split": "validation"
        },
        {
          "id": "openbookqa",
          "label": "OpenBookQA",
          "metric": "acc_norm",
          "value": 0.284,
          "stderr": 0.020186703693570774,
          "ci95": [
            0.2462377719556586,
            0.32505594312373204
          ],
          "samples": 500,
          "dataset": "allenai/openbookqa",
          "split": "test"
        },
        {
          "id": "boolq",
          "label": "BoolQ",
          "metric": "acc",
          "value": 0.5464831804281346,
          "stderr": 0.008707182331111892,
          "ci95": [
            0.5293754103731186,
            0.5634818656507202
          ],
          "samples": 3270,
          "dataset": "aps/super_glue",
          "split": "validation"
        },
        {
          "id": "lambada_openai",
          "label": "LAMBADA OpenAI",
          "metric": "acc",
          "value": 0.21831942557733358,
          "stderr": 0.00575536567756351,
          "ci95": [
            0.2072522910858988,
            0.22980622173022328
          ],
          "samples": 5153,
          "dataset": "EleutherAI/lambada_openai",
          "split": "test"
        },
        {
          "id": "arithmark3",
          "label": "ArithMark-3",
          "metric": "acc_norm",
          "value": 0.349,
          "stderr": 0.015080663991563052,
          "ci95": [
            0.3200859840139804,
            0.3790696970486572
          ],
          "samples": 1000,
          "dataset": "AxiomicLabs/Arithmark-3.0",
          "split": "train"
        },
        {
          "id": "balanced_copa",
          "label": "Balanced COPA",
          "metric": "acc",
          "value": 0.549,
          "stderr": 0.015743152379585443,
          "ci95": [
            0.5180304041955376,
            0.5795945734728781
          ],
          "samples": 1000,
          "dataset": "pkavumba/balanced-copa",
          "split": "train"
        },
        {
          "id": "blimp",
          "label": "BLiMP",
          "metric": "acc",
          "value": 0.7140149253731343,
          "stderr": 0.001537548688509093,
          "ci95": null,
          "samples": 67000,
          "dataset": "nyu-mll/blimp",
          "split": "train"
        },
        {
          "id": "commonsense_qa",
          "label": "CommonsenseQA",
          "metric": "acc",
          "value": 0.19737919737919737,
          "stderr": 0.011395305685091258,
          "ci95": [
            0.17601790285026156,
            0.22063870549844142
          ],
          "samples": 1221,
          "dataset": "tau/commonsense_qa",
          "split": "validation"
        },
        {
          "id": "mmlu_continuation",
          "label": "MMLU continuation",
          "metric": "acc",
          "value": 0.2472582253240279,
          "stderr": 0.0036325492100833206,
          "ci95": null,
          "samples": 14042,
          "dataset": "cais/mmlu",
          "split": "test"
        },
        {
          "id": "sciq",
          "label": "SciQ (with support)",
          "metric": "acc_norm",
          "value": 0.552,
          "stderr": 0.01573351656634789,
          "ci95": [
            0.5210376917018592,
            0.5825643254156431
          ],
          "samples": 1000,
          "dataset": "allenai/sciq",
          "split": "test"
        },
        {
          "id": "truthfulqa_mc2",
          "label": "TruthfulQA MC2",
          "metric": "acc",
          "value": 0.4746875947224976,
          "stderr": 0.015738963938532493,
          "ci95": null,
          "samples": 817,
          "dataset": "truthfulqa/truthful_qa",
          "split": "validation"
        },
        {
          "id": "bananamind_base",
          "label": "BananaMind Base 1.1",
          "metric": "raw_accuracy",
          "value": 0.43714285714285717,
          "stderr": 0.026552050821771547,
          "ci95": [
            0.3861369476736846,
            0.4895135763020924
          ],
          "samples": 350,
          "dataset": "BananaMind/BananaMind-Base-Bench-1.1",
          "split": "test"
        }
      ],
      "continuation": {
        "nll": 3.9086589846004736,
        "nll_ci95": [
          3.867216858441887,
          3.949137798207732
        ],
        "ppl": 49.83208155754123,
        "ppl_ci95": [
          47.80914113949765,
          51.89060736708381
        ]
      },
      "continuation_precision": "torch.float32",
      "source_url": "https://huggingface.co/d0rj/q-t5-tied-37M-base/blob/53b26902d1681cd6aa8bdd491a04390745b1801a/evaluation/results.json",
      "source_sha256": "ca4587db65d7e6d0951d7f9ba98773d442dd8b2f67a8965286192b71da1ca725",
      "card_url": "https://huggingface.co/d0rj/q-t5-tied-37M-base/blob/53b26902d1681cd6aa8bdd491a04390745b1801a/README.md",
      "delta_pp": -2.0961651922352047
    },
    {
      "repo": "d0rj/t5-dense-55M-base",
      "revision": "43e6eaa72a6b1641b87aea553ddeb306a6964c86",
      "label": "T5 dense",
      "group": "seq2seq",
      "parameters": 54809088,
      "active_parameters": null,
      "optimized_parameters": null,
      "mean_16_percent": 39.82806447280975,
      "tasks": [
        {
          "id": "hellaswag",
          "label": "HellaSwag",
          "metric": "acc_norm",
          "value": 0.2733519219279028,
          "stderr": 0.004447691405592456,
          "ci95": [
            0.2647229455043319,
            0.2821542355999805
          ],
          "samples": 10042,
          "dataset": "Rowan/hellaswag",
          "split": "validation"
        },
        {
          "id": "arc_easy",
          "label": "ARC-Easy",
          "metric": "acc_norm",
          "value": 0.37626262626262624,
          "stderr": 0.009940646221513758,
          "ci95": [
            0.3569978536758919,
            0.3959268641364546
          ],
          "samples": 2376,
          "dataset": "allenai/ai2_arc",
          "split": "test"
        },
        {
          "id": "arc_challenge",
          "label": "ARC-Challenge",
          "metric": "acc_norm",
          "value": 0.22098976109215018,
          "stderr": 0.012124929206818241,
          "ci95": [
            0.1981683230517509,
            0.2456342448080974
          ],
          "samples": 1172,
          "dataset": "allenai/ai2_arc",
          "split": "test"
        },
        {
          "id": "piqa",
          "label": "PIQA",
          "metric": "acc_norm",
          "value": 0.5642002176278563,
          "stderr": 0.011569259195486755,
          "ci95": [
            0.5414204122202725,
            0.5867122231212664
          ],
          "samples": 1838,
          "dataset": "baber/piqa",
          "split": "validation"
        },
        {
          "id": "winogrande",
          "label": "WinoGrande",
          "metric": "acc",
          "value": 0.4956590370955012,
          "stderr": 0.014051956064076884,
          "ci95": [
            0.46818333933385375,
            0.5231609783041562
          ],
          "samples": 1267,
          "dataset": "allenai/winogrande",
          "split": "validation"
        },
        {
          "id": "openbookqa",
          "label": "OpenBookQA",
          "metric": "acc_norm",
          "value": 0.25,
          "stderr": 0.019384310743640384,
          "ci95": [
            0.21404849674782103,
            0.2897636734829552
          ],
          "samples": 500,
          "dataset": "allenai/openbookqa",
          "split": "test"
        },
        {
          "id": "boolq",
          "label": "BoolQ",
          "metric": "acc",
          "value": 0.5795107033639144,
          "stderr": 0.00863377533246365,
          "ci95": [
            0.562507777985363,
            0.5963270362659402
          ],
          "samples": 3270,
          "dataset": "aps/super_glue",
          "split": "validation"
        },
        {
          "id": "lambada_openai",
          "label": "LAMBADA OpenAI",
          "metric": "acc",
          "value": 0.1892101688336891,
          "stderr": 0.005456805271056482,
          "ci95": [
            0.1787490571137562,
            0.2001343106265259
          ],
          "samples": 5153,
          "dataset": "EleutherAI/lambada_openai",
          "split": "test"
        },
        {
          "id": "arithmark3",
          "label": "ArithMark-3",
          "metric": "acc_norm",
          "value": 0.318,
          "stderr": 0.014734079309311901,
          "ci95": [
            0.2898795107350345,
            0.34751342935370755
          ],
          "samples": 1000,
          "dataset": "AxiomicLabs/Arithmark-3.0",
          "split": "train"
        },
        {
          "id": "balanced_copa",
          "label": "Balanced COPA",
          "metric": "acc",
          "value": 0.532,
          "stderr": 0.015786868759359002,
          "ci95": [
            0.5010103116491519,
            0.5627447758077727
          ],
          "samples": 1000,
          "dataset": "pkavumba/balanced-copa",
          "split": "train"
        },
        {
          "id": "blimp",
          "label": "BLiMP",
          "metric": "acc",
          "value": 0.6288358208955224,
          "stderr": 0.0016018312198498807,
          "ci95": null,
          "samples": 67000,
          "dataset": "nyu-mll/blimp",
          "split": "train"
        },
        {
          "id": "commonsense_qa",
          "label": "CommonsenseQA",
          "metric": "acc",
          "value": 0.19656019656019655,
          "stderr": 0.011377439773963867,
          "ci95": [
            0.17523627664679478,
            0.21978746731313606
          ],
          "samples": 1221,
          "dataset": "tau/commonsense_qa",
          "split": "validation"
        },
        {
          "id": "mmlu_continuation",
          "label": "MMLU continuation",
          "metric": "acc",
          "value": 0.24661729098419027,
          "stderr": 0.0036287284122921157,
          "ci95": null,
          "samples": 14042,
          "dataset": "cais/mmlu",
          "split": "test"
        },
        {
          "id": "sciq",
          "label": "SciQ (with support)",
          "metric": "acc_norm",
          "value": 0.601,
          "stderr": 0.015493193313163012,
          "ci95": [
            0.5703182462412869,
            0.6309087485446309
          ],
          "samples": 1000,
          "dataset": "allenai/sciq",
          "split": "test"
        },
        {
          "id": "truthfulqa_mc2",
          "label": "TruthfulQA MC2",
          "metric": "acc",
          "value": 0.48029257100601047,
          "stderr": 0.01587829332754489,
          "ci95": null,
          "samples": 817,
          "dataset": "truthfulqa/truthful_qa",
          "split": "validation"
        },
        {
          "id": "bananamind_base",
          "label": "BananaMind Base 1.1",
          "metric": "raw_accuracy",
          "value": 0.42,
          "stderr": 0.0264195811727017,
          "ci95": [
            0.36943522567489206,
            0.47230180483973316
          ],
          "samples": 350,
          "dataset": "BananaMind/BananaMind-Base-Bench-1.1",
          "split": "test"
        }
      ],
      "continuation": {
        "nll": 3.9129423529831406,
        "nll_ci95": [
          3.8717073203157315,
          3.9535854535823836
        ],
        "ppl": 50.04598851422466,
        "ppl_ci95": [
          48.02430900499601,
          52.12191290820044
        ]
      },
      "continuation_precision": "torch.float32",
      "source_url": "https://huggingface.co/d0rj/t5-dense-55M-base/blob/43e6eaa72a6b1641b87aea553ddeb306a6964c86/evaluation/results.json",
      "source_sha256": "e8fd93d0c9b313e9ec21bd929bc4d89b9d60072a3fa622f1b44b1a566a27cff1",
      "card_url": "https://huggingface.co/d0rj/t5-dense-55M-base/blob/43e6eaa72a6b1641b87aea553ddeb306a6964c86/README.md",
      "delta_pp": -2.9960919028642294
    },
    {
      "repo": "d0rj/t5-moe-55M-base",
      "revision": "de3885b506c0106770b59b82885b18ea1afee193",
      "label": "T5 MoE",
      "group": "seq2seq",
      "parameters": 54858240,
      "active_parameters": null,
      "optimized_parameters": null,
      "mean_16_percent": 39.7758484235927,
      "tasks": [
        {
          "id": "hellaswag",
          "label": "hellaswag",
          "metric": "acc_norm",
          "value": 0.2779326827325234,
          "stderr": 0.004470644845243263,
          "ci95": [
            0.2692569969516344,
            0.28677820246076313
          ],
          "samples": 10042,
          "dataset": "Rowan/hellaswag",
          "split": "validation"
        },
        {
          "id": "arc_easy",
          "label": "arc_easy",
          "metric": "acc_norm",
          "value": 0.38930976430976433,
          "stderr": 0.010005212782878208,
          "ci95": [
            0.3698977221380954,
            0.40907915127907174
          ],
          "samples": 2376,
          "dataset": "allenai/ai2_arc",
          "split": "test"
        },
        {
          "id": "arc_challenge",
          "label": "arc_challenge",
          "metric": "acc_norm",
          "value": 0.2380546075085324,
          "stderr": 0.012445770028026118,
          "ci95": [
            0.21455235182352161,
            0.26326840760453163
          ],
          "samples": 1172,
          "dataset": "allenai/ai2_arc",
          "split": "test"
        },
        {
          "id": "piqa",
          "label": "piqa",
          "metric": "acc_norm",
          "value": 0.5560391730141458,
          "stderr": 0.011592322161277677,
          "ci95": [
            0.5332313404997734,
            0.5786132479763156
          ],
          "samples": 1838,
          "dataset": "baber/piqa",
          "split": "validation"
        },
        {
          "id": "winogrande",
          "label": "winogrande",
          "metric": "acc",
          "value": 0.4846093133385951,
          "stderr": 0.014045826789783585,
          "ci95": [
            0.4571789703267686,
            0.512132701298903
          ],
          "samples": 1267,
          "dataset": "allenai/winogrande",
          "split": "validation"
        },
        {
          "id": "openbookqa",
          "label": "openbookqa",
          "metric": "acc_norm",
          "value": 0.262,
          "stderr": 0.019684688820194637,
          "ci95": [
            0.22537626941723765,
            0.30225291664246134
          ],
          "samples": 500,
          "dataset": "allenai/openbookqa",
          "split": "test"
        },
        {
          "id": "boolq",
          "label": "boolq",
          "metric": "acc",
          "value": 0.499388379204893,
          "stderr": 0.00874504843369544,
          "ci95": [
            0.48226179513749795,
            0.5165163985990305
          ],
          "samples": 3270,
          "dataset": "aps/super_glue",
          "split": "validation"
        },
        {
          "id": "lambada_openai",
          "label": "lambada_openai",
          "metric": "acc",
          "value": 0.14826314768096255,
          "stderr": 0.004950868601262541,
          "ci95": [
            0.13882265856941103,
            0.1582276717641896
          ],
          "samples": 5153,
          "dataset": "EleutherAI/lambada_openai",
          "split": "test"
        },
        {
          "id": "arithmark3",
          "label": "ArithMark-3",
          "metric": "acc_norm",
          "value": 0.342,
          "stderr": 0.015008706182121804,
          "ci95": [
            0.3132528988050859,
            0.37195635687634954
          ],
          "samples": 1000,
          "dataset": "AxiomicLabs/Arithmark-3.0",
          "split": "train"
        },
        {
          "id": "balanced_copa",
          "label": "Balanced COPA",
          "metric": "acc",
          "value": 0.538,
          "stderr": 0.015773547629015002,
          "ci95": [
            0.5070132970793998,
            0.5686958692756982
          ],
          "samples": 1000,
          "dataset": "pkavumba/balanced-copa",
          "split": "train"
        },
        {
          "id": "blimp",
          "label": "BLiMP",
          "metric": "acc",
          "value": 0.6988955223880597,
          "stderr": 0.0015386258071003309,
          "ci95": null,
          "samples": 67000,
          "dataset": "nyu-mll/blimp",
          "split": "train"
        },
        {
          "id": "commonsense_qa",
          "label": "CommonsenseQA",
          "metric": "acc",
          "value": 0.20065520065520065,
          "stderr": 0.011466011466011467,
          "ci95": [
            0.17914588148536167,
            0.2240421844184298
          ],
          "samples": 1221,
          "dataset": "tau/commonsense_qa",
          "split": "validation"
        },
        {
          "id": "mmlu_continuation",
          "label": "MMLU continuation",
          "metric": "acc",
          "value": 0.24932345819683804,
          "stderr": 0.003640204268569072,
          "ci95": null,
          "samples": 14042,
          "dataset": "cais/mmlu",
          "split": "test"
        },
        {
          "id": "sciq",
          "label": "SciQ (with support)",
          "metric": "acc_norm",
          "value": 0.654,
          "stderr": 0.01505026612756434,
          "ci95": [
            0.6239780184885133,
            0.6828433398979359
          ],
          "samples": 1000,
          "dataset": "allenai/sciq",
          "split": "test"
        },
        {
          "id": "truthfulqa_mc2",
          "label": "TruthfulQA MC2",
          "metric": "acc",
          "value": 0.4656644987453161,
          "stderr": 0.016000577772697092,
          "ci95": null,
          "samples": 817,
          "dataset": "truthfulqa/truthful_qa",
          "split": "validation"
        },
        {
          "id": "bananamind_base",
          "label": "BananaMind Base 1.1",
          "metric": "raw_accuracy",
          "value": 0.36,
          "stderr": 0.025693810923465083,
          "ci95": [
            0.3114835744745811,
            0.41155622892601307
          ],
          "samples": 350,
          "dataset": "BananaMind/BananaMind-Base-Bench-1.1",
          "split": "test"
        }
      ],
      "continuation": {
        "nll": 3.9203571982400116,
        "nll_ci95": [
          3.8792989141547807,
          3.960741957038948
        ],
        "ppl": 50.41845094392994,
        "ppl_ci95": [
          48.39027743758022,
          52.496261473788515
        ]
      },
      "continuation_precision": "CPU FP32",
      "source_url": "https://huggingface.co/d0rj/t5-moe-55M-base/blob/de3885b506c0106770b59b82885b18ea1afee193/evaluation/results.json",
      "source_sha256": "e18516a8a4331d10678f50f1de8c2a114d9747b0bb5698de80e354d86807c550",
      "card_url": "https://huggingface.co/d0rj/t5-moe-55M-base/blob/de3885b506c0106770b59b82885b18ea1afee193/README.md",
      "delta_pp": -3.048307952081281
    },
    {
      "repo": "d0rj/q-prefixlm-51M-base",
      "revision": "699e7db81a757772e5f83534805080b195e06ade",
      "label": "PrefixLM",
      "group": "prefix",
      "parameters": 50866688,
      "active_parameters": null,
      "optimized_parameters": null,
      "mean_16_percent": 39.01087477431766,
      "tasks": [
        {
          "id": "hellaswag",
          "label": "hellaswag",
          "metric": "acc_norm",
          "value": 0.2839075881298546,
          "stderr": 0.004499710284382373,
          "ci95": [
            0.2751726884679946,
            0.29280775221581823
          ],
          "samples": 10042,
          "dataset": "Rowan/hellaswag",
          "split": "validation"
        },
        {
          "id": "arc_easy",
          "label": "arc_easy",
          "metric": "acc_norm",
          "value": 0.36237373737373735,
          "stderr": 0.009863468202583702,
          "ci95": [
            0.34328224307333793,
            0.3819095349016189
          ],
          "samples": 2376,
          "dataset": "allenai/ai2_arc",
          "split": "test"
        },
        {
          "id": "arc_challenge",
          "label": "arc_challenge",
          "metric": "acc_norm",
          "value": 0.22781569965870307,
          "stderr": 0.012256708602326964,
          "ci95": [
            0.20471523586388393,
            0.25269460862324633
          ],
          "samples": 1172,
          "dataset": "allenai/ai2_arc",
          "split": "test"
        },
        {
          "id": "piqa",
          "label": "piqa",
          "metric": "acc_norm",
          "value": 0.5310119695321001,
          "stderr": 0.011643363511107561,
          "ci95": [
            0.5081566369012085,
            0.553737941187501
          ],
          "samples": 1838,
          "dataset": "baber/piqa",
          "split": "validation"
        },
        {
          "id": "winogrande",
          "label": "winogrande",
          "metric": "acc",
          "value": 0.4972375690607735,
          "stderr": 0.014052271211616401,
          "ci95": [
            0.4697564851267556,
            0.5247353533701599
          ],
          "samples": 1267,
          "dataset": "allenai/winogrande",
          "split": "validation"
        },
        {
          "id": "openbookqa",
          "label": "openbookqa",
          "metric": "acc_norm",
          "value": 0.256,
          "stderr": 0.019536923574747664,
          "ci95": [
            0.21970770819171886,
            0.2960129699535187
          ],
          "samples": 500,
          "dataset": "allenai/openbookqa",
          "split": "test"
        },
        {
          "id": "boolq",
          "label": "boolq",
          "metric": "acc",
          "value": 0.5486238532110091,
          "stderr": 0.008703605243425191,
          "ci95": [
            0.531520568853367,
            0.5656130290926221
          ],
          "samples": 3270,
          "dataset": "aps/super_glue",
          "split": "validation"
        },
        {
          "id": "lambada_openai",
          "label": "lambada_openai",
          "metric": "acc",
          "value": 0.2334562390840287,
          "stderr": 0.005893635758408312,
          "ci95": [
            0.2221071968527672,
            0.2452023914027916
          ],
          "samples": 5153,
          "dataset": "EleutherAI/lambada_openai",
          "split": "test"
        },
        {
          "id": "arithmark3",
          "label": "ArithMark-3",
          "metric": "acc_norm",
          "value": 0.308,
          "stderr": 0.01460648312734278,
          "ci95": [
            0.28016622636313016,
            0.33730324889532304
          ],
          "samples": 1000,
          "dataset": "AxiomicLabs/Arithmark-3.0",
          "split": "train"
        },
        {
          "id": "balanced_copa",
          "label": "Balanced COPA",
          "metric": "acc",
          "value": 0.529,
          "stderr": 0.015792669451628764,
          "ci95": [
            0.49801049370900147,
            0.5597675542988364
          ],
          "samples": 1000,
          "dataset": "pkavumba/balanced-copa",
          "split": "train"
        },
        {
          "id": "blimp",
          "label": "BLiMP",
          "metric": "acc",
          "value": 0.5564925373134327,
          "stderr": 0.0017968413884795636,
          "ci95": null,
          "samples": 67000,
          "dataset": "nyu-mll/blimp",
          "split": "train"
        },
        {
          "id": "commonsense_qa",
          "label": "CommonsenseQA",
          "metric": "acc",
          "value": 0.20065520065520065,
          "stderr": 0.011466011466011467,
          "ci95": [
            0.17914588148536167,
            0.2240421844184298
          ],
          "samples": 1221,
          "dataset": "tau/commonsense_qa",
          "split": "validation"
        },
        {
          "id": "mmlu_continuation",
          "label": "MMLU continuation",
          "metric": "acc",
          "value": 0.2469733656174334,
          "stderr": 0.0036304559083941684,
          "ci95": null,
          "samples": 14042,
          "dataset": "cais/mmlu",
          "split": "test"
        },
        {
          "id": "sciq",
          "label": "SciQ (with support)",
          "metric": "acc_norm",
          "value": 0.626,
          "stderr": 0.015308767369006505,
          "ci95": [
            0.5955817549893309,
            0.6554539018723092
          ],
          "samples": 1000,
          "dataset": "allenai/sciq",
          "split": "test"
        },
        {
          "id": "truthfulqa_mc2",
          "label": "TruthfulQA MC2",
          "metric": "acc",
          "value": 0.43133506139740835,
          "stderr": 0.015261660993259317,
          "ci95": null,
          "samples": 817,
          "dataset": "truthfulqa/truthful_qa",
          "split": "validation"
        },
        {
          "id": "bananamind_base",
          "label": "BananaMind Base 1.1",
          "metric": "raw_accuracy",
          "value": 0.40285714285714286,
          "stderr": 0.026254390562337908,
          "ci95": [
            0.3527964766499963,
            0.45502706040347723
          ],
          "samples": 350,
          "dataset": "BananaMind/BananaMind-Base-Bench-1.1",
          "split": "test"
        }
      ],
      "continuation": {
        "nll": 3.678740634950985,
        "nll_ci95": [
          3.6390307724475863,
          3.717708887943288
        ],
        "ppl": 39.596496215961274,
        "ppl_ci95": [
          38.05493495371844,
          41.1699609775064
        ]
      },
      "continuation_precision": "torch.float32",
      "source_url": "https://huggingface.co/d0rj/q-prefixlm-51M-base/blob/699e7db81a757772e5f83534805080b195e06ade/evaluation/results.json",
      "source_sha256": "198e60915347886952d76c0104323c65544d7a4674c395801b2b2bd6a40235c3",
      "card_url": "https://huggingface.co/d0rj/q-prefixlm-51M-base/blob/699e7db81a757772e5f83534805080b195e06ade/README.md",
      "delta_pp": -3.813281601356323
    },
    {
      "repo": "d0rj/diffusion-51M-base",
      "revision": "d532205e3ab906282f14404fb8b2641e756ad9bc",
      "label": "Diffusion v2",
      "group": "diffusion",
      "parameters": 51392512,
      "active_parameters": null,
      "optimized_parameters": 50867200,
      "mean_16_percent": 41.18161458433634,
      "tasks": [
        {
          "id": "hellaswag",
          "label": "hellaswag",
          "metric": "acc_norm",
          "value": 0.2795259908384784,
          "stderr": 0.004478491697891533,
          "ci95": [
            0.2708343265409323,
            0.2883862705438426
          ],
          "samples": 10042,
          "dataset": "Rowan/hellaswag",
          "split": "validation"
        },
        {
          "id": "arc_easy",
          "label": "arc_easy",
          "metric": "acc_norm",
          "value": 0.3362794612794613,
          "stderr": 0.0096941780727253,
          "ci95": [
            0.3175609903739864,
            0.3555264763913292
          ],
          "samples": 2376,
          "dataset": "allenai/ai2_arc",
          "split": "test"
        },
        {
          "id": "arc_challenge",
          "label": "arc_challenge",
          "metric": "acc_norm",
          "value": 0.2226962457337884,
          "stderr": 0.012158314774829888,
          "ci95": [
            0.19980419390295,
            0.2474001931137188
          ],
          "samples": 1172,
          "dataset": "allenai/ai2_arc",
          "split": "test"
        },
        {
          "id": "piqa",
          "label": "piqa",
          "metric": "acc_norm",
          "value": 0.5386289445048966,
          "stderr": 0.011630956681145891,
          "ci95": [
            0.5157819596226829,
            0.5613147955404465
          ],
          "samples": 1838,
          "dataset": "baber/piqa",
          "split": "validation"
        },
        {
          "id": "winogrande",
          "label": "winogrande",
          "metric": "acc",
          "value": 0.5011838989739542,
          "stderr": 0.014052446290528871,
          "ci95": [
            0.47369054474157507,
            0.5286700959026042
          ],
          "samples": 1267,
          "dataset": "allenai/winogrande",
          "split": "validation"
        },
        {
          "id": "openbookqa",
          "label": "openbookqa",
          "metric": "acc_norm",
          "value": 0.26,
          "stderr": 0.019635965529725526,
          "ci95": [
            0.22348572318538285,
            0.3001739602361623
          ],
          "samples": 500,
          "dataset": "allenai/openbookqa",
          "split": "test"
        },
        {
          "id": "boolq",
          "label": "boolq",
          "metric": "acc",
          "value": 0.5296636085626911,
          "stderr": 0.008729651343606093,
          "ci95": [
            0.5125316201633632,
            0.5467259836150076
          ],
          "samples": 3270,
          "dataset": "aps/super_glue",
          "split": "validation"
        },
        {
          "id": "lambada_openai",
          "label": "lambada_openai",
          "metric": "acc",
          "value": 0.42208422278284496,
          "stderr": 0.0068808787917354055,
          "ci95": [
            0.4086621968305708,
            0.4356223315164309
          ],
          "samples": 5153,
          "dataset": "EleutherAI/lambada_openai",
          "split": "test"
        },
        {
          "id": "arithmark3",
          "label": "ArithMark-3",
          "metric": "acc_norm",
          "value": 0.332,
          "stderr": 0.014899597242811565,
          "ci95": [
            0.30350363489239063,
            0.36178215595875596
          ],
          "samples": 1000,
          "dataset": "AxiomicLabs/Arithmark-3.0",
          "split": "train"
        },
        {
          "id": "balanced_copa",
          "label": "Balanced COPA",
          "metric": "acc",
          "value": 0.522,
          "stderr": 0.01580397942816194,
          "ci95": [
            0.4910152521271681,
            0.5528163704994674
          ],
          "samples": 1000,
          "dataset": "pkavumba/balanced-copa",
          "split": "train"
        },
        {
          "id": "blimp",
          "label": "BLiMP",
          "metric": "acc",
          "value": 0.6420746268656716,
          "stderr": 0.0016890890018278603,
          "ci95": null,
          "samples": 67000,
          "dataset": "nyu-mll/blimp",
          "split": "train"
        },
        {
          "id": "commonsense_qa",
          "label": "CommonsenseQA",
          "metric": "acc",
          "value": 0.21457821457821458,
          "stderr": 0.011753423094216953,
          "ci95": [
            0.19246524693116476,
            0.2384815135817527
          ],
          "samples": 1221,
          "dataset": "tau/commonsense_qa",
          "split": "validation"
        },
        {
          "id": "mmlu_continuation",
          "label": "MMLU continuation",
          "metric": "acc",
          "value": 0.24483691781797465,
          "stderr": 0.0036208715458871405,
          "ci95": null,
          "samples": 14042,
          "dataset": "cais/mmlu",
          "split": "test"
        },
        {
          "id": "sciq",
          "label": "SciQ (with support)",
          "metric": "acc_norm",
          "value": 0.688,
          "stderr": 0.014658474370509057,
          "ci95": [
            0.6586108249018996,
            0.7159503139075316
          ],
          "samples": 1000,
          "dataset": "allenai/sciq",
          "split": "test"
        },
        {
          "id": "truthfulqa_mc2",
          "label": "TruthfulQA MC2",
          "metric": "acc",
          "value": 0.45836334441298165,
          "stderr": 0.0158524239677764,
          "ci95": null,
          "samples": 817,
          "dataset": "truthfulqa/truthful_qa",
          "split": "validation"
        },
        {
          "id": "bananamind_base",
          "label": "BananaMind Base 1.1",
          "metric": "raw_accuracy",
          "value": 0.39714285714285713,
          "stderr": 0.02619195222772307,
          "ci95": [
            0.34726441786546686,
            0.4492546213676228
          ],
          "samples": 350,
          "dataset": "BananaMind/BananaMind-Base-Bench-1.1",
          "split": "test"
        }
      ],
      "continuation": {
        "nll": 2.764107143243973,
        "nll_ci95": [
          2.7211559490245905,
          2.8069273205348404
        ],
        "ppl": 15.864868609926912,
        "ppl_ci95": [
          15.197880069666004,
          16.558959594011032
        ]
      },
      "continuation_precision": "experimental_bfloat16_pll",
      "source_url": "https://huggingface.co/d0rj/diffusion-51M-base/blob/d532205e3ab906282f14404fb8b2641e756ad9bc/evaluation/results.json",
      "source_sha256": "0007587101804c2440ec82e8432011542c3755f3c9328b9172c5e429970e8c1f",
      "card_url": "https://huggingface.co/d0rj/diffusion-51M-base/blob/d532205e3ab906282f14404fb8b2641e756ad9bc/README.md",
      "delta_pp": null
    },
    {
      "repo": "d0rj/diffusion-51M-base-v1-failed",
      "revision": "d1bedd0c489281d521ef79accc0939dd9f05b58a",
      "label": "Diffusion v1 (failed)",
      "group": "diffusion",
      "parameters": 51392512,
      "active_parameters": null,
      "optimized_parameters": null,
      "mean_16_percent": 32.5377753172773,
      "tasks": [
        {
          "id": "hellaswag",
          "label": "HellaSwag",
          "metric": "acc_norm",
          "value": 0.2597092212706632,
          "stderr": 0.004375788991216637,
          "ci95": [
            0.251226292502746,
            0.26837592103183483
          ],
          "samples": 10042,
          "dataset": "Rowan/hellaswag",
          "split": "validation"
        },
        {
          "id": "arc_easy",
          "label": "ARC-Easy",
          "metric": "acc_norm",
          "value": 0.2668350168350168,
          "stderr": 0.009075915859266983,
          "ci95": [
            0.2494370337013486,
            0.28498573387701853
          ],
          "samples": 2376,
          "dataset": "allenai/ai2_arc",
          "split": "test"
        },
        {
          "id": "arc_challenge",
          "label": "ARC-Challenge",
          "metric": "acc_norm",
          "value": 0.24061433447098976,
          "stderr": 0.012491468532390682,
          "ci95": [
            0.21701467963390295,
            0.26590880852938104
          ],
          "samples": 1172,
          "dataset": "allenai/ai2_arc",
          "split": "test"
        },
        {
          "id": "piqa",
          "label": "PIQA",
          "metric": "acc_norm",
          "value": 0.500544069640914,
          "stderr": 0.011665817258899291,
          "ci95": [
            0.4777084238482688,
            0.523377445942761
          ],
          "samples": 1838,
          "dataset": "baber/piqa",
          "split": "validation"
        },
        {
          "id": "winogrande",
          "label": "WinoGrande",
          "metric": "acc",
          "value": 0.5098658247829518,
          "stderr": 0.014049749833367674,
          "ci95": [
            0.4823514860396483,
            0.5373205193285109
          ],
          "samples": 1267,
          "dataset": "allenai/winogrande",
          "split": "validation"
        },
        {
          "id": "openbookqa",
          "label": "OpenBookQA",
          "metric": "acc_norm",
          "value": 0.258,
          "stderr": 0.019586711785215868,
          "ci95": [
            0.2215962002228131,
            0.29809398056057823
          ],
          "samples": 500,
          "dataset": "allenai/openbookqa",
          "split": "test"
        },
        {
          "id": "boolq",
          "label": "BoolQ",
          "metric": "acc",
          "value": 0.3782874617737003,
          "stderr": 0.008482001133931097,
          "ci95": [
            0.36181754306445063,
            0.3950430105047085
          ],
          "samples": 3270,
          "dataset": "aps/super_glue",
          "split": "validation"
        },
        {
          "id": "lambada_openai",
          "label": "LAMBADA OpenAI",
          "metric": "acc",
          "value": 0.0,
          "stderr": 0.0,
          "ci95": [
            0,
            0.0007449247473224859
          ],
          "samples": 5153,
          "dataset": "EleutherAI/lambada_openai",
          "split": "test"
        },
        {
          "id": "arithmark3",
          "label": "ArithMark-3",
          "metric": "acc_norm",
          "value": 0.299,
          "stderr": 0.014484778521220433,
          "ci95": [
            0.27143763442230356,
            0.3281007224888896
          ],
          "samples": 1000,
          "dataset": "AxiomicLabs/Arithmark-3.0",
          "split": "train"
        },
        {
          "id": "balanced_copa",
          "label": "Balanced COPA",
          "metric": "acc",
          "value": 0.5,
          "stderr": 0.015819299929208316,
          "ci95": [
            0.4690696003681042,
            0.5309303996318958
          ],
          "samples": 1000,
          "dataset": "pkavumba/balanced-copa",
          "split": "train"
        },
        {
          "id": "blimp",
          "label": "BLiMP",
          "metric": "acc",
          "value": 0.573865671641791,
          "stderr": 0.0015987875104430066,
          "ci95": null,
          "samples": 67000,
          "dataset": "nyu-mll/blimp",
          "split": "train"
        },
        {
          "id": "commonsense_qa",
          "label": "CommonsenseQA",
          "metric": "acc",
          "value": 0.19574119574119575,
          "stderr": 0.011359497363584397,
          "ci95": [
            0.17445479920495016,
            0.21893608036620865
          ],
          "samples": 1221,
          "dataset": "tau/commonsense_qa",
          "split": "validation"
        },
        {
          "id": "mmlu_continuation",
          "label": "MMLU continuation",
          "metric": "acc",
          "value": 0.2337986041874377,
          "stderr": 0.003566341746171636,
          "ci95": null,
          "samples": 14042,
          "dataset": "cais/mmlu",
          "split": "test"
        },
        {
          "id": "sciq",
          "label": "SciQ (with support)",
          "metric": "acc_norm",
          "value": 0.237,
          "stderr": 0.013454070462577935,
          "ci95": [
            0.21168136426041648,
            0.2643315107029855
          ],
          "samples": 1000,
          "dataset": "allenai/sciq",
          "split": "test"
        },
        {
          "id": "truthfulqa_mc2",
          "label": "TruthfulQA MC2",
          "metric": "acc",
          "value": 0.48992550756256453,
          "stderr": 0.01640276307124247,
          "ci95": null,
          "samples": 817,
          "dataset": "truthfulqa/truthful_qa",
          "split": "validation"
        },
        {
          "id": "bananamind_base",
          "label": "BananaMind Base 1.1",
          "metric": "raw_accuracy",
          "value": 0.26285714285714284,
          "stderr": 0.023562589046968913,
          "ci95": [
            0.21949467430879996,
            0.3113686661452677
          ],
          "samples": 350,
          "dataset": "BananaMind/BananaMind-Base-Bench-1.1",
          "split": "test"
        }
      ],
      "continuation": {
        "nll": 8.238185963270181,
        "nll_ci95": [
          8.214774839260016,
          8.26161171348311
        ],
        "ppl": 3782.6721742694644,
        "ppl_ci95": [
          3695.144129529085,
          3872.330160490858
        ]
      },
      "continuation_precision": "experimental_bfloat16_pll",
      "source_url": "https://huggingface.co/d0rj/diffusion-51M-base-v1-failed/blob/d1bedd0c489281d521ef79accc0939dd9f05b58a/evaluation/results.json",
      "source_sha256": "d0c954070650926c931ba95e5e8994138711447f74a09c7132331c05c5c21447",
      "card_url": "https://huggingface.co/d0rj/diffusion-51M-base-v1-failed/blob/d1bedd0c489281d521ef79accc0939dd9f05b58a/README.md",
      "delta_pp": null
    }
  ]
}