{
  "benchmark": {
    "id": "small-models-for-glam/glam-extraction-benchmark",
    "revision": "ecc9c02582f933ca89b73e6751e4ed76888cb24d",
    "config": "nls-index-cards",
    "split": "test"
  },
  "harness_version": "0.0.1",
  "scorer_version": "2026-08-26a",
  "metrics": {
    "identifiers_wrong": {
      "label": "Identifiers wrong",
      "direction": "lower",
      "scale": [
        0,
        1
      ]
    },
    "invented_fields": {
      "label": "Invented fields",
      "direction": "lower",
      "scale": [
        0,
        1
      ]
    },
    "fields_left_blank": {
      "label": "Fields left blank",
      "direction": "lower",
      "scale": [
        0,
        1
      ]
    },
    "content_f1": {
      "label": "Fields right (F1)",
      "direction": "higher",
      "scale": [
        0,
        1
      ]
    }
  },
  "rows": [
    {
      "label": "Qwen3.8-27B",
      "params": "27.78B",
      "n": 98,
      "attested": false,
      "fields_left_blank": 0.19205909510618652,
      "identifiers_wrong": 0.12738853503184713,
      "invented_fields": 0.013513513513513514,
      "content_f1": 0.7952887755102038,
      "precision": 0.8754734693877548,
      "recall": 0.7387948979591836,
      "exact_fields_f1": 0.8904622448979591,
      "fuzzy_fields_f1": 0.5586000000000001,
      "schema_valid": 1.0,
      "identifier_clean_cards": 59,
      "verified_f1": 0.8113999999999997,
      "corrected_f1": 0.7620593750000001,
      "n_verified": 66,
      "n_corrected": 32,
      "n_ident_filled": 157,
      "model_id": "Qwen/Qwen3.8-27B"
    },
    {
      "label": "GLM-5.3-Flash",
      "params": "321.32B",
      "n": 98,
      "attested": false,
      "fields_left_blank": 0.13019390581717452,
      "identifiers_wrong": 0.19889502762430938,
      "invented_fields": 0.04054054054054054,
      "content_f1": 0.8314306122448978,
      "precision": 0.8770438775510204,
      "recall": 0.7991622448979594,
      "exact_fields_f1": 0.9005040816326528,
      "fuzzy_fields_f1": 0.6626724489795915,
      "schema_valid": 1.0,
      "identifier_clean_cards": 62,
      "verified_f1": 0.8536757575757575,
      "corrected_f1": 0.78555,
      "n_verified": 66,
      "n_corrected": 32,
      "n_ident_filled": 181,
      "model_id": "zai-org/GLM-5.3-Flash"
    },
    {
      "label": "Qwen3.5-9B",
      "params": "9.65B",
      "n": 98,
      "attested": false,
      "fields_left_blank": 0.26685133887349954,
      "identifiers_wrong": 0.2716049382716049,
      "invented_fields": 0.08444444444444445,
      "content_f1": 0.7052765306122453,
      "precision": 0.8235295918367347,
      "recall": 0.6327897959183674,
      "exact_fields_f1": 0.8051642857142858,
      "fuzzy_fields_f1": 0.4738122448979591,
      "schema_valid": 1.0,
      "identifier_clean_cards": 26,
      "verified_f1": 0.7290469696969699,
      "corrected_f1": 0.65625,
      "n_verified": 66,
      "n_corrected": 32,
      "n_ident_filled": 81,
      "model_id": "Qwen/Qwen3.5-9B"
    },
    {
      "label": "Qwen3-VL-8B",
      "params": "8.767B",
      "n": 98,
      "attested": true,
      "fields_left_blank": 0.22160664819944598,
      "identifiers_wrong": 0.27205882352941174,
      "invented_fields": 0.15040650406504066,
      "content_f1": 0.748754081632653,
      "precision": 0.8191775510204082,
      "recall": 0.7023551020408163,
      "exact_fields_f1": 0.8255816326530615,
      "fuzzy_fields_f1": 0.5637908163265304,
      "schema_valid": 1.0,
      "identifier_clean_cards": 48,
      "verified_f1": 0.7876969696969697,
      "corrected_f1": 0.6684343749999998,
      "n_verified": 66,
      "n_corrected": 32,
      "n_ident_filled": 136,
      "model_id": "Qwen/Qwen3-VL-8B-Instruct"
    },
    {
      "label": "NuExtract-3",
      "params": "4.539B",
      "n": 98,
      "attested": true,
      "fields_left_blank": 0.14496768236380425,
      "identifiers_wrong": 0.28289473684210525,
      "invented_fields": 0.12941176470588237,
      "content_f1": 0.7345663265306124,
      "precision": 0.7686163265306126,
      "recall": 0.7146030612244898,
      "exact_fields_f1": 0.7829704081632655,
      "fuzzy_fields_f1": 0.6256785714285714,
      "schema_valid": 1.0,
      "identifier_clean_cards": 55,
      "verified_f1": 0.7519757575757579,
      "corrected_f1": 0.6986593750000001,
      "n_verified": 66,
      "n_corrected": 32,
      "n_ident_filled": 152,
      "model_id": "numind/NuExtract3"
    },
    {
      "label": "Qwen3-VL-235B",
      "params": "235.67B",
      "n": 98,
      "attested": false,
      "fields_left_blank": 0.1061865189289012,
      "identifiers_wrong": 0.2903225806451613,
      "invented_fields": 0.23371647509578544,
      "content_f1": 0.7779806122448982,
      "precision": 0.8052173469387758,
      "recall": 0.7635326530612245,
      "exact_fields_f1": 0.8339346938775511,
      "fuzzy_fields_f1": 0.6450295918367347,
      "schema_valid": 1.0,
      "identifier_clean_cards": 40,
      "verified_f1": 0.7814560606060604,
      "corrected_f1": 0.7708125,
      "n_verified": 66,
      "n_corrected": 32,
      "n_ident_filled": 186,
      "model_id": "Qwen/Qwen3-VL-235B-A22B-Instruct"
    },
    {
      "label": "gemma-4-E4B",
      "params": "7.996B",
      "n": 98,
      "attested": true,
      "fields_left_blank": 0.4358264081255771,
      "identifiers_wrong": 0.34210526315789475,
      "invented_fields": 0.1188118811881188,
      "content_f1": 0.5765693877551019,
      "precision": 0.7598336734693878,
      "recall": 0.4735979591836734,
      "exact_fields_f1": 0.6733571428571427,
      "fuzzy_fields_f1": 0.3621132653061224,
      "schema_valid": 1.0,
      "identifier_clean_cards": 16,
      "verified_f1": 0.6000121212121212,
      "corrected_f1": 0.5282187500000001,
      "n_verified": 66,
      "n_corrected": 32,
      "n_ident_filled": 38,
      "model_id": "google/gemma-4-E4B-it"
    },
    {
      "label": "Qwen3.5-4B",
      "params": "4.660B",
      "n": 98,
      "attested": true,
      "fields_left_blank": 0.24930747922437674,
      "identifiers_wrong": 0.3739130434782609,
      "invented_fields": 0.056910569105691054,
      "content_f1": 0.7168418367346939,
      "precision": 0.8098377551020405,
      "recall": 0.6573040816326529,
      "exact_fields_f1": 0.8083877551020411,
      "fuzzy_fields_f1": 0.48691326530612256,
      "schema_valid": 1.0,
      "identifier_clean_cards": 35,
      "verified_f1": 0.7495696969696971,
      "corrected_f1": 0.649340625,
      "n_verified": 66,
      "n_corrected": 32,
      "n_ident_filled": 115,
      "model_id": "Qwen/Qwen3.5-4B"
    },
    {
      "label": "Qwen3.5-2B",
      "params": "2.274B",
      "n": 98,
      "attested": true,
      "fields_left_blank": 0.47830101569713757,
      "identifiers_wrong": 0.4827586206896552,
      "invented_fields": 0.0730593607305936,
      "content_f1": 0.4495744897959182,
      "precision": 0.5868938775510205,
      "recall": 0.3711132653061223,
      "exact_fields_f1": 0.5333755102040815,
      "fuzzy_fields_f1": 0.2302867346938776,
      "schema_valid": 0.8673469387755102,
      "identifier_clean_cards": 13,
      "verified_f1": 0.4665818181818182,
      "corrected_f1": 0.414496875,
      "n_verified": 66,
      "n_corrected": 32,
      "n_ident_filled": 29,
      "model_id": "Qwen/Qwen3.5-2B"
    },
    {
      "label": "Granite-Vision-4.1-4B",
      "params": "3.997B",
      "n": 98,
      "attested": true,
      "fields_left_blank": 0.32132963988919666,
      "identifiers_wrong": 0.8611111111111112,
      "invented_fields": 0.19166666666666668,
      "content_f1": 0.6258622448979592,
      "precision": 0.7269724489795919,
      "recall": 0.5588693877551022,
      "exact_fields_f1": 0.6863438775510203,
      "fuzzy_fields_f1": 0.47307755102040805,
      "schema_valid": 0.9795918367346939,
      "identifier_clean_cards": 10,
      "verified_f1": 0.6488409090909092,
      "corrected_f1": 0.57846875,
      "n_verified": 66,
      "n_corrected": 32,
      "n_ident_filled": 108,
      "model_id": "ibm-granite/granite-vision-4.1-4b"
    }
  ],
  "prediction_files": [
    {
      "file": "GLM-5.3-Flash.json",
      "model": {
        "label": "GLM-5.3-Flash",
        "kind": "router_vlm",
        "id": "zai-org/GLM-5.3-Flash",
        "params": "321.32B",
        "provider": "deepinfra"
      },
      "inference_dataset": {
        "id": "small-models-for-glam/glam-extraction-benchmark",
        "inference_revision": "ecc9c02582f933ca89b73e6751e4ed76888cb24d",
        "config": "nls-index-cards",
        "split": "test",
        "item_count": 98,
        "item_ids_sha256": "02191bd192c560538246697663ea4e144981dabebde452c632c8c31bbc62e369"
      },
      "legacy_protocol": false
    },
    {
      "file": "Granite-Vision-4.1-4B.json",
      "model": {
        "label": "Granite-Vision-4.1-4B",
        "kind": "openai",
        "id": "ibm-granite/granite-vision-4.1-4b",
        "params": "3.997B",
        "cost": "HF Jobs l40sx1 $1.80/hour"
      },
      "inference_dataset": {
        "id": "small-models-for-glam/glam-extraction-benchmark",
        "alias": null,
        "requested_revision": "ecc9c02582f933ca89b73e6751e4ed76888cb24d",
        "inference_revision": "ecc9c02582f933ca89b73e6751e4ed76888cb24d",
        "split": "test",
        "item_count": 98,
        "item_ids_sha256": "02191bd192c560538246697663ea4e144981dabebde452c632c8c31bbc62e369",
        "config": "nls-index-cards",
        "task_instructions": "Extract the fields described by target_schema from the supplied image. Return a JSON object. Use null or omit optional fields when absent."
      },
      "legacy_protocol": false
    },
    {
      "file": "NuExtract-3.json",
      "model": {
        "label": "NuExtract-3",
        "kind": "openai",
        "id": "numind/NuExtract3",
        "params": "4.539B",
        "cost": "HF Jobs l40sx1 $1.80/hour"
      },
      "inference_dataset": {
        "id": "small-models-for-glam/glam-extraction-benchmark",
        "alias": null,
        "requested_revision": "ecc9c02582f933ca89b73e6751e4ed76888cb24d",
        "inference_revision": "ecc9c02582f933ca89b73e6751e4ed76888cb24d",
        "split": "test",
        "item_count": 98,
        "item_ids_sha256": "02191bd192c560538246697663ea4e144981dabebde452c632c8c31bbc62e369",
        "config": "nls-index-cards",
        "task_instructions": "Extract the fields described by target_schema from the supplied image. Return a JSON object. Use null or omit optional fields when absent."
      },
      "legacy_protocol": false
    },
    {
      "file": "Qwen3-VL-235B.json",
      "model": {
        "label": "Qwen3-VL-235B",
        "kind": "router_vlm",
        "id": "Qwen/Qwen3-VL-235B-A22B-Instruct",
        "params": "235.67B",
        "provider": "deepinfra"
      },
      "inference_dataset": {
        "id": "small-models-for-glam/glam-extraction-benchmark",
        "inference_revision": "ecc9c02582f933ca89b73e6751e4ed76888cb24d",
        "config": "nls-index-cards",
        "split": "test",
        "item_count": 98,
        "item_ids_sha256": "02191bd192c560538246697663ea4e144981dabebde452c632c8c31bbc62e369"
      },
      "legacy_protocol": false
    },
    {
      "file": "Qwen3-VL-8B.json",
      "model": {
        "label": "Qwen3-VL-8B",
        "kind": "openai",
        "id": "Qwen/Qwen3-VL-8B-Instruct",
        "params": "8.767B",
        "cost": "HF Jobs l40sx1 $1.80/hour"
      },
      "inference_dataset": {
        "id": "small-models-for-glam/glam-extraction-benchmark",
        "alias": null,
        "requested_revision": "ecc9c02582f933ca89b73e6751e4ed76888cb24d",
        "inference_revision": "ecc9c02582f933ca89b73e6751e4ed76888cb24d",
        "split": "test",
        "item_count": 98,
        "item_ids_sha256": "02191bd192c560538246697663ea4e144981dabebde452c632c8c31bbc62e369",
        "config": "nls-index-cards",
        "task_instructions": "Extract the fields described by target_schema from the supplied image. Return a JSON object. Use null or omit optional fields when absent."
      },
      "legacy_protocol": false
    },
    {
      "file": "Qwen3.5-2B.json",
      "model": {
        "label": "Qwen3.5-2B",
        "kind": "openai",
        "id": "Qwen/Qwen3.5-2B",
        "params": "2.274B",
        "cost": "HF Jobs l4x1 $0.80/hour"
      },
      "inference_dataset": {
        "id": "small-models-for-glam/glam-extraction-benchmark",
        "alias": null,
        "requested_revision": "ecc9c02582f933ca89b73e6751e4ed76888cb24d",
        "inference_revision": "ecc9c02582f933ca89b73e6751e4ed76888cb24d",
        "split": "test",
        "item_count": 98,
        "item_ids_sha256": "02191bd192c560538246697663ea4e144981dabebde452c632c8c31bbc62e369",
        "config": "nls-index-cards",
        "task_instructions": "Extract the fields described by target_schema from the supplied image. Return a JSON object. Use null or omit optional fields when absent."
      },
      "legacy_protocol": false
    },
    {
      "file": "Qwen3.5-4B.json",
      "model": {
        "label": "Qwen3.5-4B",
        "kind": "openai",
        "id": "Qwen/Qwen3.5-4B",
        "params": "4.660B",
        "cost": "HF Jobs l40sx1 $1.80/hour"
      },
      "inference_dataset": {
        "id": "small-models-for-glam/glam-extraction-benchmark",
        "alias": null,
        "requested_revision": "ecc9c02582f933ca89b73e6751e4ed76888cb24d",
        "inference_revision": "ecc9c02582f933ca89b73e6751e4ed76888cb24d",
        "split": "test",
        "item_count": 98,
        "item_ids_sha256": "02191bd192c560538246697663ea4e144981dabebde452c632c8c31bbc62e369",
        "config": "nls-index-cards",
        "task_instructions": "Extract the fields described by target_schema from the supplied image. Return a JSON object. Use null or omit optional fields when absent."
      },
      "legacy_protocol": false
    },
    {
      "file": "Qwen3.5-9B.json",
      "model": {
        "label": "Qwen3.5-9B",
        "kind": "router_vlm",
        "id": "Qwen/Qwen3.5-9B",
        "params": "9.65B",
        "provider": "deepinfra"
      },
      "inference_dataset": {
        "id": "small-models-for-glam/glam-extraction-benchmark",
        "inference_revision": "ecc9c02582f933ca89b73e6751e4ed76888cb24d",
        "config": "nls-index-cards",
        "split": "test",
        "item_count": 98,
        "item_ids_sha256": "02191bd192c560538246697663ea4e144981dabebde452c632c8c31bbc62e369"
      },
      "legacy_protocol": false
    },
    {
      "file": "Qwen3.8-27B.json",
      "model": {
        "label": "Qwen3.8-27B",
        "kind": "router_vlm",
        "id": "Qwen/Qwen3.8-27B",
        "params": "27.78B",
        "provider": "novita"
      },
      "inference_dataset": {
        "id": "small-models-for-glam/glam-extraction-benchmark",
        "inference_revision": "ecc9c02582f933ca89b73e6751e4ed76888cb24d",
        "config": "nls-index-cards",
        "split": "test",
        "item_count": 98,
        "item_ids_sha256": "02191bd192c560538246697663ea4e144981dabebde452c632c8c31bbc62e369"
      },
      "legacy_protocol": false
    },
    {
      "file": "gemma-4-E4B.json",
      "model": {
        "label": "gemma-4-E4B",
        "kind": "openai",
        "id": "google/gemma-4-E4B-it",
        "params": "7.996B",
        "cost": "HF Jobs l40sx1 $1.80/hour"
      },
      "inference_dataset": {
        "id": "small-models-for-glam/glam-extraction-benchmark",
        "alias": null,
        "requested_revision": "ecc9c02582f933ca89b73e6751e4ed76888cb24d",
        "inference_revision": "ecc9c02582f933ca89b73e6751e4ed76888cb24d",
        "split": "test",
        "item_count": 98,
        "item_ids_sha256": "02191bd192c560538246697663ea4e144981dabebde452c632c8c31bbc62e369",
        "config": "nls-index-cards",
        "task_instructions": "Extract the fields described by target_schema from the supplied image. Return a JSON object. Use null or omit optional fields when absent."
      },
      "legacy_protocol": false
    }
  ],
  "legacy_protocol": false
}