{
  "schemaVersion": 1,
  "generatedAt": "2026-06-11T19:44:29+00:00",
  "source": {
    "manifest": "outputs/manifest.json",
    "grades": "outputs/grades.json",
    "aiGrades": "outputs/ai_grades.json"
  },
  "summary": {
    "totalCells": 8400,
    "okImages": 8396,
    "errors": 4,
    "distinctModels": 40,
    "humanLabeled": 1055,
    "humanUnlabeled": 7341,
    "humanLabelMix": {
      "correct": 436,
      "dotted": 269,
      "substituted": 218,
      "mangled": 72,
      "offtask": 60
    },
    "aiLabeledByJudge": {
      "gemini-3-5-flash": 8396,
      "claude-sonnet": 8390,
      "gemini-3-pro": 1055,
      "gpt-5-5": 1036,
      "claude-opus": 1045,
      "qwen-vl-7b": 476
    },
    "costUsd": {
      "generation": 342.490066,
      "readback": 4.3642,
      "ai": 188.203001,
      "total": 535.057267
    }
  },
  "dimensions": {
    "prompts": [
      {
        "key": "sign",
        "labelKey": "prompt.sign.name",
        "groupKey": "prompt.sign.group",
        "descriptionKey": "prompt.sign.description",
        "label": "Studio sign",
        "group": "Plain sign",
        "description": "A plain English request for a studio sign. This is the control scene.",
        "template": "a clean studio photograph of a sign that reads \"{word}\"",
        "example": "a clean studio photograph of a sign that reads \"Fırat\"",
        "messageKey": "prompt.sign",
        "language": "English",
        "languageKey": "prompt.language.english",
        "coached": "no",
        "coachedKey": "prompt.coached.no",
        "scene": "sign"
      },
      {
        "key": "serif",
        "labelKey": "prompt.serif.name",
        "groupKey": "prompt.serif.group",
        "descriptionKey": "prompt.serif.description",
        "label": "Serif poster",
        "group": "Typography",
        "description": "A typography-first request: centered serif text on a white background.",
        "template": "the text \"{word}\" in an elegant serif typeface, centered, plain white background",
        "example": "the text \"Fırat\" in an elegant serif typeface, centered, plain white background",
        "messageKey": "prompt.serif",
        "language": "English",
        "languageKey": "prompt.language.english",
        "coached": "no",
        "coachedKey": "prompt.coached.no",
        "scene": "serif"
      },
      {
        "key": "handwritten",
        "labelKey": "prompt.handwritten.name",
        "groupKey": "prompt.handwritten.group",
        "descriptionKey": "prompt.handwritten.description",
        "label": "Handwritten note",
        "group": "Handwriting",
        "description": "A cursive note on paper, where the dotless ı can be confused with a casual i.",
        "template": "a handwritten note on paper that says \"{word}\" in cursive ink",
        "example": "a handwritten note on paper that says \"Fırat\" in cursive ink",
        "messageKey": "prompt.handwritten",
        "language": "English",
        "languageKey": "prompt.language.english",
        "coached": "no",
        "coachedKey": "prompt.coached.no",
        "scene": "handwritten"
      },
      {
        "key": "ledmatrix",
        "labelKey": "prompt.ledmatrix.name",
        "groupKey": "prompt.ledmatrix.group",
        "descriptionKey": "prompt.ledmatrix.description",
        "label": "LED matrix",
        "group": "Display",
        "description": "A red dot-matrix display, where the bug becomes a literal extra lit pixel.",
        "template": "a red LED dot-matrix display showing the text \"{word}\"",
        "example": "a red LED dot-matrix display showing the text \"Fırat\"",
        "messageKey": "prompt.ledmatrix",
        "language": "English",
        "languageKey": "prompt.language.english",
        "coached": "no",
        "coachedKey": "prompt.coached.no",
        "scene": "ledmatrix"
      },
      {
        "key": "sign-tr",
        "labelKey": "prompt.sign_tr.name",
        "groupKey": "prompt.sign_tr.group",
        "descriptionKey": "prompt.sign_tr.description",
        "label": "Turkish sign prompt",
        "group": "Instruction language",
        "description": "The same studio-sign scene, but the instruction itself is written in Turkish.",
        "template": "üzerinde \"{word}\" yazan bir tabelanın temiz bir stüdyo fotoğrafı",
        "example": "üzerinde \"Fırat\" yazan bir tabelanın temiz bir stüdyo fotoğrafı",
        "messageKey": "prompt.sign_tr",
        "language": "Turkish",
        "languageKey": "prompt.language.turkish",
        "coached": "no",
        "coachedKey": "prompt.coached.no",
        "scene": "sign"
      },
      {
        "key": "sign-coach",
        "labelKey": "prompt.sign_coach.name",
        "groupKey": "prompt.sign_coach.group",
        "descriptionKey": "prompt.sign_coach.description",
        "label": "Coached sign prompt",
        "group": "Glyph coaching",
        "description": "The English sign prompt plus an explicit reminder about dotless ı and Turkish marks.",
        "template": "a clean studio photograph of a sign that reads \"{word}\". The text is Turkish — render every letter exactly: the dotless letter ı must have NO dot, and keep the marks on ç ğ ö ş ü.",
        "example": "a clean studio photograph of a sign that reads \"Fırat\". The text is Turkish — render every letter exactly: the dotless letter ı must have NO dot, and keep the marks on ç ğ ö ş ü.",
        "messageKey": "prompt.sign_coach",
        "language": "English",
        "languageKey": "prompt.language.english",
        "coached": "yes",
        "coachedKey": "prompt.coached.yes",
        "scene": "sign"
      },
      {
        "key": "sign-trcoach",
        "labelKey": "prompt.sign_trcoach.name",
        "groupKey": "prompt.sign_trcoach.group",
        "descriptionKey": "prompt.sign_trcoach.description",
        "label": "Turkish coached sign prompt",
        "group": "Language + coaching",
        "description": "The Turkish sign prompt plus the same explicit glyph reminder, also in Turkish.",
        "template": "üzerinde \"{word}\" yazan bir tabelanın temiz bir stüdyo fotoğrafı. Metin Türkçedir — her harfi tam olarak çiz: noktasız ı harfinde nokta olmamalı, ç ğ ö ş ü harflerinin işaretlerini koru.",
        "example": "üzerinde \"Fırat\" yazan bir tabelanın temiz bir stüdyo fotoğrafı. Metin Türkçedir — her harfi tam olarak çiz: noktasız ı harfinde nokta olmamalı, ç ğ ö ş ü harflerinin işaretlerini koru.",
        "messageKey": "prompt.sign_trcoach",
        "language": "Turkish",
        "languageKey": "prompt.language.turkish",
        "coached": "yes",
        "coachedKey": "prompt.coached.yes",
        "scene": "sign"
      }
    ],
    "words": [
      {
        "slug": "firat",
        "text": "Fırat",
        "translation": "a Turkish given name, and the Euphrates River; one stray dot misspells both a person and a river",
        "translationKey": "word.firat.gloss",
        "dotlessI": true
      },
      {
        "slug": "isil",
        "text": "Işıl",
        "translation": "radiant; a given name whose capital I is correct only without a dot",
        "translationKey": "word.isil.gloss",
        "dotlessI": true
      },
      {
        "slug": "isik",
        "text": "ışık",
        "translation": "light; two dotless ı flanking a ş, three chances for stylish lettering to slip",
        "translationKey": "word.isik.gloss",
        "dotlessI": true
      },
      {
        "slug": "ciglik",
        "text": "çığlık",
        "translation": "scream; ç, ğ, and two dotless ı packed into six letters",
        "translationKey": "word.ciglik.gloss",
        "dotlessI": true
      },
      {
        "slug": "kadikoy",
        "text": "Kadıköy",
        "translation": "a district in Istanbul; the ö usually survives, the dotless ı beside it often does not",
        "translationKey": "word.kadikoy.gloss",
        "dotlessI": true
      },
      {
        "slug": "istanbul",
        "text": "İstanbul",
        "translation": "Istanbul; the reverse trap, this capital İ must keep its dot",
        "translationKey": "word.istanbul.gloss",
        "dotlessI": false
      },
      {
        "slug": "iki",
        "text": "iki",
        "translation": "two; the control word, every i is dotted and simply has to stay that way",
        "translationKey": "word.iki.gloss",
        "dotlessI": false
      },
      {
        "slug": "adabimuaseret",
        "text": "adâbımuâşeret",
        "translation": "manners, etiquette; a long Ottoman mouthful where the â, the dotless ı, and the ş all have to survive",
        "translationKey": "word.adabimuaseret.gloss",
        "dotlessI": true
      },
      {
        "slug": "kiragi",
        "text": "kırağı",
        "translation": "hoarfrost; rare enough that models guess at it, and the ğ and two dotless ı punish guessing",
        "translationKey": "word.kiragi.gloss",
        "dotlessI": true
      },
      {
        "slug": "nevi-sahsina-munhasir",
        "text": "Nevi Şahsına Münhasır",
        "translation": "one of a kind; an old idiom that asks for ş, ü, and two dotless ı at once",
        "translationKey": "word.nevi.gloss",
        "dotlessI": true
      },
      {
        "slug": "kirk-kup",
        "text": "Kırk küp, kırkının da kulpu kırık küp.",
        "translation": "forty jars, all with broken handles; a tongue-twister with six dotless ı in one breath",
        "translationKey": "word.kirk_kup.gloss",
        "dotlessI": true
      },
      {
        "slug": "istanbulu-dinliyorum",
        "text": "İstanbul'u dinliyorum, gözlerim kapalı",
        "translation": "I listen to Istanbul, my eyes closed (Orhan Veli); a dotted İ, a dotless ı, and an ö share one line",
        "translationKey": "word.istanbul_poem.gloss",
        "dotlessI": true
      },
      {
        "slug": "yasamak",
        "text": "Yaşamak bir ağaç gibi tek ve hür",
        "translation": "to live like a tree, alone and free (Nazım Hikmet); no dotless ı at all, just ç, ğ, ş, and ü to keep alive",
        "translationKey": "word.yasamak.gloss",
        "dotlessI": false
      },
      {
        "slug": "kiz-dotless",
        "text": "kız",
        "translation": "girl; the minimal pair, one added dot and it stops being a Turkish word",
        "translationKey": "word.kiz.gloss",
        "dotlessI": true
      },
      {
        "slug": "four-i",
        "text": "ı i İ I",
        "translation": "the four i's side by side, dotless and dotted; the bluntest test of where the dot belongs",
        "translationKey": "word.stress_i.gloss",
        "dotlessI": true
      }
    ],
    "labels": [
      "correct",
      "dotted",
      "substituted",
      "mangled",
      "offtask"
    ]
  },
  "methodology": {
    "grid": "15 words x 40 models x 7 prompt contexts x 2 seeds",
    "groundTruth": "Every image gets one of five labels. Correct: every requested glyph survives. Dotted: the dotless ı gained a dot it never asked for. Substituted: a real but wrong letter took its place. Mangled: the text collapsed into shapes that are no longer letters. Off task: the model drew something else instead of writing the text. Dotted is only meaningful for targets that actually carry the dotless ı. Human labels are the ground truth, and all 1,055 of them come from one reviewer: me, Fırat Gelbal, working image by image in the project's own labeling tool. Most models in the main production grid carry 53 human labels each, a few carry 23, and the 15 later backfill models carry 20 each. The Observed line on a specimen card is what the human reviewer read in the image; where no human has looked yet, a vision-model transcription stands in.",
    "groundTruthKey": "method.how_labels.copy",
    "labels": [
      "correct",
      "dotted",
      "substituted",
      "mangled",
      "offtask"
    ],
    "promptTemplates": [
      {
        "key": "sign",
        "labelKey": "prompt.sign.name",
        "groupKey": "prompt.sign.group",
        "descriptionKey": "prompt.sign.description",
        "label": "Studio sign",
        "group": "Plain sign",
        "description": "A plain English request for a studio sign. This is the control scene.",
        "template": "a clean studio photograph of a sign that reads \"{word}\"",
        "example": "a clean studio photograph of a sign that reads \"Fırat\"",
        "messageKey": "prompt.sign",
        "language": "English",
        "languageKey": "prompt.language.english",
        "coached": "no",
        "coachedKey": "prompt.coached.no",
        "scene": "sign"
      },
      {
        "key": "serif",
        "labelKey": "prompt.serif.name",
        "groupKey": "prompt.serif.group",
        "descriptionKey": "prompt.serif.description",
        "label": "Serif poster",
        "group": "Typography",
        "description": "A typography-first request: centered serif text on a white background.",
        "template": "the text \"{word}\" in an elegant serif typeface, centered, plain white background",
        "example": "the text \"Fırat\" in an elegant serif typeface, centered, plain white background",
        "messageKey": "prompt.serif",
        "language": "English",
        "languageKey": "prompt.language.english",
        "coached": "no",
        "coachedKey": "prompt.coached.no",
        "scene": "serif"
      },
      {
        "key": "handwritten",
        "labelKey": "prompt.handwritten.name",
        "groupKey": "prompt.handwritten.group",
        "descriptionKey": "prompt.handwritten.description",
        "label": "Handwritten note",
        "group": "Handwriting",
        "description": "A cursive note on paper, where the dotless ı can be confused with a casual i.",
        "template": "a handwritten note on paper that says \"{word}\" in cursive ink",
        "example": "a handwritten note on paper that says \"Fırat\" in cursive ink",
        "messageKey": "prompt.handwritten",
        "language": "English",
        "languageKey": "prompt.language.english",
        "coached": "no",
        "coachedKey": "prompt.coached.no",
        "scene": "handwritten"
      },
      {
        "key": "ledmatrix",
        "labelKey": "prompt.ledmatrix.name",
        "groupKey": "prompt.ledmatrix.group",
        "descriptionKey": "prompt.ledmatrix.description",
        "label": "LED matrix",
        "group": "Display",
        "description": "A red dot-matrix display, where the bug becomes a literal extra lit pixel.",
        "template": "a red LED dot-matrix display showing the text \"{word}\"",
        "example": "a red LED dot-matrix display showing the text \"Fırat\"",
        "messageKey": "prompt.ledmatrix",
        "language": "English",
        "languageKey": "prompt.language.english",
        "coached": "no",
        "coachedKey": "prompt.coached.no",
        "scene": "ledmatrix"
      },
      {
        "key": "sign-tr",
        "labelKey": "prompt.sign_tr.name",
        "groupKey": "prompt.sign_tr.group",
        "descriptionKey": "prompt.sign_tr.description",
        "label": "Turkish sign prompt",
        "group": "Instruction language",
        "description": "The same studio-sign scene, but the instruction itself is written in Turkish.",
        "template": "üzerinde \"{word}\" yazan bir tabelanın temiz bir stüdyo fotoğrafı",
        "example": "üzerinde \"Fırat\" yazan bir tabelanın temiz bir stüdyo fotoğrafı",
        "messageKey": "prompt.sign_tr",
        "language": "Turkish",
        "languageKey": "prompt.language.turkish",
        "coached": "no",
        "coachedKey": "prompt.coached.no",
        "scene": "sign"
      },
      {
        "key": "sign-coach",
        "labelKey": "prompt.sign_coach.name",
        "groupKey": "prompt.sign_coach.group",
        "descriptionKey": "prompt.sign_coach.description",
        "label": "Coached sign prompt",
        "group": "Glyph coaching",
        "description": "The English sign prompt plus an explicit reminder about dotless ı and Turkish marks.",
        "template": "a clean studio photograph of a sign that reads \"{word}\". The text is Turkish — render every letter exactly: the dotless letter ı must have NO dot, and keep the marks on ç ğ ö ş ü.",
        "example": "a clean studio photograph of a sign that reads \"Fırat\". The text is Turkish — render every letter exactly: the dotless letter ı must have NO dot, and keep the marks on ç ğ ö ş ü.",
        "messageKey": "prompt.sign_coach",
        "language": "English",
        "languageKey": "prompt.language.english",
        "coached": "yes",
        "coachedKey": "prompt.coached.yes",
        "scene": "sign"
      },
      {
        "key": "sign-trcoach",
        "labelKey": "prompt.sign_trcoach.name",
        "groupKey": "prompt.sign_trcoach.group",
        "descriptionKey": "prompt.sign_trcoach.description",
        "label": "Turkish coached sign prompt",
        "group": "Language + coaching",
        "description": "The Turkish sign prompt plus the same explicit glyph reminder, also in Turkish.",
        "template": "üzerinde \"{word}\" yazan bir tabelanın temiz bir stüdyo fotoğrafı. Metin Türkçedir — her harfi tam olarak çiz: noktasız ı harfinde nokta olmamalı, ç ğ ö ş ü harflerinin işaretlerini koru.",
        "example": "üzerinde \"Fırat\" yazan bir tabelanın temiz bir stüdyo fotoğrafı. Metin Türkçedir — her harfi tam olarak çiz: noktasız ı harfinde nokta olmamalı, ç ğ ö ş ü harflerinin işaretlerini koru.",
        "messageKey": "prompt.sign_trcoach",
        "language": "Turkish",
        "languageKey": "prompt.language.turkish",
        "coached": "yes",
        "coachedKey": "prompt.coached.yes",
        "scene": "sign"
      }
    ]
  }
}