{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "title": "apizt independent benchmark attempts v1",
  "description": "Only redacted metadata. billedUsd is individually verified actual billed USD, never a price estimate; null is unknown. elapsedMs is whole-client-request time. inputTokens/outputTokens are source-reported values; cache/reasoning may be subsets, never add them. Each real retry needs a new trialId. Unknown/truncated/http-error passed must be null. No prompt, answer, key, URL, response ID or error content. Export only contains supported identifiers/configuration metadata/status counters/numeric aggregates. Unsupported fields and finish-marker text are not exported. Unknown custom reasoning settings should be null. Remove never-submitted placeholder rows before analyzing; retain real attempts with unknown submission state.",
  "type": "object",
  "required": [
    "schemaVersion",
    "kind",
    "configs",
    "records"
  ],
  "additionalProperties": false,
  "properties": {
    "schemaVersion": {
      "const": 1
    },
    "kind": {
      "const": "benchmark-attempts"
    },
    "note": {
      "type": "string",
      "maxLength": 500
    },
    "configs": {
      "type": "array",
      "minItems": 1,
      "maxItems": 200,
      "items": {
        "$ref": "#/$defs/config"
      }
    },
    "records": {
      "type": "array",
      "minItems": 1,
      "maxItems": 10000,
      "items": {
        "$ref": "#/$defs/attempt"
      }
    }
  },
  "$defs": {
    "config": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "id",
        "protocol",
        "requestedGenerationCap",
        "budgetMappingVerified"
      ],
      "properties": {
        "id": {
          "type": "string",
          "minLength": 1,
          "maxLength": 160,
          "pattern": "^[A-Za-z0-9][A-Za-z0-9._:/-]*$",
          "description": "A public model/dataset/sample/trial/config identifier only. Runtime also rejects known credential patterns; never put any secret here.",
          "not": {
            "anyOf": [
              {
                "pattern": "(?:^|[._:/-])[sS][kK][-_]"
              },
              {
                "pattern": "(?:AIza|gh[pousr]_|github_pat_|glpat-|xox[baprs]-|AKIA[0-9A-Z]{12}|ASIA[0-9A-Z]{12})"
              },
              {
                "pattern": "(?:^|[._:/-])eyJ[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+$"
              },
              {
                "pattern": "[bB][eE][aA][rR][eE][rR]|://"
              }
            ]
          }
        },
        "protocol": {
          "enum": [
            "responses",
            "messages",
            "chat"
          ]
        },
        "requestedGenerationCap": {
          "type": [
            "integer",
            "null"
          ],
          "minimum": 1,
          "maximum": 1000000
        },
        "effort": {
          "enum": [
            "none",
            "minimal",
            "low",
            "medium",
            "high",
            "xhigh",
            "max",
            "unknown",
            null
          ]
        },
        "thinking": {
          "enum": [
            "adaptive",
            "enabled",
            "disabled",
            "unknown",
            null
          ]
        },
        "toolChoice": {
          "enum": [
            "auto",
            "none",
            "required",
            "any",
            "unknown",
            null
          ]
        },
        "temperature": {
          "type": [
            "number",
            "null"
          ],
          "minimum": 0,
          "maximum": 9007199254740991
        },
        "topP": {
          "type": [
            "number",
            "null"
          ],
          "minimum": 0,
          "maximum": 9007199254740991
        },
        "budgetMappingVerified": {
          "type": "boolean",
          "description": "User assertion only. Tool cannot independently verify gateway parameter mapping."
        }
      }
    },
    "attempt": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "trialId",
        "model",
        "datasetId",
        "sampleId",
        "configId",
        "status",
        "passed",
        "elapsedMs",
        "billedUsd"
      ],
      "properties": {
        "trialId": {
          "type": "string",
          "minLength": 1,
          "maxLength": 160,
          "pattern": "^[A-Za-z0-9][A-Za-z0-9._:/-]*$",
          "description": "A public model/dataset/sample/trial/config identifier only. Runtime also rejects known credential patterns; never put any secret here.",
          "not": {
            "anyOf": [
              {
                "pattern": "(?:^|[._:/-])[sS][kK][-_]"
              },
              {
                "pattern": "(?:AIza|gh[pousr]_|github_pat_|glpat-|xox[baprs]-|AKIA[0-9A-Z]{12}|ASIA[0-9A-Z]{12})"
              },
              {
                "pattern": "(?:^|[._:/-])eyJ[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+$"
              },
              {
                "pattern": "[bB][eE][aA][rR][eE][rR]|://"
              }
            ]
          }
        },
        "model": {
          "type": "string",
          "minLength": 1,
          "maxLength": 160,
          "pattern": "^[A-Za-z0-9][A-Za-z0-9._:/-]*$",
          "description": "A public model/dataset/sample/trial/config identifier only. Runtime also rejects known credential patterns; never put any secret here.",
          "not": {
            "anyOf": [
              {
                "pattern": "(?:^|[._:/-])[sS][kK][-_]"
              },
              {
                "pattern": "(?:AIza|gh[pousr]_|github_pat_|glpat-|xox[baprs]-|AKIA[0-9A-Z]{12}|ASIA[0-9A-Z]{12})"
              },
              {
                "pattern": "(?:^|[._:/-])eyJ[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+$"
              },
              {
                "pattern": "[bB][eE][aA][rR][eE][rR]|://"
              }
            ]
          }
        },
        "datasetId": {
          "type": "string",
          "minLength": 1,
          "maxLength": 160,
          "pattern": "^[A-Za-z0-9][A-Za-z0-9._:/-]*$",
          "description": "A public model/dataset/sample/trial/config identifier only. Runtime also rejects known credential patterns; never put any secret here.",
          "not": {
            "anyOf": [
              {
                "pattern": "(?:^|[._:/-])[sS][kK][-_]"
              },
              {
                "pattern": "(?:AIza|gh[pousr]_|github_pat_|glpat-|xox[baprs]-|AKIA[0-9A-Z]{12}|ASIA[0-9A-Z]{12})"
              },
              {
                "pattern": "(?:^|[._:/-])eyJ[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+$"
              },
              {
                "pattern": "[bB][eE][aA][rR][eE][rR]|://"
              }
            ]
          }
        },
        "sampleId": {
          "type": "string",
          "minLength": 1,
          "maxLength": 160,
          "pattern": "^[A-Za-z0-9][A-Za-z0-9._:/-]*$",
          "description": "A public model/dataset/sample/trial/config identifier only. Runtime also rejects known credential patterns; never put any secret here.",
          "not": {
            "anyOf": [
              {
                "pattern": "(?:^|[._:/-])[sS][kK][-_]"
              },
              {
                "pattern": "(?:AIza|gh[pousr]_|github_pat_|glpat-|xox[baprs]-|AKIA[0-9A-Z]{12}|ASIA[0-9A-Z]{12})"
              },
              {
                "pattern": "(?:^|[._:/-])eyJ[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+$"
              },
              {
                "pattern": "[bB][eE][aA][rR][eE][rR]|://"
              }
            ]
          }
        },
        "configId": {
          "type": "string",
          "minLength": 1,
          "maxLength": 160,
          "pattern": "^[A-Za-z0-9][A-Za-z0-9._:/-]*$",
          "description": "A public model/dataset/sample/trial/config identifier only. Runtime also rejects known credential patterns; never put any secret here.",
          "not": {
            "anyOf": [
              {
                "pattern": "(?:^|[._:/-])[sS][kK][-_]"
              },
              {
                "pattern": "(?:AIza|gh[pousr]_|github_pat_|glpat-|xox[baprs]-|AKIA[0-9A-Z]{12}|ASIA[0-9A-Z]{12})"
              },
              {
                "pattern": "(?:^|[._:/-])eyJ[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+$"
              },
              {
                "pattern": "[bB][eE][aA][rR][eE][rR]|://"
              }
            ]
          }
        },
        "status": {
          "enum": [
            "completed",
            "unknown",
            "truncated",
            "http-error"
          ]
        },
        "passed": {
          "type": [
            "boolean",
            "null"
          ]
        },
        "elapsedMs": {
          "type": [
            "number",
            "null"
          ],
          "minimum": 0,
          "maximum": 9007199254740991
        },
        "billedUsd": {
          "type": [
            "number",
            "null"
          ],
          "minimum": 0,
          "maximum": 9007199254740991
        },
        "inputTokens": {
          "type": [
            "integer",
            "null"
          ],
          "minimum": 0,
          "maximum": 9007199254740991,
          "description": "Normalized total input tokens, including cache read/write when reported. For native Messages input_tokens, first add complete cache_read_input_tokens and cache_creation_input_tokens. Unknown/incomplete totals are null. Source Responses/Chat prompt/input already includes cache; do not add cache again."
        },
        "outputTokens": {
          "type": [
            "integer",
            "null"
          ],
          "minimum": 0,
          "maximum": 9007199254740991,
          "description": "Normalized source-reported total output tokens, including reasoning/thinking when the provider includes them. Do not add the reasoning subset again; unknown total is null."
        },
        "cachedInputTokens": {
          "type": [
            "integer",
            "null"
          ],
          "minimum": 0,
          "maximum": 9007199254740991,
          "description": "Cache-read portion reported by provider, a subset of normalized inputTokens; never add it again or infer it from missing data."
        },
        "reasoningTokens": {
          "type": [
            "integer",
            "null"
          ],
          "minimum": 0,
          "maximum": 9007199254740991,
          "description": "Reported reasoning/thinking portion. May already be included in normalized outputTokens; never add it again."
        },
        "finishReason": {
          "type": [
            "string",
            "null"
          ],
          "description": "Known terminal markers: completed/stop/end_turn/stop_sequence; tool_calls/function_call/tool_use ends only the proposal turn. refusal/content_filter force not passed; length/max_tokens/incomplete/model_context_window_exceeded are truncated; pause_turn/failed/cancelled/unknown and unrecognized strings downgrade to unknown. No answer or error text."
        },
        "completionConfirmed": {
          "type": "boolean"
        }
      },
      "allOf": [
        {
          "if": {
            "properties": {
              "status": {
                "const": "completed"
              }
            }
          },
          "then": {
            "properties": {
              "passed": {
                "type": "boolean"
              }
            }
          },
          "else": {
            "properties": {
              "passed": {
                "const": null
              }
            }
          }
        }
      ]
    }
  }
}