{
  "openapi": "3.1.0",
  "info": {
    "title": "EyeSay API",
    "version": "1.0.0",
    "summary": "Tags, labels, search vectors, re-ranking and descriptions for photos.",
    "description": "The public developer API of EyeSay. Every call is a `POST` with a JSON body to `/v1/<endpoint>`. Human-readable documentation: https://eyesay.app/docs. A short guide for assistants: https://eyesay.app/llms.txt.\n\n**Authentication.** Send your account key as the header `X-Auth-Token` (create or replace it at https://eyesay.app/dashboard). An app the account holder connected with OAuth sends `Authorization: Bearer <access token>` instead; the authorization server metadata is at `/.well-known/oauth-authorization-server`. When both headers are sent, `X-Auth-Token` is used.\n\n**Photos.** A photo is a base64 string, a data URL (`data:image/jpeg;base64,...`), or `upload:<batch>/<n>` for a photo put on https://eyesay.app/upload by the same account (usable for thirty minutes). A request may be up to 32 MB and a photo up to 16 million pixels.\n\n**Allowance.** Photos are counted in units by what is done with each one (see `x-eyesay-units` on each operation); one unit is EUR 0.0001. Text sent to `/v1/embed` or `/v1/rerank` is free. A free account has 3,000 units each calendar month and up to 60 requests a minute (600 while it has bought units left); when the month's units run out, calls are paid from bought units. An account takes at most 50,000 image vectors a day (UTC) through `/v1/embed` and `/v1/analyze`.\n\n**Versions.** Every successful answer carries `model_version`, so you can tell when results may shift. Vectors carry `space`: vectors from different spaces must not be mixed.\n\n**Assistants.** Assistants that speak the Model Context Protocol connect to https://eyesay.app/mcp (Streamable HTTP, OAuth) instead of this API; see https://eyesay.app/docs#claude-apps.",
    "contact": {
      "email": "support@eyesay.app"
    },
    "termsOfService": "https://eyesay.app/terms"
  },
  "externalDocs": {
    "description": "API documentation",
    "url": "https://eyesay.app/docs"
  },
  "servers": [
    {
      "url": "https://eyesay.app"
    }
  ],
  "security": [
    {
      "ApiKey": []
    },
    {
      "OAuthBearer": []
    }
  ],
  "x-eyesay-limits": {
    "unit_eur": 0.0001,
    "free_units_per_month": 3000,
    "requests_per_minute": 60,
    "requests_per_minute_with_bought_units": 600,
    "image_vectors_per_day": 50000,
    "max_request_bytes": 33554432,
    "max_image_pixels": 16000000
  },
  "tags": [
    {
      "name": "Tags",
      "description": "Words for what a photo shows."
    },
    {
      "name": "Labels",
      "description": "Sorting a photo into your own labels."
    },
    {
      "name": "Search",
      "description": "Vectors for search, and re-ranking candidates."
    },
    {
      "name": "Descriptions",
      "description": "A description of a photo, or an answer about it."
    },
    {
      "name": "Service",
      "description": "Whether the service can answer. No key needed; nothing is counted."
    }
  ],
  "paths": {
    "/v1/tags": {
      "post": {
        "tags": [
          "Tags"
        ],
        "operationId": "tags",
        "summary": "Tags for a photo",
        "description": "Open-vocabulary tags for a photo, best first. A tag whose `p` is below `min_score` is a suggestion rather than a tag. Add `\"with_embedding\": true` to get the photo's search vector in the same answer, counted as one photo. Send `vocab` instead to score your own list of words: the answer then ranks your words, and `p` is the share of your list, not presence.",
        "x-eyesay-units": {
          "per": "image",
          "units": 3,
          "price_class": "tags"
        },
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/TagsRequest"
              },
              "examples": {
                "tags": {
                  "summary": "Eight tags for a photo (from /docs)",
                  "value": {
                    "image": "data:image/jpeg;base64,...",
                    "k": 8
                  }
                },
                "with_embedding": {
                  "summary": "Tags and the search vector in one call",
                  "value": {
                    "image": "data:image/jpeg;base64,...",
                    "with_embedding": true
                  }
                },
                "vocab": {
                  "summary": "Score your own list of words",
                  "value": {
                    "image": "data:image/jpeg;base64,...",
                    "vocab": [
                      "beach",
                      "mountain",
                      "city"
                    ]
                  }
                }
              }
            }
          }
        },
        "responses": {
          "200": {
            "description": "The tags. Open tags without `vocab`; your own words ranked with `vocab`.",
            "content": {
              "application/json": {
                "schema": {
                  "anyOf": [
                    {
                      "$ref": "#/components/schemas/OpenTagsResponse"
                    },
                    {
                      "$ref": "#/components/schemas/VocabTagsResponse"
                    }
                  ]
                }
              }
            }
          },
          "400": {
            "$ref": "#/components/responses/BadRequest"
          },
          "401": {
            "$ref": "#/components/responses/Unauthorized"
          },
          "403": {
            "$ref": "#/components/responses/Forbidden"
          },
          "413": {
            "$ref": "#/components/responses/PayloadTooLarge"
          },
          "422": {
            "$ref": "#/components/responses/ContentBlocked"
          },
          "429": {
            "$ref": "#/components/responses/TooManyRequests"
          },
          "500": {
            "$ref": "#/components/responses/InternalError"
          },
          "502": {
            "$ref": "#/components/responses/BadGateway"
          },
          "503": {
            "$ref": "#/components/responses/Unavailable"
          }
        }
      },
      "head": {
        "tags": [
          "Service"
        ],
        "operationId": "tagsReady",
        "summary": "Whether tagging can be served right now",
        "description": "No key needed, no body, nothing counted. Answers about the service, not about your key.",
        "security": [],
        "responses": {
          "204": {
            "description": "The header says whether the route can serve now.",
            "headers": {
              "X-PhotoTagger-Ready": {
                "description": "`true` when tagging can be served now, `false` when it cannot.",
                "schema": {
                  "type": "string",
                  "enum": [
                    "true",
                    "false"
                  ]
                }
              }
            }
          }
        }
      }
    },
    "/v1/tags/aggregate": {
      "post": {
        "tags": [
          "Tags"
        ],
        "operationId": "tagsAggregate",
        "summary": "Tags merged with facts you already know",
        "description": "Tags for one photo, merged with facts you already know about it (for now `{\"is_screenshot\": true}`), each tag saying where it came from. Only `image`, `metadata` and `k` are accepted; any other field is a 400. A fact is taken as given and never checked against the photo.",
        "x-eyesay-units": {
          "per": "image",
          "units": 3,
          "price_class": "tags"
        },
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/TagsAggregateRequest"
              },
              "examples": {
                "screenshot": {
                  "summary": "A photo you know is a screenshot",
                  "value": {
                    "image": "data:image/png;base64,...",
                    "metadata": {
                      "is_screenshot": true
                    },
                    "k": 5
                  }
                }
              }
            }
          }
        },
        "responses": {
          "200": {
            "description": "The merged tags and the evidence behind them.",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/TagsAggregateResponse"
                }
              }
            }
          },
          "400": {
            "$ref": "#/components/responses/BadRequest"
          },
          "401": {
            "$ref": "#/components/responses/Unauthorized"
          },
          "403": {
            "$ref": "#/components/responses/Forbidden"
          },
          "413": {
            "$ref": "#/components/responses/PayloadTooLarge"
          },
          "422": {
            "description": "No tag could be given for the photo (`code: no_tag_candidates`, with the same fields as a 200 and an empty `tags`), or the content was refused (`code: content_blocked`).",
            "content": {
              "application/json": {
                "schema": {
                  "anyOf": [
                    {
                      "$ref": "#/components/schemas/NoTagCandidates"
                    },
                    {
                      "$ref": "#/components/schemas/ContentRejected"
                    }
                  ]
                }
              }
            }
          },
          "429": {
            "$ref": "#/components/responses/TooManyRequests"
          },
          "500": {
            "$ref": "#/components/responses/InternalError"
          },
          "502": {
            "$ref": "#/components/responses/BadGateway"
          },
          "503": {
            "$ref": "#/components/responses/Unavailable"
          }
        }
      }
    },
    "/v1/classify": {
      "post": {
        "tags": [
          "Labels"
        ],
        "operationId": "classify",
        "summary": "Sort a photo into your own labels",
        "description": "Scores the photo against your labels, up to 64 of them. The scores are shares of your list (they sum to 1 over all labels), and `top` is the best label.",
        "x-eyesay-units": {
          "per": "image",
          "units": 3,
          "price_class": "tags"
        },
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/ClassifyRequest"
              },
              "examples": {
                "labels": {
                  "summary": "Food or receipt",
                  "value": {
                    "image": "data:image/jpeg;base64,...",
                    "labels": [
                      "food",
                      "receipt"
                    ]
                  }
                }
              }
            }
          }
        },
        "responses": {
          "200": {
            "description": "A score for each label.",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/ClassifyResponse"
                }
              }
            }
          },
          "400": {
            "$ref": "#/components/responses/BadRequest"
          },
          "401": {
            "$ref": "#/components/responses/Unauthorized"
          },
          "403": {
            "$ref": "#/components/responses/Forbidden"
          },
          "413": {
            "$ref": "#/components/responses/PayloadTooLarge"
          },
          "422": {
            "$ref": "#/components/responses/ContentBlocked"
          },
          "429": {
            "$ref": "#/components/responses/TooManyRequests"
          },
          "500": {
            "$ref": "#/components/responses/InternalError"
          },
          "502": {
            "$ref": "#/components/responses/BadGateway"
          },
          "503": {
            "$ref": "#/components/responses/Unavailable"
          }
        }
      }
    },
    "/v1/embed": {
      "post": {
        "tags": [
          "Search"
        ],
        "operationId": "embed",
        "summary": "Search vectors for photos or text",
        "description": "Vectors for search, up to 64 inputs a call, exactly one of `image`, `images`, `text`, `texts`. Photos and text land in the same space, so a text vector finds photos. `embeddings[i]` is the vector of input `i`. Keep the `space` value with what you store: vectors from different spaces must not be mixed. Image vectors count toward the account's daily limit of 50,000; text is free.",
        "x-eyesay-units": {
          "per": "image",
          "units": 1,
          "price_class": "vector",
          "text": 0
        },
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/EmbedRequest"
              },
              "examples": {
                "image": {
                  "summary": "One photo",
                  "value": {
                    "image": "data:image/jpeg;base64,..."
                  }
                },
                "texts": {
                  "summary": "Search texts (free)",
                  "value": {
                    "texts": [
                      "a dog on the beach",
                      "birthday cake"
                    ]
                  }
                }
              }
            }
          }
        },
        "responses": {
          "200": {
            "description": "The vectors, in input order, with what an index needs to store them.",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/EmbedResponse"
                }
              }
            }
          },
          "400": {
            "$ref": "#/components/responses/BadRequest"
          },
          "401": {
            "$ref": "#/components/responses/Unauthorized"
          },
          "403": {
            "$ref": "#/components/responses/Forbidden"
          },
          "413": {
            "$ref": "#/components/responses/PayloadTooLarge"
          },
          "422": {
            "$ref": "#/components/responses/ContentBlocked"
          },
          "429": {
            "$ref": "#/components/responses/TooManyRequests"
          },
          "500": {
            "$ref": "#/components/responses/InternalError"
          },
          "502": {
            "$ref": "#/components/responses/BadGateway"
          },
          "503": {
            "$ref": "#/components/responses/Unavailable"
          }
        }
      }
    },
    "/v1/rerank": {
      "post": {
        "tags": [
          "Search"
        ],
        "operationId": "rerank",
        "summary": "Re-order candidates against a query",
        "description": "Re-orders up to 128 candidates (photos or text) against a query, best first. Each photo in the query or the candidates counts as one image; text is free.",
        "x-eyesay-units": {
          "per": "image",
          "units": 1,
          "price_class": "vector",
          "text": 0
        },
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/RerankRequest"
              },
              "examples": {
                "text_query": {
                  "summary": "A text query against photos",
                  "value": {
                    "query": "a dog on the beach",
                    "documents": [
                      {
                        "id": "IMG_0001",
                        "image": "data:image/jpeg;base64,..."
                      },
                      {
                        "id": "IMG_0002",
                        "image": "data:image/jpeg;base64,..."
                      }
                    ],
                    "top_k": 1
                  }
                }
              }
            }
          }
        },
        "responses": {
          "200": {
            "description": "The candidates, best first.",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/RerankResponse"
                }
              }
            }
          },
          "400": {
            "$ref": "#/components/responses/BadRequest"
          },
          "401": {
            "$ref": "#/components/responses/Unauthorized"
          },
          "403": {
            "$ref": "#/components/responses/Forbidden"
          },
          "413": {
            "$ref": "#/components/responses/PayloadTooLarge"
          },
          "422": {
            "$ref": "#/components/responses/ContentBlocked"
          },
          "429": {
            "$ref": "#/components/responses/TooManyRequests"
          },
          "500": {
            "$ref": "#/components/responses/InternalError"
          },
          "502": {
            "$ref": "#/components/responses/BadGateway"
          },
          "503": {
            "$ref": "#/components/responses/Unavailable"
          }
        }
      }
    },
    "/v1/ask": {
      "post": {
        "tags": [
          "Descriptions"
        ],
        "operationId": "ask",
        "summary": "Describe a photo or answer about it",
        "description": "The text decides what is answered. No text: a one-sentence description, usable as a caption or alt text (`mode: caption`; written by AI, so check it before publishing). A yes/no question: the probability of yes (`mode: yesno`). Any other question: a short answer (`mode: qa`). Text that is not a question (a statement to match against the photo) is refused with `400` and nothing is counted.",
        "x-eyesay-units": {
          "per": "image",
          "units": 9,
          "price_class": "gen"
        },
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/AskRequest"
              },
              "examples": {
                "describe": {
                  "summary": "A description (caption / alt text)",
                  "value": {
                    "image": "data:image/jpeg;base64,..."
                  }
                },
                "yesno": {
                  "summary": "A yes/no question",
                  "value": {
                    "image": "data:image/jpeg;base64,...",
                    "text": "Is there a dog?"
                  }
                },
                "question": {
                  "summary": "An open question",
                  "value": {
                    "image": "data:image/jpeg;base64,...",
                    "text": "What color is the car?"
                  }
                }
              }
            }
          }
        },
        "responses": {
          "200": {
            "description": "The answer. Which fields appear depends on `mode`.",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/AskResponse"
                }
              }
            }
          },
          "400": {
            "$ref": "#/components/responses/BadRequest"
          },
          "401": {
            "$ref": "#/components/responses/Unauthorized"
          },
          "403": {
            "$ref": "#/components/responses/Forbidden"
          },
          "413": {
            "$ref": "#/components/responses/PayloadTooLarge"
          },
          "422": {
            "$ref": "#/components/responses/ContentBlocked"
          },
          "429": {
            "$ref": "#/components/responses/TooManyRequests"
          },
          "500": {
            "$ref": "#/components/responses/InternalError"
          },
          "502": {
            "$ref": "#/components/responses/BadGateway"
          },
          "503": {
            "$ref": "#/components/responses/Unavailable"
          }
        }
      }
    },
    "/v1/analyze": {
      "post": {
        "tags": [
          "Descriptions"
        ],
        "operationId": "analyze",
        "summary": "Description, tags and search vector in one call",
        "description": "A description, tags and a search vector in one call; with `labels`, a score for each label, and with `text`, how well the photo matches it. The vector counts toward the account's daily limit of 50,000 image vectors.",
        "x-eyesay-units": {
          "per": "image",
          "units": 9,
          "price_class": "gen"
        },
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/AnalyzeRequest"
              },
              "examples": {
                "labels": {
                  "summary": "With labels",
                  "value": {
                    "image": "data:image/jpeg;base64,...",
                    "labels": [
                      "food",
                      "receipt"
                    ]
                  }
                }
              }
            }
          }
        },
        "responses": {
          "200": {
            "description": "The description, tags, vector and scores.",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/AnalyzeResponse"
                }
              }
            }
          },
          "400": {
            "$ref": "#/components/responses/BadRequest"
          },
          "401": {
            "$ref": "#/components/responses/Unauthorized"
          },
          "403": {
            "$ref": "#/components/responses/Forbidden"
          },
          "413": {
            "$ref": "#/components/responses/PayloadTooLarge"
          },
          "422": {
            "$ref": "#/components/responses/ContentBlocked"
          },
          "429": {
            "$ref": "#/components/responses/TooManyRequests"
          },
          "500": {
            "$ref": "#/components/responses/InternalError"
          },
          "502": {
            "$ref": "#/components/responses/BadGateway"
          },
          "503": {
            "$ref": "#/components/responses/Unavailable"
          }
        }
      }
    },
    "/health": {
      "get": {
        "tags": [
          "Service"
        ],
        "operationId": "health",
        "summary": "Whether the service is up",
        "description": "No key needed; nothing is counted.",
        "security": [],
        "responses": {
          "200": {
            "description": "The service can answer.",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/Health"
                }
              }
            }
          },
          "503": {
            "description": "The service cannot answer inference requests right now (`ok` is false).",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/Health"
                }
              }
            }
          }
        }
      }
    }
  },
  "components": {
    "securitySchemes": {
      "ApiKey": {
        "type": "apiKey",
        "in": "header",
        "name": "X-Auth-Token",
        "description": "The account key from https://eyesay.app/dashboard. Keep it secret; replacing it there stops the old one at once."
      },
      "OAuthBearer": {
        "type": "http",
        "scheme": "bearer",
        "description": "An OAuth access token for an app the account holder approved. Authorization server metadata: `/.well-known/oauth-authorization-server`. Ignored when `X-Auth-Token` is also sent."
      }
    },
    "schemas": {
      "Image": {
        "type": "string",
        "description": "A photo: base64, a data URL (`data:image/jpeg;base64,...`), or `upload:<batch>/<n>` for a photo this account put on the upload page in the last thirty minutes. Up to 16 million pixels.",
        "examples": [
          "data:image/jpeg;base64,..."
        ]
      },
      "Labels": {
        "type": "array",
        "description": "Your labels: 1 to 64 different strings of 1 to 100 characters. A repeated label is a 400.",
        "minItems": 1,
        "maxItems": 64,
        "uniqueItems": true,
        "items": {
          "type": "string",
          "minLength": 1,
          "maxLength": 100
        }
      },
      "ModelVersion": {
        "type": "string",
        "description": "The model that answered. When it changes, results may shift."
      },
      "TagsRequest": {
        "type": "object",
        "required": [
          "image"
        ],
        "properties": {
          "image": {
            "$ref": "#/components/schemas/Image"
          },
          "k": {
            "type": "integer",
            "minimum": 1,
            "maximum": 20,
            "default": 5,
            "description": "At most this many tags."
          },
          "with_embedding": {
            "type": "boolean",
            "default": false,
            "description": "Also return the photo's search vector (`embedding`), counted as one photo. Only with open tags: not with `vocab`."
          },
          "vocab": {
            "$ref": "#/components/schemas/Labels",
            "description": "Score these words instead of open tags: 1 to 64 different strings of 1 to 100 characters."
          }
        }
      },
      "Tag": {
        "type": "object",
        "required": [
          "tag",
          "p"
        ],
        "properties": {
          "tag": {
            "type": "string"
          },
          "p": {
            "type": "number",
            "minimum": 0,
            "maximum": 1,
            "description": "The tag's score, 4 decimal places."
          }
        }
      },
      "OpenTag": {
        "allOf": [
          {
            "$ref": "#/components/schemas/Tag"
          }
        ],
        "type": "object",
        "properties": {
          "slot": {
            "type": "string",
            "description": "Which kind of word the tag is (a key of `groups`, such as `object` or `scene`)."
          },
          "line": {
            "type": "number",
            "description": "For a word with its own display line: the score it needs to be shown as a tag."
          }
        }
      },
      "OpenTagsResponse": {
        "type": "object",
        "description": "Open-vocabulary tags (a request without `vocab`).",
        "required": [
          "mode",
          "tags",
          "min_score",
          "model_version"
        ],
        "properties": {
          "mode": {
            "const": "tags"
          },
          "slots": {
            "const": true
          },
          "tags": {
            "type": "array",
            "description": "The tags, best first, at most `k`.",
            "items": {
              "$ref": "#/components/schemas/OpenTag"
            }
          },
          "min_score": {
            "type": "number",
            "description": "A tag whose `p` is below this is a suggestion."
          },
          "slot_min_score": {
            "type": "object",
            "description": "Per slot: the score a tag from that slot should reach to be shown as a tag rather than a suggestion.",
            "additionalProperties": {
              "type": "number"
            }
          },
          "groups": {
            "type": "object",
            "description": "The candidates each slot read, before the list was merged.",
            "additionalProperties": {
              "type": "array",
              "items": {
                "$ref": "#/components/schemas/Tag"
              }
            }
          },
          "visible_slots": {
            "type": "array",
            "items": {
              "type": "string"
            }
          },
          "below_threshold_fallback": {
            "type": "boolean",
            "description": "True when even the first tag is below `min_score`."
          },
          "model_version": {
            "$ref": "#/components/schemas/ModelVersion"
          },
          "embedding": {
            "$ref": "#/components/schemas/EmbedResponse",
            "description": "Only with `\"with_embedding\": true`: the photo's search vector, as `/v1/embed` returns it."
          }
        }
      },
      "VocabTagsResponse": {
        "type": "object",
        "description": "Your own words, ranked (a request with `vocab`). `p` is the share of your list.",
        "required": [
          "mode",
          "vocab_size",
          "tags",
          "model_version"
        ],
        "properties": {
          "mode": {
            "const": "tags"
          },
          "vocab_size": {
            "type": "integer"
          },
          "tags": {
            "type": "array",
            "items": {
              "$ref": "#/components/schemas/Tag"
            }
          },
          "model_version": {
            "$ref": "#/components/schemas/ModelVersion"
          }
        }
      },
      "TagsAggregateRequest": {
        "type": "object",
        "required": [
          "image"
        ],
        "additionalProperties": false,
        "properties": {
          "image": {
            "$ref": "#/components/schemas/Image"
          },
          "metadata": {
            "type": "object",
            "additionalProperties": false,
            "description": "Facts you already know about the photo. Leave a fact out when it is unknown.",
            "properties": {
              "is_screenshot": {
                "type": "boolean",
                "description": "Only `true` adds the tag `screenshot`; `false` adds nothing."
              }
            }
          },
          "k": {
            "type": "integer",
            "minimum": 1,
            "maximum": 16,
            "default": 5,
            "description": "At most this many tags."
          }
        }
      },
      "AggregateTag": {
        "type": "object",
        "required": [
          "tag",
          "source",
          "score",
          "score_type",
          "below_threshold",
          "inferred",
          "fallback"
        ],
        "properties": {
          "tag": {
            "type": "string"
          },
          "source": {
            "type": "string",
            "enum": [
              "photos_metadata",
              "controlled_classifier",
              "open_cloze"
            ],
            "description": "`photos_metadata`: a fact you sent. `controlled_classifier`: a server-configured group. `open_cloze`: an open tag."
          },
          "score": {
            "type": [
              "number",
              "null"
            ],
            "description": "Null for a fact you sent. Scores of different `score_type` are not comparable."
          },
          "score_type": {
            "type": [
              "string",
              "null"
            ],
            "enum": [
              "cloze_score",
              "group_softmax",
              null
            ]
          },
          "below_threshold": {
            "type": [
              "boolean",
              "null"
            ],
            "description": "For open tags: whether the score is below `open_min_score`. Null otherwise."
          },
          "inferred": {
            "type": "boolean",
            "description": "False only for a fact you sent."
          },
          "fallback": {
            "type": "boolean",
            "description": "True for the single best candidate given when nothing else qualified."
          }
        }
      },
      "AggregateCandidate": {
        "allOf": [
          {
            "$ref": "#/components/schemas/AggregateTag"
          }
        ],
        "type": "object",
        "required": [
          "group",
          "group_rejected",
          "accepted",
          "filter_reason"
        ],
        "properties": {
          "group": {
            "type": [
              "string",
              "null"
            ],
            "description": "`capture` or `content` for a server-configured group, null otherwise."
          },
          "group_rejected": {
            "type": "boolean"
          },
          "accepted": {
            "type": "boolean",
            "description": "Whether this candidate is in `tags`."
          },
          "filter_reason": {
            "type": [
              "string",
              "null"
            ],
            "enum": [
              "raw_evidence_only",
              "below_threshold",
              "group_rejected",
              "not_group_top1",
              "controlled_assignment_disabled",
              "duplicate",
              "k_limit",
              null
            ],
            "description": "Why the candidate is not in `tags`; null when it is, or when nothing ruled it out."
          },
          "stage": {
            "type": "string",
            "enum": [
              "raw",
              "filtered"
            ],
            "description": "Open tags only: the raw reading or the filtered one."
          }
        }
      },
      "TagsAggregateResponse": {
        "type": "object",
        "required": [
          "tags",
          "candidates",
          "groups",
          "policy_version",
          "vocab_version",
          "model_version",
          "open_min_score",
          "controlled_assignment"
        ],
        "properties": {
          "tags": {
            "type": "array",
            "description": "Facts first, then server-configured groups, then open tags; at most `k`.",
            "items": {
              "$ref": "#/components/schemas/AggregateTag"
            }
          },
          "candidates": {
            "type": "array",
            "description": "Every candidate considered, with why it was or was not taken.",
            "items": {
              "$ref": "#/components/schemas/AggregateCandidate"
            }
          },
          "groups": {
            "type": "object",
            "description": "The state of each server-configured group (`capture`, `content`).",
            "additionalProperties": {
              "type": "object",
              "required": [
                "status"
              ],
              "properties": {
                "status": {
                  "type": "string",
                  "enum": [
                    "skipped",
                    "rejected",
                    "candidate_only",
                    "selected"
                  ]
                }
              }
            }
          },
          "policy_version": {
            "type": "string"
          },
          "vocab_version": {
            "type": "string"
          },
          "model_version": {
            "$ref": "#/components/schemas/ModelVersion"
          },
          "open_min_score": {
            "type": "number"
          },
          "controlled_assignment": {
            "type": "boolean"
          }
        }
      },
      "NoTagCandidates": {
        "description": "No tag could be given: the fields of a 200 answer with an empty `tags`, plus `error` and `code`.",
        "allOf": [
          {
            "$ref": "#/components/schemas/TagsAggregateResponse"
          }
        ],
        "type": "object",
        "required": [
          "error",
          "code"
        ],
        "properties": {
          "error": {
            "type": "string"
          },
          "code": {
            "const": "no_tag_candidates"
          }
        }
      },
      "ClassifyRequest": {
        "type": "object",
        "required": [
          "image",
          "labels"
        ],
        "properties": {
          "image": {
            "$ref": "#/components/schemas/Image"
          },
          "labels": {
            "$ref": "#/components/schemas/Labels"
          },
          "k": {
            "type": "integer",
            "minimum": 1,
            "maximum": 20,
            "description": "Return only the `k` best labels in `class_scores`. Without it, every label."
          }
        }
      },
      "ClassifyResponse": {
        "type": "object",
        "required": [
          "mode",
          "class_scores",
          "top",
          "confidence",
          "model_version"
        ],
        "properties": {
          "mode": {
            "const": "classify"
          },
          "class_scores": {
            "type": "object",
            "description": "Label to score (4 decimal places); over all your labels the scores sum to 1.",
            "additionalProperties": {
              "type": "number"
            }
          },
          "top": {
            "type": "string",
            "description": "The best label."
          },
          "confidence": {
            "type": "number",
            "description": "The best label's score."
          },
          "model_version": {
            "$ref": "#/components/schemas/ModelVersion"
          }
        }
      },
      "EmbedRequest": {
        "description": "Exactly one of `image`, `images`, `text`, `texts`. Lists hold 1 to 64 items. Text longer than 2,000 characters is cut.",
        "oneOf": [
          {
            "type": "object",
            "title": "One photo",
            "required": [
              "image"
            ],
            "properties": {
              "image": {
                "$ref": "#/components/schemas/Image"
              }
            }
          },
          {
            "type": "object",
            "title": "Photos",
            "required": [
              "images"
            ],
            "properties": {
              "images": {
                "type": "array",
                "minItems": 1,
                "maxItems": 64,
                "items": {
                  "$ref": "#/components/schemas/Image"
                }
              }
            }
          },
          {
            "type": "object",
            "title": "One text",
            "required": [
              "text"
            ],
            "properties": {
              "text": {
                "type": "string",
                "minLength": 1
              }
            }
          },
          {
            "type": "object",
            "title": "Texts",
            "required": [
              "texts"
            ],
            "properties": {
              "texts": {
                "type": "array",
                "minItems": 1,
                "maxItems": 64,
                "items": {
                  "type": "string",
                  "minLength": 1
                }
              }
            }
          }
        ]
      },
      "EmbedResponse": {
        "type": "object",
        "required": [
          "mode",
          "embeddings",
          "input_kind",
          "count",
          "dim",
          "normalized",
          "metric",
          "precision",
          "space",
          "model_version"
        ],
        "properties": {
          "mode": {
            "const": "embed"
          },
          "embeddings": {
            "type": "array",
            "description": "One vector per input, in input order.",
            "items": {
              "type": "array",
              "items": {
                "type": "number"
              }
            }
          },
          "embedding": {
            "type": "array",
            "description": "Only for a single `image` or `text`: the same vector as `embeddings[0]`.",
            "items": {
              "type": "number"
            }
          },
          "input_kind": {
            "type": "string",
            "enum": [
              "image",
              "text"
            ]
          },
          "count": {
            "type": "integer"
          },
          "dim": {
            "type": "integer",
            "description": "Length of each vector."
          },
          "normalized": {
            "const": "l2"
          },
          "metric": {
            "const": "cosine"
          },
          "precision": {
            "type": "integer",
            "description": "Decimal places each number is rounded to."
          },
          "space": {
            "type": "string",
            "description": "An opaque id of the vector space. Store it with the vectors: vectors from different spaces must not be mixed."
          },
          "model_version": {
            "$ref": "#/components/schemas/ModelVersion"
          }
        }
      },
      "RerankItem": {
        "type": "object",
        "description": "Exactly one of `text` and `image`. Text longer than 2,000 characters is cut.",
        "properties": {
          "text": {
            "type": "string",
            "minLength": 1
          },
          "image": {
            "$ref": "#/components/schemas/Image"
          }
        },
        "oneOf": [
          {
            "required": [
              "text"
            ]
          },
          {
            "required": [
              "image"
            ]
          }
        ]
      },
      "RerankRequest": {
        "type": "object",
        "required": [
          "query",
          "documents"
        ],
        "properties": {
          "query": {
            "description": "A text, or an object with exactly one of `text` and `image`.",
            "oneOf": [
              {
                "type": "string",
                "minLength": 1
              },
              {
                "$ref": "#/components/schemas/RerankItem"
              }
            ]
          },
          "documents": {
            "type": "array",
            "minItems": 1,
            "maxItems": 128,
            "items": {
              "allOf": [
                {
                  "$ref": "#/components/schemas/RerankItem"
                }
              ],
              "type": "object",
              "properties": {
                "id": {
                  "type": [
                    "string",
                    "integer"
                  ],
                  "description": "Your id for the candidate, echoed back."
                }
              }
            }
          },
          "top_k": {
            "type": "integer",
            "minimum": 1,
            "description": "Return only the best `top_k`. Default: all."
          }
        }
      },
      "RerankResponse": {
        "type": "object",
        "required": [
          "mode",
          "scorer",
          "model_version",
          "calibration_version",
          "results"
        ],
        "properties": {
          "mode": {
            "const": "rerank"
          },
          "scorer": {
            "type": "string",
            "description": "How `score` was computed."
          },
          "model_version": {
            "$ref": "#/components/schemas/ModelVersion"
          },
          "calibration_version": {
            "type": "string"
          },
          "results": {
            "type": "array",
            "description": "Best first.",
            "items": {
              "type": "object",
              "required": [
                "index",
                "id",
                "score"
              ],
              "properties": {
                "index": {
                  "type": "integer",
                  "description": "Position of the candidate in `documents`."
                },
                "id": {
                  "type": [
                    "string",
                    "integer",
                    "null"
                  ],
                  "description": "The candidate's `id`, or null when it had none."
                },
                "score": {
                  "type": "number",
                  "description": "What the list is ordered by, highest first."
                },
                "similarity": {
                  "type": "number",
                  "description": "Cosine similarity between query and candidate."
                }
              }
            }
          }
        }
      },
      "AskRequest": {
        "type": "object",
        "required": [
          "image"
        ],
        "properties": {
          "image": {
            "$ref": "#/components/schemas/Image"
          },
          "text": {
            "type": "string",
            "description": "A question; leave it out for a description. Longer than 2,000 characters is cut."
          }
        }
      },
      "AskResponse": {
        "type": "object",
        "required": [
          "mode",
          "model_version"
        ],
        "properties": {
          "mode": {
            "type": "string",
            "enum": [
              "caption",
              "yesno",
              "qa"
            ],
            "description": "What was answered: a description, a yes/no question or an open question."
          },
          "answer": {
            "type": "string",
            "description": "`caption`: the description. `yesno`: `yes` or `no`. `qa`: the answer."
          },
          "confidence": {
            "type": [
              "number",
              "null"
            ]
          },
          "p_yes": {
            "type": "number",
            "description": "`yesno` only: the probability of yes."
          },
          "cut": {
            "type": "number",
            "description": "`yesno` only: `answer` is `yes` when `p_yes` is at or above this."
          },
          "model_version": {
            "$ref": "#/components/schemas/ModelVersion"
          }
        }
      },
      "AnalyzeRequest": {
        "type": "object",
        "required": [
          "image"
        ],
        "properties": {
          "image": {
            "$ref": "#/components/schemas/Image"
          },
          "text": {
            "type": "string",
            "description": "A statement to match the photo against. Longer than 2,000 characters is cut."
          },
          "labels": {
            "$ref": "#/components/schemas/Labels"
          }
        }
      },
      "AnalyzeResponse": {
        "type": "object",
        "required": [
          "mode",
          "schema_version",
          "model_version",
          "caption",
          "caption_confidence",
          "embedding",
          "class_scores",
          "image_text_score",
          "tags",
          "review_priority"
        ],
        "properties": {
          "mode": {
            "const": "analyze"
          },
          "schema_version": {
            "const": "manifest/0.9"
          },
          "model_version": {
            "$ref": "#/components/schemas/ModelVersion"
          },
          "caption": {
            "type": "string",
            "description": "A one-sentence description."
          },
          "caption_confidence": {
            "type": "number"
          },
          "embedding": {
            "type": "array",
            "description": "The photo's search vector (L2-normalized, 5 decimal places).",
            "items": {
              "type": "number"
            }
          },
          "class_scores": {
            "type": "object",
            "description": "Label to score for `labels`; empty without them.",
            "additionalProperties": {
              "type": "number"
            }
          },
          "image_text_score": {
            "type": [
              "number",
              "null"
            ],
            "description": "With `text`: similarity between photo and text. Null without it."
          },
          "tags": {
            "type": "array",
            "description": "Up to 5 tags.",
            "items": {
              "$ref": "#/components/schemas/Tag"
            }
          },
          "review_priority": {
            "type": "string",
            "enum": [
              "low",
              "medium",
              "high"
            ],
            "description": "How much a person should look at this answer."
          },
          "support": {
            "type": "integer",
            "minimum": -100,
            "maximum": 100,
            "description": "With `text`, when the statement is scored: how strongly the photo supports it."
          },
          "match_verdict": {
            "type": "string",
            "enum": [
              "matched",
              "review",
              "mismatched"
            ],
            "description": "With `text`, when the statement is scored."
          }
        }
      },
      "Health": {
        "type": "object",
        "required": [
          "ok"
        ],
        "properties": {
          "ok": {
            "type": "boolean"
          },
          "pressure": {
            "type": "boolean",
            "description": "True while the service is under load or short of capacity."
          }
        }
      },
      "Error": {
        "type": "object",
        "required": [
          "error"
        ],
        "properties": {
          "error": {
            "type": "string",
            "description": "What went wrong, in words."
          }
        }
      },
      "ContentRejected": {
        "type": "object",
        "required": [
          "error",
          "code",
          "stage"
        ],
        "properties": {
          "error": {
            "type": "string"
          },
          "code": {
            "type": "string",
            "enum": [
              "content_blocked",
              "moderation_unavailable"
            ]
          },
          "stage": {
            "type": "string",
            "enum": [
              "input",
              "output"
            ],
            "description": "Whether the request or the answer was refused."
          },
          "policy_version": {
            "type": "string"
          }
        }
      }
    },
    "headers": {
      "Retry-After": {
        "description": "Seconds to wait before retrying.",
        "schema": {
          "type": "integer",
          "minimum": 1
        }
      }
    },
    "responses": {
      "BadRequest": {
        "description": "The body is not what the endpoint takes, or a photo cannot be read; the message says what is wrong.",
        "content": {
          "application/json": {
            "schema": {
              "$ref": "#/components/schemas/Error"
            }
          }
        }
      },
      "Unauthorized": {
        "description": "No key, a wrong key, or an access token that is invalid, expired or ended.",
        "headers": {
          "WWW-Authenticate": {
            "description": "Where an OAuth client finds the authorization server (`Bearer resource_metadata=\"...\"`).",
            "schema": {
              "type": "string"
            }
          }
        },
        "content": {
          "application/json": {
            "schema": {
              "$ref": "#/components/schemas/Error"
            }
          }
        }
      },
      "Forbidden": {
        "description": "The endpoint is not part of your account's plan.",
        "content": {
          "application/json": {
            "schema": {
              "$ref": "#/components/schemas/Error"
            }
          }
        }
      },
      "PayloadTooLarge": {
        "description": "The request is too large (over 32 MB, or its photos exceed the memory one request may use); send fewer or smaller photos.",
        "content": {
          "application/json": {
            "schema": {
              "$ref": "#/components/schemas/Error"
            }
          }
        }
      },
      "ContentBlocked": {
        "description": "The request or its answer was refused by content screening (`content_blocked`).",
        "content": {
          "application/json": {
            "schema": {
              "$ref": "#/components/schemas/ContentRejected"
            }
          }
        }
      },
      "TooManyRequests": {
        "description": "Refused for now; the message says why. With `Retry-After`: too many requests a minute, too many wrong keys from one address, or the service is busy or full; wait that many seconds and retry. Without it: this month's units are used up, or the account's image vectors for the day are; retrying does not help until they renew.",
        "headers": {
          "Retry-After": {
            "$ref": "#/components/headers/Retry-After"
          }
        },
        "content": {
          "application/json": {
            "schema": {
              "$ref": "#/components/schemas/Error"
            }
          }
        }
      },
      "InternalError": {
        "description": "Something failed on our side.",
        "content": {
          "application/json": {
            "schema": {
              "$ref": "#/components/schemas/Error"
            }
          }
        }
      },
      "BadGateway": {
        "description": "The inference service gave no usable answer; retry.",
        "content": {
          "application/json": {
            "schema": {
              "$ref": "#/components/schemas/Error"
            }
          }
        }
      },
      "Unavailable": {
        "description": "Inference, or content screening, is temporarily unavailable; retry shortly.",
        "headers": {
          "Retry-After": {
            "$ref": "#/components/headers/Retry-After"
          }
        },
        "content": {
          "application/json": {
            "schema": {
              "anyOf": [
                {
                  "$ref": "#/components/schemas/Error"
                },
                {
                  "$ref": "#/components/schemas/ContentRejected"
                }
              ]
            }
          }
        }
      }
    }
  }
}