{
  "patent_number": "US 20250200392",
  "country": "US",
  "title": "How to Check if a Large Language Model's Answer is True",
  "original_title": "Large language model verification",
  "summary": "Amazon's patent details a system for verifying large language model responses by having a second AI translate the answer into formal logic, which an automated theorem prover then checks for validity or invalidity.",
  "what_it_does": "This patent describes a method for verifying answers from a large language model (LLM). First, a system obtains a 'conversational text' which includes a user's question (query) and the answer from a first LLM. Then, a *second* large language model translates this answer into 'first-order logic,' which is a formal way to represent statements and relationships (Claim 1). An automated theorem prover, specifically a 'satisfiability modulo theories solver,' then checks this logic translation. It determines if the answer is 'valid' by checking if its negation is impossible (unsatisfiable), and if it's 'invalid' by checking if the answer itself is impossible (Claim 1). Finally, the system outputs whether the answer is valid, invalid, or neither, to a user interface, database, or report.",
  "what_it_does_not_cover": [
    "Does not cover verifying LLM answers without translating them into first-order logic.",
    "Does not cover verification methods that do not use an automated theorem prover or satisfiability modulo theories solver.",
    "Does not cover systems where the first LLM directly provides the first-order logic translation without a second LLM.",
    "Does not cover verifying answers that are not derived from a query-response conversational text from an LLM.",
    "Does not cover systems that only determine validity or invalidity without also identifying 'neither valid nor invalid'."
  ],
  "filed": "2023-12-15",
  "granted": null,
  "expires": "2043-12-15",
  "status": "active",
  "holder": "Amazon Technologies",
  "holder_url": "https://patentbrief.org/company/amazon-technologies",
  "inventors": [
    {
      "name": "Michael William Whalen",
      "url": "https://patentbrief.org/inventor/michael-william-whalen"
    },
    {
      "name": "Nadia Labai",
      "url": "https://patentbrief.org/inventor/nadia-labai"
    },
    {
      "name": "Benjamin KIESL-REITER",
      "url": "https://patentbrief.org/inventor/benjamin-kiesl-reiter"
    },
    {
      "name": "John Byron Cook",
      "url": "https://patentbrief.org/inventor/john-byron-cook"
    }
  ],
  "times_cited": 12,
  "tags": [
    "ai_ml",
    "software",
    "telecommunications",
    "consumer_electronics"
  ],
  "abstract": "Verifying large language model responses involves obtaining a query and its corresponding answer from a large language model. This conversational text is then fed into a second large language model, which translates the answer into first-order logic. The verification process uses an automated theorem prover. It checks the validity of this logic translation by determining the unsatisfiability of two scenarios: one where the negation of the logic translation and domain-specific logic formulas are combined, and another where the logic translation itself is combined with these formulas. Based on this analysis, the theorem prover ascertains whether the translated answer is valid, invalid, or neither. The final step is communicating this verification status through an appropriate output medium, such as a graphical user interface, a database, or a report, providing a structured and methodical approach to assessing the accuracy and reliability of language model responses.",
  "url": "https://patentbrief.org/patent/us/20250200392/large-language-model-verification",
  "markdown_url": "https://patentbrief.org/patent/us/20250200392/large-language-model-verification/md",
  "google_patents_url": "https://patents.google.com/patent/US20250200392",
  "relatedPatents": [
    {
      "patentNumber": "11989527",
      "countryCode": "US",
      "title": "How a Computer System Checks and Improves AI Text",
      "url": "https://patentbrief.org/patent/us/11989527/computer-implemented-methods-for-the-automated-analysis-or-use-of-data-including-11989527"
    },
    {
      "patentNumber": "12073180",
      "countryCode": "US",
      "title": "How a Computer System Fact-Checks AI Language Models",
      "url": "https://patentbrief.org/patent/us/12073180/computer-implemented-methods-for-the-automated-analysis-or-use-of-data-including-12073180"
    },
    {
      "patentNumber": "12531820",
      "countryCode": "US",
      "title": "How Chatbots Use AI and Fact Databases to Learn and Improve",
      "url": "https://patentbrief.org/patent/us/12531820/knowledge-graph-assisted-large-language-models"
    },
    {
      "patentNumber": "12067362",
      "countryCode": "US",
      "title": "How a Computer System Improves Large Language Model Output with Structured Data",
      "url": "https://patentbrief.org/patent/us/12067362/computer-implemented-methods-for-the-automated-analysis-or-use-of-data-including-12067362"
    },
    {
      "patentNumber": "12353827",
      "countryCode": "US",
      "title": "Using Non-AI Systems to Improve AI Text Generation",
      "url": "https://patentbrief.org/patent/us/12353827/computer-implemented-methods-for-the-automated-analysis-or-use-of-data-including"
    }
  ]
}