{
  "patent_number": "US 12032919",
  "country": "US",
  "title": "How to Make Large Language Models More Confident in Their Answers",
  "original_title": "Post-calibration of large language model confidence scoring via combined techniques",
  "summary": "Snowflake's patent details a method to make large language models more trustworthy when extracting information from documents by precisely adjusting their confidence scores using a mix of mathematical techniques.",
  "what_it_does": "This patent describes a system that improves how reliably large language models (LLMs) report their confidence when processing electronic text documents. First, it receives results from an ML model, including a confidence score (Claim 1). To address accuracy challenges, it then applies a special recalibration process to these confidence scores. This recalibration involves a combination of techniques: temperature scaling, selecting top-k probabilities from the softmax denominator, and polynomial regression (Claim 4). After adjusting the confidence scores, the system extracts specific pieces of information from the document and stores them in a database, each with its newly adjusted, more accurate confidence score (Claim 1). For example, if an LLM extracts a date from a contract, this method ensures the confidence score for that date is a more accurate reflection of the LLM's certainty.",
  "what_it_does_not_cover": [
    "Does not cover improving confidence scores for machine learning models not specifically configured for document processing and understanding of electronic text documents.",
    "Does not cover calibration methods that do not combine temperature scaling, softmax denominator top-k probability selection, and polynomial regression.",
    "Does not cover systems that only provide a confidence score without then extracting individual elements of information and storing them with adjusted scores in a database.",
    "Does not cover ML models that process non-textual data, such as images or audio, for confidence scoring.",
    "Does not cover methods that improve selective prediction accuracy without also improving calibration quality."
  ],
  "filed": "2023-08-16",
  "granted": "2024-07-09",
  "expires": "2043-08-16",
  "status": "active",
  "holder": "Snowflake",
  "holder_url": "https://patentbrief.org/company/snowflake",
  "inventors": [
    {
      "name": "Andrzej Szwabe",
      "url": "https://patentbrief.org/inventor/andrzej-szwabe"
    }
  ],
  "times_cited": 10,
  "tags": [
    "software",
    "ai_ml",
    "telecommunications",
    "finance",
    "legal_tech"
  ],
  "abstract": "Examples provide a large language model confidence scoring post-calibration based on a combination of temperature scaling, softmax denominator top-k probabilities selection, and polynomial regression. A secure machine learning system receives results generated by a machine learning (ML) model, the results including at least one confidence score. The secure ML system identifies at least one challenge in accuracy of the results generated by the ML model configured to perform document processing and understanding. The secure machine learning system implements confidence scoring recalibration to address at least one challenge, the confidence scoring recalibration including functionality to assess reliability of the results generated by the ML model, and applies post-processing calibration to the at least one confidence score generated by the confidence scoring recalibration to enhance performance of the ML model, the post-processing calibration including adjusting the at least one confidence score generated by the confidence scoring recalibration.",
  "url": "https://patentbrief.org/patent/us/12032919/post-calibration-of-large-language-model-confidence-scoring-via-combined-techniq",
  "markdown_url": "https://patentbrief.org/patent/us/12032919/post-calibration-of-large-language-model-confidence-scoring-via-combined-techniq/md",
  "google_patents_url": "https://patents.google.com/patent/US12032919",
  "relatedPatents": [
    {
      "patentNumber": "12236201",
      "countryCode": "US",
      "title": "How AI Models Can Be More Confident in Their Document Answers",
      "url": "https://patentbrief.org/patent/us/12236201/enhanced-machine-learning-model-accuracy-through-post-hoc-confidence-score-calib"
    },
    {
      "patentNumber": "12299406",
      "countryCode": "US",
      "title": "How AI Checks Documents for Policy Rules and Compliance",
      "url": "https://patentbrief.org/patent/us/12299406/large-language-model-artificial-intelligence-text-evaluation-system"
    },
    {
      "patentNumber": "11989527",
      "countryCode": "US",
      "title": "How a Computer System Checks and Improves AI Text",
      "url": "https://patentbrief.org/patent/us/11989527/computer-implemented-methods-for-the-automated-analysis-or-use-of-data-including-11989527"
    },
    {
      "patentNumber": "12073180",
      "countryCode": "US",
      "title": "How a Computer System Fact-Checks AI Language Models",
      "url": "https://patentbrief.org/patent/us/12073180/computer-implemented-methods-for-the-automated-analysis-or-use-of-data-including-12073180"
    },
    {
      "patentNumber": "12067362",
      "countryCode": "US",
      "title": "How a Computer System Improves Large Language Model Output with Structured Data",
      "url": "https://patentbrief.org/patent/us/12067362/computer-implemented-methods-for-the-automated-analysis-or-use-of-data-including-12067362"
    }
  ]
}