{
  "patent_number": "US 10121471",
  "country": "US",
  "title": "How Voice Assistants Know When You've Finished Talking",
  "original_title": "Language model speech endpointing",
  "summary": "This patent describes a method for speech recognition systems to accurately detect the end of a spoken phrase by considering multiple interpretations of the audio and how much silence each interpretation suggests.",
  "what_it_does": "This patent details a computer-implemented method for determining when a speaker has finished talking during automatic speech recognition (ASR) processing. It works by receiving audio data, like from a mobile device microphone, and then performing ASR to generate a \"plurality of hypotheses\" (Claim 1). Each hypothesis is a different guess about what was said. For each guess, the system figures out its probability and how many \"non-speech audio frames\" (silence or background noise) immediately came before a potential end point. It then calculates a \"respective score\" for each hypothesis by multiplying its probability by a factor related to its non-speech frames. These scores are summed to create a \"cumulative score.\" If this cumulative score goes above a certain \"first threshold,\" the system decides that the speaker has likely finished speaking at that point. For example, if you say \"Hey Alexa, what's the weather?\" the system constantly checks if the weighted sum of non-speech from all its guesses indicates you've stopped talking after \"weather.\"",
  "what_it_does_not_cover": [
    "Does not cover endpointing methods that rely solely on a single best guess (hypothesis) of what was said, rather than considering multiple possibilities.",
    "Does not cover systems that only detect silence or a drop in audio volume without considering the linguistic context or probabilities of different interpretations.",
    "Does not cover endpointing based on a fixed duration of silence after detected speech, without weighting by hypothesis probability.",
    "Does not cover endpointing methods that do not use a \"plurality of hypotheses\" from an ASR decoder.",
    "Does not cover systems that determine an endpoint based on a user pressing a button to indicate the end of speech."
  ],
  "filed": "2015-06-29",
  "granted": "2018-11-06",
  "expires": "2035-06-29",
  "status": "active",
  "holder": "Amazon Technologies",
  "holder_url": "https://patentbrief.org/company/amazon-technologies",
  "inventors": [
    {
      "name": "Ariya Rastrow",
      "url": "https://patentbrief.org/inventor/ariya-rastrow"
    },
    {
      "name": "Bjorn Hoffmeister",
      "url": "https://patentbrief.org/inventor/bjorn-hoffmeister"
    },
    {
      "name": "Baiyang Liu",
      "url": "https://patentbrief.org/inventor/baiyang-liu"
    }
  ],
  "times_cited": 10,
  "tags": [
    "software",
    "telecommunications",
    "consumer_electronics",
    "ai_ml"
  ],
  "abstract": "An automatic speech recognition (ASR) system detects an endpoint of an utterance using the active hypotheses under consideration by a decoder. The ASR system calculates the amount of non-speech detected by a plurality of hypotheses and weights the non-speech duration by the probability of each hypotheses. When the aggregate weighted non-speech exceeds a threshold, an endpoint may be declared.",
  "url": "https://patentbrief.org/patent/us/10121471/language-model-speech-endpointing",
  "markdown_url": "https://patentbrief.org/patent/us/10121471/language-model-speech-endpointing/md",
  "google_patents_url": "https://patents.google.com/patent/US10121471",
  "relatedPatents": [
    {
      "patentNumber": "10923111",
      "countryCode": "US",
      "title": "How Voice Assistants Distinguish Speakers After a Wakeword",
      "url": "https://patentbrief.org/patent/us/10923111/speech-detection-and-speech-recognition"
    },
    {
      "patentNumber": "10847149",
      "countryCode": "US",
      "title": "How Voice Assistants Listen for More Commands Without a Wake Word",
      "url": "https://patentbrief.org/patent/us/10847149/speech-based-attention-span-for-voice-user-interface"
    },
    {
      "patentNumber": "11361763",
      "countryCode": "US",
      "title": "How Smart Speakers Know You're Talking to Them After a Command",
      "url": "https://patentbrief.org/patent/us/11361763/detecting-system-directed-speech"
    },
    {
      "patentNumber": "11514901",
      "countryCode": "US",
      "title": "How Voice Assistants Recognize Who is Talking",
      "url": "https://patentbrief.org/patent/us/11514901/anchored-speech-detection-and-speech-recognition"
    },
    {
      "patentNumber": "11645473",
      "countryCode": "US",
      "title": "How AI Predicts Who Will Speak Next in a Conversation",
      "url": "https://patentbrief.org/patent/us/11645473/palm-pathways-language-model"
    }
  ]
}