{
  "patent_number": "US 10452978",
  "country": "US",
  "title": "How AI Models Understand Language Using 'Attention'",
  "original_title": "Attention-based sequence transduction neural networks",
  "summary": "This patent describes a neural network architecture, known as a Transformer, that uses a \"self-attention\" mechanism to process sequences of information, like words in a sentence, by weighing the importance of different parts of the input.",
  "what_it_does": "The system described takes an input sequence, such as a sentence, and converts it into an output sequence, like a translation. It uses an \"encoder neural network\" (Claim 1) to process the input, which is made up of multiple \"encoder subnetworks.\" Each subnetwork contains an \"encoder self-attention sub-layer\" (Claim 1). This sub-layer processes each part of the input by applying a self-attention mechanism, which involves determining a \"query\" from the current input position, and \"keys\" and \"values\" from all input positions (Claim 1). These are then used to generate a specific output for that input position, allowing the network to understand how different parts of the sequence relate to each other. For example, when translating a sentence, this mechanism helps the AI determine which words are most relevant to understanding the meaning of a particular word.",
  "what_it_does_not_cover": [
    "Does not cover neural networks that process sequences without using a self-attention mechanism.",
    "Does not cover attention mechanisms where the \"query,\" \"keys,\" and \"values\" are not derived from the *same* subnetwork inputs for self-attention.",
    "Does not cover systems that lack either an \"encoder neural network\" or a \"decoder neural network\" as part of the overall sequence transduction network (Claim 1).",
    "Does not cover models that do not explicitly determine and use \"query,\" \"keys,\" and \"values\" from the subnetwork inputs as described in Claim 1.",
    "Does not cover systems where the initial encoder subnetwork inputs are not formed by combining embedded representations with positional embeddings, as described in Claim 2."
  ],
  "filed": "2018-06-28",
  "granted": "2019-10-22",
  "expires": "2038-06-28",
  "status": "active",
  "holder": "Google LLC",
  "holder_url": "https://patentbrief.org/company/google-llc",
  "inventors": [
    {
      "name": "Lukasz Mieczyslaw Kaiser",
      "url": "https://patentbrief.org/inventor/lukasz-mieczyslaw-kaiser"
    },
    {
      "name": "Illia Polosukhin",
      "url": "https://patentbrief.org/inventor/illia-polosukhin"
    },
    {
      "name": "Llion Owen Jones",
      "url": "https://patentbrief.org/inventor/llion-owen-jones"
    },
    {
      "name": "Noam M. Shazeer",
      "url": "https://patentbrief.org/inventor/noam-m-shazeer"
    },
    {
      "name": "Niki J. Parmar",
      "url": "https://patentbrief.org/inventor/niki-j-parmar"
    },
    {
      "name": "Ashish Teku Vaswani",
      "url": "https://patentbrief.org/inventor/ashish-teku-vaswani"
    },
    {
      "name": "Jakob D. Uszkoreit",
      "url": "https://patentbrief.org/inventor/jakob-d-uszkoreit"
    },
    {
      "name": "Aidan Nicholas Gomez",
      "url": "https://patentbrief.org/inventor/aidan-nicholas-gomez"
    }
  ],
  "times_cited": 45,
  "tags": [
    "ai_ml",
    "software",
    "telecommunications",
    "consumer_electronics"
  ],
  "abstract": "Methods, systems, and apparatus, including computer programs encoded on a computer storage medium, for generating an output sequence from an input sequence. In one aspect, one of the systems includes an encoder neural network configured to receive the input sequence and generate encoded representations of the network inputs, the encoder neural network comprising a sequence of one or more encoder subnetworks, each encoder subnetwork configured to receive a respective encoder subnetwork input for each of the input positions and to generate a respective subnetwork output for each of the input positions, and each encoder subnetwork comprising: an encoder self-attention sub-layer that is configured to receive the subnetwork input for each of the input positions and, for each particular input position in the input order: apply an attention mechanism over the encoder subnetwork inputs using one or more queries derived from the encoder subnetwork input at the particular input position.",
  "url": "https://patentbrief.org/patent/us/10452978/transformer-attention-mechanism",
  "markdown_url": "https://patentbrief.org/patent/us/10452978/transformer-attention-mechanism/md",
  "google_patents_url": "https://patents.google.com/patent/US10452978",
  "relatedPatents": [
    {
      "patentNumber": "9965705",
      "countryCode": "US",
      "title": "How AI Uses Question-Guided Attention to Answer Questions About Images",
      "url": "https://patentbrief.org/patent/us/9965705/systems-and-methods-for-attention-based-configurable-convolutional-neural-networks-abc-cnn-for-visual-question-answering"
    },
    {
      "patentNumber": "20220383078",
      "countryCode": "US",
      "title": "Adapting AI Models to Fit Device Resources",
      "url": "https://patentbrief.org/patent/us/20220383078/data-processing-method-and-related-device"
    },
    {
      "patentNumber": "10909459",
      "countryCode": "US",
      "title": "Teaching Computers to Understand Document Similarity Using AI",
      "url": "https://patentbrief.org/patent/us/10909459/federated-learning"
    },
    {
      "patentNumber": "10289962",
      "countryCode": "US",
      "title": "How to Shrink Large AI Models Using Knowledge Distillation",
      "url": "https://patentbrief.org/patent/us/10289962/deep-q-networks-dqn"
    },
    {
      "patentNumber": "20250363357",
      "countryCode": "US",
      "title": "How to Update AI on Small Devices with Slow Internet",
      "url": "https://patentbrief.org/patent/us/20250363357/systems-and-methods-for-deploying-and-updating-neural-networks-at-the-edge-of-a-"
    }
  ]
}