{
  "patent_number": "US 11295174",
  "country": "US",
  "title": "How AI Agents Learn Faster by Predicting Opponents in Parallel",
  "original_title": "Opponent modeling with asynchronous methods in deep RL",
  "summary": "This patent describes a computer system that trains artificial intelligence agents more efficiently by using many processors to simultaneously learn from a simulated environment, while also explicitly predicting the actions of other agents.",
  "what_it_does": "This system trains a neural network for artificial intelligence agents using a method called parallelized asynchronous reinforcement learning, which means many 'worker' processors learn at the same time. Each worker interacts with a simulation, calculates how much its actions helped or hurt (local gradient computation), and then helps update the main AI's brain (global network parameters) (Claim 1). Crucially, the system adds 'agent modeling' by including a special part in its learning calculation, called a 'supervised cross entropy loss' (Claim 1). This loss term helps the AI predict what other agents will do by comparing its predictions to what those agents actually did (observed one-hot encoded agent action) (Claim 2). For example, an AI learning to play a complex strategy game could use this to predict an opponent's next move, like where they might build a base or attack, making the learning agent smarter and faster to train.",
  "what_it_does_not_cover": [
    "Does not cover reinforcement learning systems that train a single AI agent without modeling other agents or opponents (Claim 1, Claim 10).",
    "Does not cover AI training methods that do not use parallel processing with multiple worker processes (Claim 1: 'parallelized asynchronous reinforcement learning').",
    "Does not cover learning systems where the loss function does not include at least a policy loss, a value loss, and a supervised cross entropy loss (Claim 1).",
    "Does not cover agent modeling that doesn't involve comparing predicted actions against observed, one-hot encoded actions of other agents (Claim 2).",
    "Does not cover neural networks that do not learn opponent or teammate policies, potentially through a latent space (Claim 5, Claim 7)."
  ],
  "filed": "2019-11-05",
  "granted": "2022-04-05",
  "expires": "2039-11-05",
  "status": "active",
  "holder": "Royal Bank of Canada",
  "holder_url": "https://patentbrief.org/company/royal-bank-of-canada",
  "inventors": [
    {
      "name": "Pablo Francisco HERNANDEZ LEAL",
      "url": "https://patentbrief.org/inventor/pablo-francisco-hernandez-leal"
    },
    {
      "name": "Bilal KARTAL",
      "url": "https://patentbrief.org/inventor/bilal-kartal"
    },
    {
      "name": "Matthew Edmund TAYLOR",
      "url": "https://patentbrief.org/inventor/matthew-edmund-taylor"
    }
  ],
  "times_cited": 1,
  "tags": [
    "ai_ml",
    "software",
    "finance",
    "telecommunications",
    "gaming"
  ],
  "abstract": "A computer system and method for extending parallelized asynchronous reinforcement learning to include agent modeling for training a neural network is described. Coordinated operation of plurality of hardware processors or threads is utilized such that each functions as a worker process that is configured to simultaneously interact with a target computing environment for local gradient computation based on a loss determination mechanism and to update global network parameters. The loss determination mechanism includes at least a policy loss term (actor), a value loss term (critic), and a supervised cross entropy loss. Variations are described further where the neural network is adapted to include a latent space to track agent policy features.",
  "url": "https://patentbrief.org/patent/us/11295174/opponent-modeling-with-asynchronous-methods-in-deep-rl",
  "markdown_url": "https://patentbrief.org/patent/us/11295174/opponent-modeling-with-asynchronous-methods-in-deep-rl/md",
  "google_patents_url": "https://patents.google.com/patent/US11295174",
  "relatedPatents": [
    {
      "patentNumber": "11170293",
      "countryCode": "US",
      "title": "How AI Systems Learn to Predict and Act Simultaneously",
      "url": "https://patentbrief.org/patent/us/11170293/alphago-policy-and-value-networks"
    },
    {
      "patentNumber": "11429762",
      "countryCode": "US",
      "title": "How Computers Train AI Models Using Separate Virtual Simulations",
      "url": "https://patentbrief.org/patent/us/11429762/simulation-orchestration-for-training-reinforcement-learning-models"
    },
    {
      "patentNumber": "12518214",
      "countryCode": "US",
      "title": "Training AI on Private Data Without Seeing It",
      "url": "https://patentbrief.org/patent/us/12518214/distributed-machine-learning-systems-including-generation-of-synthetic-data"
    },
    {
      "patentNumber": "11836577",
      "countryCode": "US",
      "title": "Training Robot AI Models Faster Using Smart Simulations",
      "url": "https://patentbrief.org/patent/us/11836577/reinforcement-learning-model-training-through-simulation"
    },
    {
      "patentNumber": "11120365",
      "countryCode": "US",
      "title": "How AI Models Learn Complex Tasks by Breaking Them Down",
      "url": "https://patentbrief.org/patent/us/11120365/for-hierarchical-decomposition-deep-reinforcement-learning-for-an-artificial-int"
    }
  ]
}