{
  "patent_number": "US 11836577",
  "country": "US",
  "title": "Training Robot AI Models Faster Using Smart Simulations",
  "original_title": "Reinforcement learning model training through simulation",
  "summary": "This patent describes a cloud service that helps train artificial intelligence models for robots by running simulations, even suggesting improvements to the AI's learning rules before starting.",
  "what_it_does": "This patent details a computer-implemented method where a 'simulation management service' receives code from a customer. This code defines a 'reinforcement function' for training an AI model for a system, like a robot (Claim 1). The service then evaluates this code and suggests ways to improve it, based on past experiences with similar code (Claim 1). After modifying the code, the service creates a 'simulation environment' and injects the improved code into a 'simulation application' for the robot (Claim 1). Finally, it performs the reinforcement learning within this simulated world. For example, the simulation might select a robot's 'state' (like its position) and 'actions' (like moving forward), then provide a 'reward value' based on how well the action performed, which helps the AI model learn and improve (Claim 2, 4).",
  "what_it_does_not_cover": [
    "Does not cover training reinforcement learning models directly on physical robots without using a simulation environment.",
    "Does not cover systems that train AI models without first evaluating and suggesting modifications to the customer's reinforcement function code.",
    "Does not cover other types of machine learning, like supervised or unsupervised learning, that do not involve a reinforcement function and reward-based training.",
    "Does not cover a simulation system where the user's code is not injected into a pre-existing simulation application.",
    "Does not cover a system that doesn't use prior code or historical data to generate suggestions for modifying the reinforcement function."
  ],
  "filed": "2018-11-27",
  "granted": "2023-12-05",
  "expires": "2038-11-27",
  "status": "active",
  "holder": "Amazon Technologies",
  "holder_url": "https://patentbrief.org/company/amazon-technologies",
  "inventors": [
    {
      "name": "Leo Parker Dirac",
      "url": "https://patentbrief.org/inventor/leo-parker-dirac"
    },
    {
      "name": "Eric Li Sun",
      "url": "https://patentbrief.org/inventor/eric-li-sun"
    },
    {
      "name": "Marthinus Coenraad De Clercq Wentzel",
      "url": "https://patentbrief.org/inventor/marthinus-coenraad-de-clercq-wentzel"
    },
    {
      "name": "Sahika Genc",
      "url": "https://patentbrief.org/inventor/sahika-genc"
    },
    {
      "name": "Bharathan Balaji",
      "url": "https://patentbrief.org/inventor/bharathan-balaji"
    },
    {
      "name": "Sunil Mallya Kasaragod",
      "url": "https://patentbrief.org/inventor/sunil-mallya-kasaragod"
    }
  ],
  "times_cited": 0,
  "tags": [
    "ai_ml",
    "robotics",
    "software",
    "telecommunications",
    "consumer_electronics"
  ],
  "abstract": "A simulation management service receives a request to perform reinforcement learning for a robotic device. The request can include computer-executable code defining a reinforcement function for training a reinforcement learning model for the robotic device. In response to the request, the simulation management service generates a simulation environment and injects the computer-executable code into a simulation application for the robotic device. Using the simulation application and the computer-executable code, the simulation management service performs the reinforcement learning within the simulation environment.",
  "url": "https://patentbrief.org/patent/us/11836577/reinforcement-learning-model-training-through-simulation",
  "markdown_url": "https://patentbrief.org/patent/us/11836577/reinforcement-learning-model-training-through-simulation/md",
  "google_patents_url": "https://patents.google.com/patent/US11836577",
  "relatedPatents": [
    {
      "patentNumber": "11436484",
      "countryCode": "US",
      "title": "Nvidia's Method for Training Self-Driving Car AI in Simulations",
      "url": "https://patentbrief.org/patent/us/11436484/alphazero"
    },
    {
      "patentNumber": "10282665",
      "countryCode": "US",
      "title": "How AI Agents Learn to Pick the Best Future Actions",
      "url": "https://patentbrief.org/patent/us/10282665/action-selection-with-a-reward-estimator-applied-to-machine-learning"
    },
    {
      "patentNumber": "12518214",
      "countryCode": "US",
      "title": "Training AI on Private Data Without Seeing It",
      "url": "https://patentbrief.org/patent/us/12518214/distributed-machine-learning-systems-including-generation-of-synthetic-data"
    },
    {
      "patentNumber": "10607134",
      "countryCode": "US",
      "title": "How AI Learns to Control Game Characters Based on Their Surroundings",
      "url": "https://patentbrief.org/patent/us/10607134/artificially-intelligent-systems-devices-and-methods-for-learning-andor-using-an-avatars-circumstances-for-autonomous-avatar-operation"
    },
    {
      "patentNumber": "12423586",
      "countryCode": "US",
      "title": "Making AI Smarter by Focusing on Unsure 'Nodes'",
      "url": "https://patentbrief.org/patent/us/12423586/training-nodes-of-a-neural-network-to-be-decisive"
    }
  ]
}