{
  "patent_number": "US 11429762",
  "country": "US",
  "title": "How Computers Train AI Models Using Separate Virtual Simulations",
  "original_title": "Simulation orchestration for training reinforcement learning models",
  "summary": "This patent describes a system where one virtual computer runs simulations of a system, like a robot, and another virtual computer uses the simulation data to teach an AI model how to make better decisions.",
  "what_it_does": "The patent outlines a method for training a reinforcement learning model by orchestrating two virtual compute nodes. A 'simulation workflow manager' (from the abstract) configures a first virtual computer with a training application and a second virtual computer with a simulation application (claim 1). The second computer runs a simulation of a system, such as a 'robotic device' (claim 1), which performs actions chosen by the model. Data from this simulation, including the action taken, the resulting new state, and a reward value (claim 3), is then sent to the first virtual computer. The training application on the first computer uses this data to improve the model. This process can repeat, with the updated model being sent back to the simulation for further refinement (claim 4).",
  "what_it_does_not_cover": [
    "Training AI models without using a separate, dedicated simulation environment.",
    "Simulations that do not involve a 'robotic device' performing actions in the simulation environment (claim 1).",
    "Training AI models where the model does not randomize the selection of actions within the simulation (claim 1).",
    "Training AI models where the simulation and the model training occur on the same single compute node.",
    "Simulations performed for purposes other than training a reinforcement learning model for system optimization."
  ],
  "filed": "2018-11-27",
  "granted": "2022-08-30",
  "expires": "2038-11-27",
  "status": "active",
  "holder": "Amazon Technologies",
  "holder_url": "https://patentbrief.org/company/amazon-technologies",
  "inventors": [
    {
      "name": "Leo Parker Dirac",
      "url": "https://patentbrief.org/inventor/leo-parker-dirac"
    },
    {
      "name": "Sahika Gene",
      "url": "https://patentbrief.org/inventor/sahika-gene"
    },
    {
      "name": "Eric Li Sun",
      "url": "https://patentbrief.org/inventor/eric-li-sun"
    },
    {
      "name": "Marthinus Coenraad De Clercq Wentzel",
      "url": "https://patentbrief.org/inventor/marthinus-coenraad-de-clercq-wentzel"
    },
    {
      "name": "Brian James Townsend",
      "url": "https://patentbrief.org/inventor/brian-james-townsend"
    },
    {
      "name": "Pramod Ravikumar Kumar",
      "url": "https://patentbrief.org/inventor/pramod-ravikumar-kumar"
    },
    {
      "name": "Bharathan Balaji",
      "url": "https://patentbrief.org/inventor/bharathan-balaji"
    },
    {
      "name": "Sunil Mallya Kasaragod",
      "url": "https://patentbrief.org/inventor/sunil-mallya-kasaragod"
    }
  ],
  "times_cited": 13,
  "tags": [
    "software",
    "ai_ml",
    "telecommunications",
    "robotics",
    "automotive"
  ],
  "abstract": "A simulation workflow manager obtains a set of parameters for simulation of a system and training of a reinforcement learning model for optimizing an application of the system. In response to obtaining the set of parameters, the simulation workflow manager configures a first compute node that includes a training application for training the reinforcement learning model. The simulation workflow manager also configures a second compute note with a simulation application to perform the simulation of the system in a simulation environment. Data is generated through execution of the simulation in the second compute node that is provided to the first compute node to cause the training application to use the data to train the reinforcement learning model.",
  "url": "https://patentbrief.org/patent/us/11429762/simulation-orchestration-for-training-reinforcement-learning-models",
  "markdown_url": "https://patentbrief.org/patent/us/11429762/simulation-orchestration-for-training-reinforcement-learning-models/md",
  "google_patents_url": "https://patents.google.com/patent/US11429762",
  "relatedPatents": []
}