{
  "patent_number": "US 12001320",
  "country": "US",
  "title": "How AI Learns to Test Software in the Cloud",
  "original_title": "Cloud architecture for reinforcement learning",
  "summary": "This patent describes a cloud system where a single AI model learns to test many live software applications at once by trying actions, getting feedback, and improving itself.",
  "what_it_does": "This technology provides a way for a single artificial intelligence (AI) model, specifically a reinforcement learning model, to automatically test multiple versions of a software application running live in a cloud environment. The AI model first selects an action to take in the user interface of a software instance, such as clicking a button (Claim 1). It then causes that action to be performed and receives a 'reward' based on the outcome, like whether a task was completed (Claim 1, Claim 8). This process is repeated for other software instances, potentially running on different devices (Claim 5, Claim 10). The AI model then uses these actions and rewards from all instances to update and improve its testing strategy (Claim 1). For example, the AI could be testing a new mobile app, learning which sequence of taps and swipes leads to a successful login or exposes a bug.",
  "what_it_does_not_cover": [
    "Does not cover software testing performed solely by human users.",
    "Does not cover testing in simulated or offline environments, as it specifies 'live environments'.",
    "Does not cover testing where the AI does not interact with a graphical 'user interface' or 'interface element'.",
    "Does not cover using a separate reinforcement learning model for each individual software instance being tested.",
    "Does not cover testing methods that do not involve determining a 'reward' to train the learning model."
  ],
  "filed": "2022-06-30",
  "granted": "2024-06-04",
  "expires": "2042-06-30",
  "status": "active",
  "holder": "Microsoft Technology Licensing",
  "holder_url": "https://patentbrief.org/company/microsoft-technology-licensing",
  "inventors": [
    {
      "name": "Steve K. LIM",
      "url": "https://patentbrief.org/inventor/steve-k-lim"
    },
    {
      "name": "Marc Mas MEZQUITA",
      "url": "https://patentbrief.org/inventor/marc-mas-mezquita"
    },
    {
      "name": "Levent OZGUR",
      "url": "https://patentbrief.org/inventor/levent-ozgur"
    },
    {
      "name": "Kashyap Maheshkumar PATEL",
      "url": "https://patentbrief.org/inventor/kashyap-maheshkumar-patel"
    },
    {
      "name": "Timothy James Chapman",
      "url": "https://patentbrief.org/inventor/timothy-james-chapman"
    },
    {
      "name": "Xiaoyan Liu",
      "url": "https://patentbrief.org/inventor/xiaoyan-liu"
    },
    {
      "name": "Taylor Paul SPANGLER",
      "url": "https://patentbrief.org/inventor/taylor-paul-spangler"
    }
  ],
  "times_cited": 5,
  "tags": [
    "software",
    "ai_ml",
    "telecommunications",
    "consumer_electronics"
  ],
  "abstract": "The technology described herein provides a cloud reinforcement-learning architecture that allows a single reinforcement-learning model to interact with multiple live software environments. The live software environments and the single reinforcement-learning model run in a distributed computing environment (e.g., cloud environment). The single reinforcement-learning model may run on a first computing device(s) with a graphical processing unit (GPU) to aid in training the single reinforcement-learning model. At a high level, the single reinforcement-learning model may receive state telemetry data from the multiple live environments. The single reinforcement-learning model selects an available action for each set of state telemetry data received and communicates the selection to appropriate the test agent. The test agent then facilitates completion of the action within the software instance being tested in the live environment. A reward is then determined for the action. The single reinforcement-learning model may then be trained.",
  "url": "https://patentbrief.org/patent/us/12001320/cloud-architecture-for-reinforcement-learning",
  "markdown_url": "https://patentbrief.org/patent/us/12001320/cloud-architecture-for-reinforcement-learning/md",
  "google_patents_url": "https://patents.google.com/patent/US12001320",
  "relatedPatents": [
    {
      "patentNumber": "11238372",
      "countryCode": "US",
      "title": "How AI Learns to Manage Computer Applications Using Digital Simulators",
      "url": "https://patentbrief.org/patent/us/11238372/simulator-training-for-automated-reinforcement-learning-based-application-manage"
    },
    {
      "patentNumber": "10963313",
      "countryCode": "US",
      "title": "How Computer Systems Learn to Manage Themselves and Fix Their Goals",
      "url": "https://patentbrief.org/patent/us/10963313/automated-reinforcement-learning-based-application-manager-that-learns-and-impro"
    },
    {
      "patentNumber": "11836577",
      "countryCode": "US",
      "title": "Training Robot AI Models Faster Using Smart Simulations",
      "url": "https://patentbrief.org/patent/us/11836577/reinforcement-learning-model-training-through-simulation"
    },
    {
      "patentNumber": "11429762",
      "countryCode": "US",
      "title": "How Computers Train AI Models Using Separate Virtual Simulations",
      "url": "https://patentbrief.org/patent/us/11429762/simulation-orchestration-for-training-reinforcement-learning-models"
    },
    {
      "patentNumber": "9754221",
      "countryCode": "US",
      "title": "A Computer Chip for Training Many AI Agents at Once",
      "url": "https://patentbrief.org/patent/us/9754221/processor-for-implementing-reinforcement-learning-operations"
    }
  ]
}