{
  "patent_number": "US 10599957",
  "country": "US",
  "title": "How to Automatically Detect and Fix Changes in AI Model Data",
  "original_title": "Systems and methods for detecting data drift for data used in machine learning models",
  "summary": "This patent describes a system that automatically notices when the real-world data an AI model sees changes, causing its predictions to become less accurate, and then fixes the model.",
  "what_it_does": "The system detects when the data used by a machine learning model changes over time, a problem called 'data drift'. It does this by first receiving model training data and generating a predictive model. Then, it takes new model input data to generate predicted data. Crucially, it receives 'event data' (real-world outcomes) and compares a 'data profile' of the predicted data to a data profile of the event data (Claim 1). If these profiles differ significantly, indicating data drift, the system then automatically corrects the model. For example, a credit card fraud detection model might be trained on past transaction data. If new types of fraud emerge, the system would compare the model's fraud predictions (predicted data) with actual confirmed fraud cases (event data) to see if the patterns have shifted. If drift is detected, the system could retrain the model using the newer event data (Claim 13) or adjust its internal settings, called hyperparameters (Claim 9).",
  "what_it_does_not_cover": [
    "Does not cover detecting data drift without comparing the *profile* of predicted data to the *profile* of real-world event data, as specified in Claim 1.",
    "Does not cover systems that only detect data drift but do not automatically initiate a correction of the model, as 'correcting the model' is a required step in Claim 1.",
    "Does not cover detecting data drift based solely on changes in the input data *before* it's processed by the model, without considering the model's predictions or real-world outcomes (event data).",
    "Does not cover detecting data drift in models that are not 'predictive models', as the claims consistently refer to this specific type.",
    "Does not cover methods of drift detection that do not involve receiving 'event data' for comparison with predicted data."
  ],
  "filed": "2018-10-26",
  "granted": "2020-03-24",
  "expires": "2038-10-26",
  "status": "active",
  "holder": "Capital One Services",
  "holder_url": "https://patentbrief.org/company/capital-one-services",
  "inventors": [
    {
      "name": "Austin Walters",
      "url": "https://patentbrief.org/inventor/austin-walters"
    },
    {
      "name": "Kate Key",
      "url": "https://patentbrief.org/inventor/kate-key"
    },
    {
      "name": "Mark Watson",
      "url": "https://patentbrief.org/inventor/mark-watson"
    },
    {
      "name": "Jeremy Goodsitt",
      "url": "https://patentbrief.org/inventor/jeremy-goodsitt"
    },
    {
      "name": "Vincent Pham",
      "url": "https://patentbrief.org/inventor/vincent-pham"
    },
    {
      "name": "Anh Truong",
      "url": "https://patentbrief.org/inventor/anh-truong"
    },
    {
      "name": "Reza Farivar",
      "url": "https://patentbrief.org/inventor/reza-farivar"
    },
    {
      "name": "Fardin Abdi Taghi Abad",
      "url": "https://patentbrief.org/inventor/fardin-abdi-taghi-abad"
    }
  ],
  "times_cited": 32,
  "tags": [
    "software",
    "ai_ml",
    "finance",
    "telecommunications",
    "consumer_electronics"
  ],
  "abstract": "A system and method for detecting data drift is disclosed. The system may be configured to perform a method, the method including receiving model training data and generating a predictive model. Generating the predictive model may include model training or hyperparameter tuning. The method may include receiving model input data and generating predicted data using the predictive model, based on the model input data. The method may include receiving event data and detecting data drift based on the predicted data and the event data. The method may include receiving current data and detecting data drift based on the data profile of the current data. The method may include model training and detecting data drift based on a difference in a trained model parameter from a baseline model parameter. The method may include hyperparameter tuning and detecting data drift based on a difference in a tuned hyperparameter from a baseline hyperparameter. The method may include correcting the model based on the detected data drift.",
  "url": "https://patentbrief.org/patent/us/10599957/systems-and-methods-for-detecting-data-drift-for-data-used-in-machine-learning-m",
  "markdown_url": "https://patentbrief.org/patent/us/10599957/systems-and-methods-for-detecting-data-drift-for-data-used-in-machine-learning-m/md",
  "google_patents_url": "https://patents.google.com/patent/US10599957",
  "relatedPatents": [
    {
      "patentNumber": "12518214",
      "countryCode": "US",
      "title": "Training AI on Private Data Without Seeing It",
      "url": "https://patentbrief.org/patent/us/12518214/distributed-machine-learning-systems-including-generation-of-synthetic-data"
    },
    {
      "patentNumber": "10824959",
      "countryCode": "US",
      "title": "How to Make Artificial Intelligence Explain Its Own Decisions",
      "url": "https://patentbrief.org/patent/us/10824959/explainers-for-machine-learning-classifiers"
    },
    {
      "patentNumber": "12438891",
      "countryCode": "US",
      "title": "How Multiple AI Models Detect Unusual Behavior on Computer Networks",
      "url": "https://patentbrief.org/patent/us/12438891/anomaly-detection-based-on-ensemble-machine-learning-model"
    },
    {
      "patentNumber": "11606373",
      "countryCode": "US",
      "title": "AI System That Learns Normal Email Use to Spot and Stop Cyber Threats",
      "url": "https://patentbrief.org/patent/us/11606373/cyber-threat-defense-system-protecting-email-networks-with-machine-learning-mode"
    },
    {
      "patentNumber": "10410117",
      "countryCode": "US",
      "title": "How to Save and Reuse Skills Learned by Artificial Intelligence Hardware",
      "url": "https://patentbrief.org/patent/us/10410117/method-and-a-system-for-creating-dynamic-neural-function-libraries"
    }
  ]
}