{
  "patent_number": "US 11429864",
  "country": "US",
  "title": "Making Neural Networks Faster by Skipping Unnecessary Calculations",
  "original_title": "System and method for bank-balanced sparse activation and joint-activation-weight-sparse training of neural networks",
  "summary": "A method to speed up AI training by keeping data sparse, meaning it ignores zeros to save memory and processing power during both forward and backward passes.",
  "what_it_does": "This patent describes a way to train neural networks more efficiently by using 'sparse' data structures. Instead of calculating every single value in a neural network, which involves many zeros that don't change the outcome, the system keeps tensors (multidimensional arrays of numbers) sparse by removing or ignoring these zeros. During the forward pass, it generates a dense output but immediately sparsifies it. During the backward pass, it uses these sparse tensors to calculate gradients, ensuring that the training process only focuses on the most significant data points. It specifically uses a 'top-K' activation function to decide which values are important enough to keep, effectively pruning the network during the training process itself.",
  "what_it_does_not_cover": [
    "Does not cover training methods that do not use sparse tensors or sparsity-based pruning.",
    "Does not cover standard dense neural network training where all weights are updated equally.",
    "Does not cover hardware-specific implementations that do not utilize the described bank-based top-K selection logic.",
    "Does not cover non-neural network machine learning models like linear regression or decision trees."
  ],
  "filed": "2021-08-16",
  "granted": "2022-08-30",
  "expires": null,
  "status": "active",
  "holder": "Moffett International Co Ltd Hong Kong",
  "holder_url": "https://patentbrief.org/company/moffett-international-co-ltd-hong-kong",
  "inventors": [
    {
      "name": "Enxu Yan",
      "url": "https://patentbrief.org/inventor/enxu-yan"
    }
  ],
  "times_cited": 1,
  "tags": [
    "ai_ml",
    "semiconductors",
    "software"
  ],
  "abstract": "Methods, systems, and apparatus, including computer programs encoded on computer storage media, for optimizing neural network training, are described. The method may include: during a forward propagation at a current layer of a neural network, generating, based on a sparse input tensor and a sparse weight tensor of the current layer, a dense output tensor, and sparsifying the dense output tensor to obtain a sparse output tensor; during a backward propagation at the current layer of the neural network: determining a first sparse derivative tensor based on the sparse output tensor, obtaining a dense derivative tensor based on the first sparse derivative tensor and the sparse weight tensor of the current layer, and sparsifying the dense derivative tensor to obtain a second sparse derivative tensor; and training weight tensors of the neural network based on the first sparse derivative tensor and the second sparse derivative tensor.",
  "url": "https://patentbrief.org/patent/us/11429864/dlss-deep-learning-super-sampling",
  "markdown_url": "https://patentbrief.org/patent/us/11429864/dlss-deep-learning-super-sampling/md",
  "google_patents_url": "https://patents.google.com/patent/US11429864",
  "relatedPatents": []
}