From 07324b56d55a675813a3732dc9d9dd1927c7ff85 Mon Sep 17 00:00:00 2001 From: David Hou Date: Fri, 15 Mar 2024 23:06:50 -0700 Subject: [PATCH] [experimenting] use contiguous instead of realize in optim (#3770) * run CI * comment * remove t.grad to try * Revert "remove t.grad to try" This reverts commit 05ec2d3b89b55b784020e3823aaf05451730c48c. --- tinygrad/nn/optim.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tinygrad/nn/optim.py b/tinygrad/nn/optim.py index 74f3d1297a..aa1d3f40a9 100644 --- a/tinygrad/nn/optim.py +++ b/tinygrad/nn/optim.py @@ -34,10 +34,9 @@ class SGD(Optimizer): def step(self) -> None: for i, t in enumerate(self.params): assert t.grad is not None - # this is needed since the grads can form a "diamond" + # contiguous is needed since the grads can allegedly form a "diamond" # TODO: fix this in lazy.py - t.grad.realize() - g = t.grad + self.wd * t.detach() + g = t.grad.contiguous() + self.wd * t.detach() if self.momentum: self.b[i].assign(self.momentum * self.b[i] + g) # NOTE: self.b[i] is zero on the first run, no if required g = (g + self.momentum * self.b[i]) if self.nesterov else self.b[i]