From 30b795874abbbfd767e9d94bb7746641ffe0d57a Mon Sep 17 00:00:00 2001 From: George Hotz Date: Mon, 20 Mar 2023 12:31:34 -0700 Subject: [PATCH] remove RMSprop, nobody uses it anymore --- README.md | 2 +- test/models/test_mnist.py | 7 ------- test/test_optim.py | 10 ++-------- tinygrad/nn/optim.py | 22 ---------------------- 4 files changed, 3 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 11028fad76..47bf371a88 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ Change to `DEBUG=4` to see the generated code. ## Neural networks? -It turns out, a decent autograd tensor library is 90% of what you need for neural networks. Add an optimizer (SGD, RMSprop, and Adam implemented) from tinygrad.nn.optim, write some boilerplate minibatching code, and you have all you need. +It turns out, a decent autograd tensor library is 90% of what you need for neural networks. Add an optimizer (SGD, Adam, AdamW implemented) from tinygrad.nn.optim, write some boilerplate minibatching code, and you have all you need. ### Neural network example (from test/models/test_mnist.py) diff --git a/test/models/test_mnist.py b/test/models/test_mnist.py index 2bc687a92e..ec1dd8554a 100644 --- a/test/models/test_mnist.py +++ b/test/models/test_mnist.py @@ -108,12 +108,5 @@ class TestMNIST(unittest.TestCase): train(model, X_train, Y_train, optimizer, steps=600) assert evaluate(model, X_test, Y_test) > 0.94 # CPU gets 0.9494 sometimes - def test_rmsprop(self): - np.random.seed(1337) - model = TinyBobNet() - optimizer = optim.RMSprop(model.parameters(), lr=0.0002, alpha=0.9) - train(model, X_train, Y_train, optimizer, steps=400) - assert evaluate(model, X_test, Y_test) > 0.95 - if __name__ == '__main__': unittest.main() diff --git a/test/test_optim.py b/test/test_optim.py index 23ff404b26..858de084c8 100644 --- a/test/test_optim.py +++ b/test/test_optim.py @@ -2,7 +2,7 @@ import numpy as np import torch import unittest from tinygrad.tensor import Tensor -from tinygrad.nn.optim import Adam, SGD, RMSprop, AdamW +from tinygrad.nn.optim import Adam, SGD, AdamW np.random.seed(1337) x_init = np.random.randn(1,4).astype(np.float32) @@ -39,7 +39,6 @@ class TestOptim(unittest.TestCase): np.testing.assert_allclose(x, y, atol=atol, rtol=rtol) def _test_sgd(self, steps, opts, atol, rtol): self._test_optim(SGD, torch.optim.SGD, steps, opts, atol, rtol) - def _test_rmsprop(self, steps, opts, atol, rtol): self._test_optim(RMSprop, torch.optim.RMSprop, steps, opts, atol, rtol) def _test_adam(self, steps, opts, atol, rtol): self._test_optim(Adam, torch.optim.Adam, steps, opts, atol, rtol) def _test_adamw(self, steps, opts, atol, rtol): self._test_optim(AdamW, torch.optim.AdamW, steps, opts, atol, rtol) @@ -55,19 +54,14 @@ class TestOptim(unittest.TestCase): def test_multistep_sgd_nesterov_momentum(self): self._test_sgd(10, {'lr': 0.001, 'momentum': 0.9, 'nesterov': True}, 1e-5, 0) def test_multistep_sgd_high_lr_nesterov_momentum(self): self._test_sgd(10, {'lr': 10, 'momentum': 0.9, 'nesterov': True}, 1e-5, 3e-4) - def test_rmsprop(self): self._test_rmsprop(1, {'lr': 0.001, 'alpha': 0.99}, 1e-5, 0) - def test_rmsprop_high_lr(self): self._test_rmsprop(1, {'lr': 10, 'alpha': 0.99}, 1e-5, 1e-5) def test_adam(self): self._test_adam(1, {'lr': 0.001}, 1e-5, 0) def test_adam_high_lr(self): self._test_adam(1, {'lr': 10}, 1e-5, 1e-5) def test_adamw(self): self._test_adamw(1, {'lr': 0.001}, 1e-5, 0) def test_adamw_high_lr(self): self._test_adamw(1, {'lr': 10}, 1e-5, 1e-5) - def test_multistep_rmsprop(self): self._test_rmsprop(10, {'lr': 0.001}, 1e-5, 0) - def test_multistep_rmsprop_high_lr(self): self._test_rmsprop(10, {'lr': 10}, 1e-5, 3e-4) - def test_multistep_adam(self): self._test_adam(10, {'lr': 0.001}, 1e-5, 0) def test_multistep_adam_high_lr(self): self._test_adam(10, {'lr': 10}, 1e-5, 3e-4) - + def test_multistep_adamw(self): self._test_adamw(10, {'lr': 0.001}, 1e-5, 0) def test_multistep_adamw_high_lr(self): self._test_adamw(10, {'lr': 10}, 1e-5, 3e-4) diff --git a/tinygrad/nn/optim.py b/tinygrad/nn/optim.py index d6a2cd15c2..e0e4a2c411 100644 --- a/tinygrad/nn/optim.py +++ b/tinygrad/nn/optim.py @@ -11,13 +11,6 @@ class Optimizer: self.params: List[Tensor] = [x for x in params if x.requires_grad] self.buffers: List[Tensor] = [x for x in params if not x.requires_grad] # buffers are still realized - # TODO: this probably shouldn't change the gradients, just the ones used by the optimizer - def clipnorm(self, amount=1): - for param in self.params: - assert param.grad is not None - # clipnorm is the L2 norm, not value: is this right? - param.grad.assign(param.grad.clip(-(amount**2), (amount**2))) - def zero_grad(self): for param in self.params: param.grad = None @@ -44,21 +37,6 @@ class SGD(Optimizer): t.assign(t.detach() - g * self.lr) self.realize(self.b) -class RMSprop(Optimizer): - def __init__(self, params: List[Tensor], lr=0.001, alpha=0.99, eps=1e-8): - super().__init__(params) - self.lr, self.alpha, self.eps = lr, alpha, eps - - self.v = [Tensor.zeros(*t.shape, device=t.device, requires_grad=False) for t in self.params] - - def step(self) -> None: - for i, t in enumerate(self.params): - assert t.grad is not None - g = t.grad.realize() - self.v[i].assign(self.alpha * self.v[i] + (1.0 - self.alpha) * (g * g)).realize() - t.assign(t.detach() - (g * self.lr).div(self.v[i].sqrt() + self.eps)) - self.realize(self.v) - class AdamW(Optimizer): def __init__(self, params: List[Tensor], lr=0.001, b1=0.9, b2=0.999, eps=1e-8, wd=0.01): super().__init__(params)