From 2aa6151b09e1874631284173d863f0e079ab6aea Mon Sep 17 00:00:00 2001 From: bhavnicksm Date: Sat, 4 Mar 2023 23:33:19 +0530 Subject: [PATCH 01/10] Add BaseOptimiser file --- src/nadir/BaseOptimiser.py | 36 ---------- src/nadir/base.py | 132 +++++++++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+), 36 deletions(-) delete mode 100644 src/nadir/BaseOptimiser.py create mode 100644 src/nadir/base.py diff --git a/src/nadir/BaseOptimiser.py b/src/nadir/BaseOptimiser.py deleted file mode 100644 index e18b524..0000000 --- a/src/nadir/BaseOptimiser.py +++ /dev/null @@ -1,36 +0,0 @@ -### Copyright 2023 [Dawn Of Eve] - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Dict, Tuple, Any, Optional - -import torch -from torch.optim.optimizer import Optimizer - - -class BaseConfig(): - pass - - -class BaseOptimizer(Optimizer): - def __init__( - self, - params, - config : BaseConfig, - defaults: Dict[str, Any] - - - ): - defaults = config.__dict__ - - super().__init__(params, defaults) \ No newline at end of file diff --git a/src/nadir/base.py b/src/nadir/base.py new file mode 100644 index 0000000..7b62c99 --- /dev/null +++ b/src/nadir/base.py @@ -0,0 +1,132 @@ +### Copyright 2023 [Dawn Of Eve] + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, Tuple, Any, Optional +from dataclasses import dataclass + +import torch +from torch.optim.optimizer import Optimizer + +__all__ = ['BaseConfig', 'BaseOptimizer'] + +@dataclass +class BaseConfig: + lr : float = 1E-3 + momentum : bool = False + adaptive : bool = False + beta_1 : float = 0.0 + beta_2 : float = 0.0 + eps : float = 1E-8 + + def dict(self): + return self.__dict__ + + +class BaseOptimizer(Optimizer): + + def __init__ (self, params, config: BaseConfig = BaseConfig()): + if not config.lr > 0.0: + raise ValueError(f"Invalid value for lr in config: {config.lr} ", + "Value must be > 0") + if not 1.0 > config.beta_1 >= 0.0: + raise ValueError(f"Invalid value for beta_1 in config: {config.beta_1} ", + "Value must be 1 > x >= 0") + if not 1.0 > config.beta_2 >= 0.0: + raise ValueError(f"Invalid value for beta_2 in config: {config.beta_2} ", + "Value must be 1 > x >= 0") + super().__init__(params, config.dict()) + + self.config = config + + def init_state(self, + state, + group, + param): + + if self.config.momentum: + state['momentum_step'] = 0 + state['momentum'] = torch.zeros_like(param, memory_format=torch.preserve_format) + if self.config.adaptive: + state['adaptive_step'] = 0 + state['adaptivity'] = torch.zeros_like(param, memory_format=torch.preserve_format) + + def momentum(self, + state, + grad): + step = state['momentum_step'] + m = state['momentum'] + beta_1 = self.config.beta_1 + + m.mul_(beta_1).add_(grad, alpha= (1 - beta_1)) + m.div_(1 - beta_1**(step + 1)) + + state['momentum'] = m + state['momentum_step'] = step + 1 + return m + + def adaptivity(self, + state, + grad): + + step = state['adaptive_step'] + v = state['adaptivity'] + beta_2 = self.config.beta_2 + + v.mul_(beta_2).addcmul_(grad, grad, value = (1 - beta_2)) + v.div_(1 - beta_2**(step + 1)) + + state['adaptivity'] = v + state['step'] = step + 1 + return torch.sqrt(v + self.config.eps) + + def update(self, + state: Dict[str, any], + group: Dict[str, any], + grad: torch.Tensor, + param: torch.Tensor): + + lr = group['lr'] + + if self.config.momentum: + m = self.momentum(state, grad) + upd = m + else: + upd = grad + + if self.config.adaptive: + v = self.adaptivity(state, grad) + param.data.addcdiv_(upd, v, value = -1 * lr) + else: + param.data.add_(upd, alpha = -1 * lr) + + @torch.no_grad() + def step(self, closure = None): + loss = None + if closure is not None: + with torch.enable_grad(): + loss = closure() + + for group in self.param_groups: + for param in group['params']: + if param.grad is None: + continue + grad = param.grad.data + if grad.is_sparse: + raise RuntimeError('This Optimizer does not support sparse gradients,' + ' please consider SparseAdam instead') + state = self.state[param] + if len(state) == 0: + self.init_state(state, group, param) + self.update(state, group, grad, param) + return loss \ No newline at end of file From 74eeb31b3a3c766cf98b6b9d2d99d2df42847ff7 Mon Sep 17 00:00:00 2001 From: bhavnicksm Date: Sat, 4 Mar 2023 23:33:36 +0530 Subject: [PATCH 02/10] Add SGD --- src/nadir/sgd.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 src/nadir/sgd.py diff --git a/src/nadir/sgd.py b/src/nadir/sgd.py new file mode 100644 index 0000000..4a6e11d --- /dev/null +++ b/src/nadir/sgd.py @@ -0,0 +1,33 @@ +### Copyright 2023 [Dawn Of Eve] + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Dict, Any, Optional +from dataclasses import dataclass + +import torch + +from .base import BaseOptimizer +from .base import BaseConfig + + +__all__ = ['SGDConfig', 'SGD'] + +@dataclass +class SGDConfig(BaseConfig): + lr : float = 1E-3 + + +class SGD(BaseOptimizer): + + def __init__(self, params, config: SGDConfig = SGDConfig()): + super().__init__(params, config) \ No newline at end of file From ce5ebdeda6b56aecd84fd1afb8472ba12e796949 Mon Sep 17 00:00:00 2001 From: bhavnicksm Date: Sat, 4 Mar 2023 23:33:48 +0530 Subject: [PATCH 03/10] Add momentum --- src/nadir/momentum.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 src/nadir/momentum.py diff --git a/src/nadir/momentum.py b/src/nadir/momentum.py new file mode 100644 index 0000000..a13c3cd --- /dev/null +++ b/src/nadir/momentum.py @@ -0,0 +1,38 @@ +### Copyright 2023 [Dawn Of Eve] + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Dict, Any, Optional +from dataclasses import dataclass + +import torch + +from .base import BaseOptimizer +from .base import BaseConfig + + +__all__ = ['MomentumConfig', 'Momentum'] + +@dataclass +class MomentumConfig(BaseConfig): + lr : float = 1E-3 + momentum : bool = True + beta_1 : float = 0.95 + +class Momentum(BaseOptimizer): + + def __init__(self, params, config : MomentumConfig = MomentumConfig()): + if not config.momentum: + raise ValueError(f"Invalid value for momentum in config: {config.momentum} ", + "Value must be True") + super().__init__(params, config) + self.config = config \ No newline at end of file From fa2ef5a138914b1161d045ee73e9c213ccbff61e Mon Sep 17 00:00:00 2001 From: bhavnicksm Date: Sat, 4 Mar 2023 23:33:57 +0530 Subject: [PATCH 04/10] Add RMSProp --- src/nadir/rmsprop.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 src/nadir/rmsprop.py diff --git a/src/nadir/rmsprop.py b/src/nadir/rmsprop.py new file mode 100644 index 0000000..4d304b0 --- /dev/null +++ b/src/nadir/rmsprop.py @@ -0,0 +1,43 @@ +### Copyright 2023 [Dawn Of Eve] + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, Any, Optional +from dataclasses import dataclass + +import torch + +from .base import BaseOptimizer +from .base import BaseConfig + + +__all__ = ['RMSPropConfig', 'RMSProp'] + +@dataclass +class RMSPropConfig(BaseConfig): + lr : float = 1E-3 + adaptive : bool = True + beta_2 : float = 0.9 + eps : float = 1E-8 + + + +class RMSProp(BaseOptimizer): + + def __init__ (self, params, config:RMSPropConfig = RMSPropConfig()): + if not config.adaptive: + raise ValueError(f"Invalid value for adaptive in config: {config.adaptive} ", + "Value must be True") + + super().__init__(params, config) + self.config = config \ No newline at end of file From 3189998131297391a9279473438aeb624cfa51fb Mon Sep 17 00:00:00 2001 From: bhavnicksm Date: Sat, 4 Mar 2023 23:34:05 +0530 Subject: [PATCH 05/10] Add Adam --- src/nadir/adam.py | 88 ++++++++++++++--------------------------------- 1 file changed, 25 insertions(+), 63 deletions(-) diff --git a/src/nadir/adam.py b/src/nadir/adam.py index e87ed59..12bb66f 100644 --- a/src/nadir/adam.py +++ b/src/nadir/adam.py @@ -12,76 +12,38 @@ # See the License for the specific language governing permissions and # limitations under the License. -import math -import torch +from typing import Dict, Any, Optional from dataclasses import dataclass -from .BaseOptimiser import BaseOptimizer -from .BaseOptimiser import DoEConfig -from typing import Dict, Tuple, Any, Optional - -__all__ = ['Adam'] - -@dataclass -class AdamConfig(DoEConfig): - lr : float = 1e-3 - betas: Tuple[float, float] = (0.9, 0.99) - eps: float = 1e-16 - - - -class Adam(BaseOptimizer): - def __init__(self, params, config: AdamConfig, defaults: Optional[Dict[str, Any]] = None): - if not 0.0 <= config.lr: - raise ValueError(f"Invalid learning rate: {config.lr}") - if not 0.0 <= config.eps: - raise ValueError(f"Invalid epsilon value: {config.eps}") - if not 0.0 <= config.betas[0] < 1.0: - raise ValueError(f"Invalid beta parameter at index 0: {config.betas[0]}") - if not 0.0 <= config.betas[1] < 1.0: - raise ValueError(f"Invalid beta parameter at index 1:{config.betas[1]}") - defaults = {} if defaults is None else defaults - super().__init__(params, config, defaults) - - - def get_mv(self, state: Dict[str, Any], group: Dict[str, Any], grad: torch.Tensor): - beta1, beta2 = group['betas'] - m, v = state['exp_avg'], state['exp_avg_sq'] - m.mul_(beta1).add_(grad, alpha=1 - beta1) - v.mul_(beta2).addcmul_(grad, grad, value=1 - beta2) - return m, v - +import torch - def get_lr(self, state: Dict[str, any], group: Dict[str, any]): - return group['lr'] +from .base import BaseOptimizer +from .base import BaseConfig +__all__ = ['AdamConfig', 'Adam'] - @torch.no_grad() - def step(self): - loss = None - for group in self.param_groups: - # weight_decay = group['weight_decay'] - beta1, beta2 = group['betas'] - eps = group['eps'] +@dataclass +class AdamConfig(BaseConfig): + lr : float = 3E-4 + momentum : bool = True + adaptive : bool = True + beta_1 : float = 0.9 + beta_2 : float = 0.999 + eps : float = 1E-8 - for p in group['params']: - if p.grad is None: - continue - d_p = p.grad.data - param_state = self.state[p] - param_state['step'] = 0 - param_state['exp_avg'] = torch.zeros_like(p, memory_format=torch.preserve_format) - param_state['exp_avg_sq'] = torch.zeros_like(p, memory_format=torch.preserve_format) - m, v = self.get_mv(param_state, group, d_p) - param_state['step'] += 1 - bias_correction1 = 1 - beta1 ** param_state['step'] - bias_correction2 = 1 - beta2 ** param_state['step'] - lr = self.get_lr(param_state, group) - denominator = v.sqrt().add_(eps) - step_size = lr * math.sqrt(bias_correction2) / bias_correction1 - p.data.addcdiv_(m, denominator, value=-step_size) - return loss +class Adam(BaseOptimizer): + def __init__(self, params, config : AdamConfig = AdamConfig()): + if not config.momentum: + raise ValueError(f"Invalid value for momentum in config: {config.momentum} ", + "Value must be True") + if not config.adaptive: + raise ValueError(f"Invalid value for adaptive in config: {config.adaptive} ", + "Value must be True") + + super().__init__(params, config) + + self.config = config \ No newline at end of file From 3930054a1e7addc1a0b69f3e154885eb889af99d Mon Sep 17 00:00:00 2001 From: bhavnicksm Date: Sat, 4 Mar 2023 23:34:13 +0530 Subject: [PATCH 06/10] Add Adamax --- src/nadir/adamax.py | 53 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 src/nadir/adamax.py diff --git a/src/nadir/adamax.py b/src/nadir/adamax.py new file mode 100644 index 0000000..92fa1bf --- /dev/null +++ b/src/nadir/adamax.py @@ -0,0 +1,53 @@ +### Copyright 2023 [Dawn Of Eve] + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, Any, Optional +from dataclasses import dataclass + +import torch + +from .base import BaseOptimizer +from .base import BaseConfig + + +__all__ = ['AdamaxConfig', 'Adamax'] + +@dataclass +class AdamaxConfig(BaseConfig): + lr : float = 2E-3 + momentum : bool = True + adaptive : bool = True + beta_1 : float = 0.9 + beta_2 : float = 0.999 + eps : float = 1E-8 + + +class Adamax(BaseOptimizer): + def __init__ (self, params, config : AdamaxConfig = AdamaxConfig()): + if not config.momentum: + raise ValueError(f"Invalid value for momentum in config: {config.momentum} ", + "Value must be True") + if not config.adaptive: + raise ValueError(f"Invalid value for adaptive in config: {config.adaptive} ", + "Value must be True") + + super().__init__(params, config) + self.config = config + + def adaptivity(self, state, grad): + u = state['adaptivity'] + beta_2 = self.config.beta_2 + u = torch.max(torch.mul(u, beta_2), torch.abs(grad) + self.config.eps) + state['adaptivity'] = u + return u \ No newline at end of file From 29531c4cb5731634ebe415181bf758fbe189d6ef Mon Sep 17 00:00:00 2001 From: bhavnicksm Date: Sat, 4 Mar 2023 23:34:22 +0530 Subject: [PATCH 07/10] Add Adagrad --- src/nadir/adagrad.py | 48 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 src/nadir/adagrad.py diff --git a/src/nadir/adagrad.py b/src/nadir/adagrad.py new file mode 100644 index 0000000..8168902 --- /dev/null +++ b/src/nadir/adagrad.py @@ -0,0 +1,48 @@ +### Copyright 2023 [Dawn Of Eve] + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Dict, Any, Optional +from dataclasses import dataclass + +import torch + +from .base import BaseOptimizer +from .base import BaseConfig + + +__all__ = ['AdagradConfig', 'Adagrad'] + +@dataclass +class AdagradConfig(BaseConfig): + lr : float = 1E-3 + adaptive : bool = True + eps : float = 1E-8 + +class Adagrad(BaseOptimizer): + + def __init__ (self, params, config : AdagradConfig = AdagradConfig()): + if not config.adaptive: + raise ValueError(f"Invalid value for adaptive in config: {config.adaptive} ", + "Value must be True") + super().__init__(params, config) + self.config = config + + def adaptivity(self, + state, + grad): + + v = state['adaptivity'] + v.add_(torch.pow(grad, 2)) + state['adaptivity'] = v + + return torch.sqrt(v + self.config.eps) \ No newline at end of file From 8bcbac8629d31156c43bfa56161b6e733735a1ab Mon Sep 17 00:00:00 2001 From: bhavnicksm Date: Sat, 4 Mar 2023 23:34:29 +0530 Subject: [PATCH 08/10] Add adadelta --- src/nadir/adadelta.py | 60 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 src/nadir/adadelta.py diff --git a/src/nadir/adadelta.py b/src/nadir/adadelta.py new file mode 100644 index 0000000..ccb9a05 --- /dev/null +++ b/src/nadir/adadelta.py @@ -0,0 +1,60 @@ +### Copyright 2023 [Dawn Of Eve] + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Dict, Any, Optional +from dataclasses import dataclass + +import torch + +from .base import BaseOptimizer +from .base import BaseConfig + + +__all__ = ['AdadeltaConfig', 'Adadelta'] + +@dataclass +class AdadeltaConfig(BaseConfig): + lr : float = 1 + adaptive : bool = True + rho : float = 0.90 + beta_2 : float = 0.90 + eps : float = 1E-6 + + +class Adadelta(BaseOptimizer): + def __init__ (self, params, config : AdadeltaConfig = AdadeltaConfig()): + super().__init__(params, config) + + self.config = config + if self.config.rho != self.config.beta_2: + self.config.beta_2 = self.config.rho + + def init_state(self, state, group, param): + state['adaptive_step'] = 0 + state['adaptivity'] = torch.zeros_like(param, memory_format=torch.preserve_format) + state['acc_delta'] = torch.zeros_like(param, memory_format=torch.preserve_format) + + def update(self, state, group, grad, param): + eps = self.config.eps + rho = self.config.rho + lr = group['lr'] + m = state['acc_delta'] + + denom = self.adaptivity(state, grad) + + delta = m.add(eps).sqrt_().div_(denom).mul_(grad) + + param.data.add_(delta, alpha = -1 * lr) + + m.mul_(rho).addcmul_(delta, delta, value=(1 - rho)) + state['acc_delta'] = m \ No newline at end of file From b09ecd3d5b144a0c801b11d0200eaf2f83acd5f6 Mon Sep 17 00:00:00 2001 From: bhavnicksm Date: Sat, 4 Mar 2023 23:34:37 +0530 Subject: [PATCH 09/10] List optimizers in __all__ --- src/nadir/__init__.py | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/src/nadir/__init__.py b/src/nadir/__init__.py index 18aadc4..aeb70f2 100644 --- a/src/nadir/__init__.py +++ b/src/nadir/__init__.py @@ -14,11 +14,33 @@ from typing import Dict, List, Type from torch.optim.optimizer import Optimizer -from .BaseOptimiser import BaseOptimizer, BaseConfig -from .SGD import SGD, SGDConfig + +from .adadelta import Adadelta, AdadeltaConfig +from .adagrad import Adagrad, AdagradConfig from .adam import Adam, AdamConfig +from .adamax import Adamax, AdamaxConfig +from .base import BaseOptimizer, BaseConfig +from .momentum import Momentum, MomentumConfig +from .rmsprop import RMSProp, RMSPropConfig +from .sgd import SGD, SGDConfig + -__all__ = ('SGD', 'SGDConfig', 'Adam', 'AdamConfig') +__version__ = "0.0.1" -__version__ = "0.0.1.dev2" \ No newline at end of file +__all__ = ('Adadelta', + 'AdadeltaConfig', + 'Adagrad', + 'AdagradConfig', + 'Adam', + 'AdamConfig', + 'Adamax', + 'AdamaxConfig', + 'BaseOptimizer', + 'BaseConfig', + 'Momentum', + 'MomentumConfig', + 'RMSProp', + 'RMSPropConfig', + 'SGD', + 'SGDConfig') From fbf3ff0e0a2e6d9789674e191244c7cd772ddceb Mon Sep 17 00:00:00 2001 From: bhavnicksm Date: Sat, 4 Mar 2023 23:35:05 +0530 Subject: [PATCH 10/10] Update README --- README.md | 15 ++++++-- examples/example.ipynb | 31 +++++++++------- pyproject.toml | 2 +- src/nadir/SGD.py | 83 ------------------------------------------ 4 files changed, 29 insertions(+), 102 deletions(-) delete mode 100644 src/nadir/SGD.py diff --git a/README.md b/README.md index a0546ce..5c08a9d 100644 --- a/README.md +++ b/README.md @@ -16,10 +16,17 @@ PyTorch is a popular machine learning framework that provides a flexible and eff ## Supported Optimisers -| Optimiser | Paper | -|:---------: |:-----: | -| **SGD** | | -| **Adam** | | +| Optimiser | Paper | +|:---------: |:-----: | +| **SGD** | https://paperswithcode.com/method/sgd | +| **Momentum** | https://paperswithcode.com/method/sgd-with-momentum | +| **Adagrad** | https://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf | +| **RMSProp** | https://paperswithcode.com/method/rmsprop | +| **Adam** | https://arxiv.org/abs/1412.6980v9 | +| **Adamax** | https://arxiv.org/abs/1412.6980v9 | +| **Adadelta** | https://arxiv.org/abs/1212.5701v1 | + + ## Installation diff --git a/examples/example.ipynb b/examples/example.ipynb index 84f457c..71c3430 100644 --- a/examples/example.ipynb +++ b/examples/example.ipynb @@ -6,14 +6,11 @@ "metadata": {}, "outputs": [ { - "ename": "ModuleNotFoundError", - "evalue": "No module named 'torch'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", - "\u001b[1;32m/home/ec3dev/personal/github/nadir/examples/example.ipynb Cell 1\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39mtorch\u001b[39;00m\n\u001b[1;32m 2\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39margparse\u001b[39;00m\n\u001b[1;32m 3\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39mwandb\u001b[39;00m\n", - "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'torch'" + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/ec3dev/personal/github/pyenv/nadir/lib/python3.8/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" ] } ], @@ -34,15 +31,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ - "from nadir import tests\n", - "from nadir import nadir as optim\n", - "from nadir.tests import mnist" + "import nadir as nd\n", + "from nadir import SGD, SGDConfig" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "code", "execution_count": null, @@ -138,9 +141,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "nadir", "language": "python", - "name": "python3" + "name": "nadir" }, "language_info": { "codemirror_mode": { diff --git a/pyproject.toml b/pyproject.toml index 6851d7a..35f11c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "nadir" -version = "0.0.1.dev1" +version = "0.0.1" authors = [ { name="Bhavnick Minhas", email="bhavnicksm@gmail.com" }, ] diff --git a/src/nadir/SGD.py b/src/nadir/SGD.py deleted file mode 100644 index a9b7812..0000000 --- a/src/nadir/SGD.py +++ /dev/null @@ -1,83 +0,0 @@ -### Copyright 2023 [Dawn Of Eve] - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import torch -from dataclasses import dataclass - -from .BaseOptimiser import BaseOptimizer -from .BaseOptimiser import BaseConfig -from typing import Dict, Any, Optional - - -__all__ = ['SGD', 'sgd'] - -@dataclass -class SGDConfig(BaseConfig): - lr : float = 1e-3 - momentum: float = 0 - nesterov: bool = False - -class SGD(BaseOptimizer): - def __init__(self, params, config: SGDConfig, defaults: Optional[Dict[str, Any]] = None): - if not 0.0 <= config.lr: - raise ValueError(f"Invalid learning rate: {config.lr}") - if not 0.0 <= config.momentum < 1.0: - raise ValueError(f"Invalid momentum: {config.momentum}") - defaults = {} if defaults is None else defaults - super().__init__(params, config, defaults) - - - @torch.no_grad() - def step(self): - """Performs a single optimization step. - - Args: - closure (Callable, optional): A closure that reevaluates the model - and returns the loss. - """ - loss = None - # if closure is not None: - # with torch.enable_grad(): - # loss = closure() - - for group in self.param_groups: - # weight_decay = group['weight_decay'] - momentum = group['momentum'] - # dampening = group['dampening'] - nesterov = group['nesterov'] - - for p in group['params']: - if p.grad is None: - continue - d_p = p.grad.data - # if weight_decay != 0: - # d_p.add_(weight_decay, p.data) - # Apply learning rate - d_p.mul_(group['lr']) - if momentum != 0: - param_state = self.state[p] - if 'momentum_buffer' not in param_state: - buf = param_state['momentum_buffer'] = torch.zeros_like(p.data) - buf.mul_(momentum).add_(d_p) - # else: - # buf = param_state['momentum_buffer'] - # buf.mul_(momentum).add_(1 - dampening, d_p) - if nesterov: - d_p = d_p.add(momentum, buf) - else: - d_p = buf - - p.data.add_(d_p, alpha=-1) - - return loss \ No newline at end of file