Initial import: grid-bot — grid trading bot for BTC-USDT on Cifra Markets
This commit is contained in:
Kolp
+173
@@ -0,0 +1,173 @@
|
||||
# mypy: allow-untyped-defs
|
||||
|
||||
import warnings
|
||||
import weakref
|
||||
from functools import wraps
|
||||
|
||||
from torch.ao.pruning.sparsifier.base_sparsifier import BaseSparsifier
|
||||
|
||||
|
||||
__all__ = ["BaseScheduler"]
|
||||
|
||||
|
||||
class BaseScheduler:
|
||||
def __init__(self, sparsifier, last_epoch=-1, verbose=False):
|
||||
# Attach sparsifier
|
||||
if not isinstance(sparsifier, BaseSparsifier):
|
||||
raise TypeError(
|
||||
f"{type(sparsifier).__name__} is not an instance of torch.ao.pruning.BaseSparsifier"
|
||||
)
|
||||
self.sparsifier = sparsifier
|
||||
|
||||
# Initialize epoch and base sparsity levels
|
||||
|
||||
self.base_sl = [group["sparsity_level"] for group in sparsifier.groups]
|
||||
self.last_epoch = last_epoch
|
||||
|
||||
# Following https://github.com/pytorch/pytorch/issues/20124
|
||||
# We would like to ensure that `scheduler.step()` is called after
|
||||
# `sparsifier.step()`
|
||||
def with_counter(method):
|
||||
if getattr(method, "_with_counter", False):
|
||||
# `sparsifier.step()` has already been replaced, return.
|
||||
return method
|
||||
|
||||
# Keep a weak reference to the sparsifier instance to prevent
|
||||
# cyclic references.
|
||||
instance_ref = weakref.ref(method.__self__)
|
||||
# Get the unbound method for the same purpose.
|
||||
func = method.__func__
|
||||
cls = instance_ref().__class__
|
||||
del method
|
||||
|
||||
@wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
instance = instance_ref()
|
||||
instance._step_count += 1 # type: ignore[union-attr]
|
||||
wrapped = func.__get__(instance, cls)
|
||||
return wrapped(*args, **kwargs)
|
||||
|
||||
# Note that the returned function here is no longer a bound method,
|
||||
# so attributes like `__func__` and `__self__` no longer exist.
|
||||
wrapper._with_counter = True # type: ignore[attr-defined]
|
||||
return wrapper
|
||||
|
||||
self.sparsifier.step = with_counter(self.sparsifier.step) # type: ignore[assignment]
|
||||
self.sparsifier._step_count = 0 # type: ignore[attr-defined]
|
||||
self._step_count: int = 0
|
||||
self.verbose = verbose
|
||||
|
||||
# Housekeeping
|
||||
self._get_sl_called_within_step: bool = False
|
||||
|
||||
self.step()
|
||||
|
||||
def state_dict(self):
|
||||
"""Returns the state of the scheduler as a :class:`dict`.
|
||||
|
||||
It contains an entry for every variable in self.__dict__ which
|
||||
is not the sparsifier.
|
||||
"""
|
||||
return {
|
||||
key: value for key, value in self.__dict__.items() if key != "sparsifier"
|
||||
}
|
||||
|
||||
def load_state_dict(self, state_dict):
|
||||
"""Loads the schedulers state.
|
||||
|
||||
Args:
|
||||
state_dict (dict): scheduler state. Should be an object returned
|
||||
from a call to :meth:`state_dict`.
|
||||
"""
|
||||
self.__dict__.update(state_dict)
|
||||
|
||||
def get_last_sl(self):
|
||||
"""Return last computed sparsity level by current scheduler."""
|
||||
return self._last_sl
|
||||
|
||||
def get_sl(self):
|
||||
# Compute sparsity level using chainable form of the scheduler
|
||||
# Note: This method is not intended to be called directly, and is only
|
||||
# used by the ".step" method. Use .get_last_sl() instead.
|
||||
if not self._get_sl_called_within_step:
|
||||
warnings.warn(
|
||||
"To get the last sparsity level computed by the scheduler, "
|
||||
"please use `get_last_sl()`.",
|
||||
stacklevel=2,
|
||||
)
|
||||
raise NotImplementedError
|
||||
|
||||
def print_sl(self, is_verbose, group, sl, epoch=None):
|
||||
"""Display the current sparsity level."""
|
||||
if is_verbose:
|
||||
if epoch is None:
|
||||
print(f"Adjusting sparsity level of group {group} to {sl:.4e}.")
|
||||
else:
|
||||
print(
|
||||
f"Epoch {epoch:5d}: adjusting sparsity level of group {group} to {sl:.4e}."
|
||||
)
|
||||
|
||||
def __repr__(self):
|
||||
format_string = self.__class__.__name__ + " ("
|
||||
format_string += "\n"
|
||||
format_string += f"Sparsifier {self.sparsifier}\n"
|
||||
format_string += f" base_sl: {self.base_sl}\n"
|
||||
format_string += ")"
|
||||
return format_string
|
||||
|
||||
def step(self, epoch=None):
|
||||
# Raise warning if trying to call scheduler step before the sparsifier.
|
||||
# https://github.com/pytorch/pytorch/issues/20124
|
||||
if self._step_count == 1:
|
||||
if not hasattr(self.sparsifier.step, "_with_counter"):
|
||||
warnings.warn(
|
||||
"Seems like `sparsifier.step()` has been overridden after sparsity scheduler "
|
||||
"initialization. Please, make sure to call `sparsifier.step()` before "
|
||||
"`scheduler.step()`.",
|
||||
UserWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
# Just check if there were two first scheduler.step() calls before sparsifier.step()
|
||||
elif self.sparsifier._step_count < 1: # type: ignore[attr-defined]
|
||||
warnings.warn(
|
||||
"Detected call of `scheduler.step()` before `sparsifier.step()`. "
|
||||
"You have to make sure you run the sparsifier.step() BEFORE any "
|
||||
"calls to the scheduler.step().",
|
||||
UserWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
self._step_count += 1
|
||||
|
||||
class _enable_get_sl_call:
|
||||
def __init__(self, o):
|
||||
self.o = o
|
||||
|
||||
def __enter__(self):
|
||||
self.o._get_sl_called_within_step = True
|
||||
return self
|
||||
|
||||
def __exit__(self, type, value, traceback):
|
||||
self.o._get_sl_called_within_step = False
|
||||
|
||||
with _enable_get_sl_call(self):
|
||||
self.last_epoch += 1
|
||||
values = self.get_sl()
|
||||
|
||||
for i, data in enumerate(zip(self.sparsifier.groups, values)):
|
||||
param_group, sl = data
|
||||
param_group["sparsity_level"] = sl
|
||||
self.print_sl(self.verbose, i, sl, epoch)
|
||||
|
||||
self._last_sl = [group["sparsity_level"] for group in self.sparsifier.groups]
|
||||
self.sparsifier.enable_mask_update = True
|
||||
|
||||
def _make_sure_a_list(self, var):
|
||||
r"""Utility that extends it to the same length as the .groups, ensuring it is a list"""
|
||||
n = len(self.sparsifier.groups)
|
||||
if not isinstance(var, (list, tuple)):
|
||||
return [var] * n
|
||||
else:
|
||||
if len(var) != n:
|
||||
raise ValueError(f"Expected variable of length {n}, but got {len(var)}")
|
||||
return list(var) # We want the result to be in a list, not tuple
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
# mypy: allow-untyped-defs
|
||||
import warnings
|
||||
|
||||
from .base_scheduler import BaseScheduler
|
||||
|
||||
|
||||
__all__ = ["CubicSL"]
|
||||
|
||||
|
||||
def _clamp(x, lo, hi):
|
||||
return max(lo, min(hi, x))
|
||||
|
||||
|
||||
class CubicSL(BaseScheduler):
|
||||
r"""Sets the sparsity level of each parameter group to the final sl
|
||||
plus a given exponential function.
|
||||
|
||||
.. math::
|
||||
|
||||
s_i = s_f + (s_0 - s_f) \cdot \left( 1 - \frac{t - t_0}{n\Delta t} \right)^3
|
||||
|
||||
where :math:`s_i` is the sparsity at epoch :math:`t`, :math;`s_f` is the final
|
||||
sparsity level, :math:`f(i)` is the function to be applied to the current epoch
|
||||
:math:`t`, initial epoch :math:`t_0`, and final epoch :math:`t_f`.
|
||||
:math:`\Delta t` is used to control how often the update of the sparsity level
|
||||
happens. By default,
|
||||
|
||||
Args:
|
||||
sparsifier (BaseSparsifier): Wrapped sparsifier.
|
||||
init_sl (int, list): Initial level of sparsity
|
||||
init_t (int, list): Initial step, when pruning starts
|
||||
delta_t (int, list): Pruning frequency
|
||||
total_t (int, list): Total number of pruning steps
|
||||
initially_zero (bool, list): If True, sets the level of sparsity to 0
|
||||
before init_t (:math:`t_0`). Otherwise, the sparsity level before
|
||||
init_t (:math:`t_0`) is set to init_sl(:math:`s_0`)
|
||||
last_epoch (int): The index of last epoch. Default: -1.
|
||||
verbose (bool): If ``True``, prints a message to stdout for
|
||||
each update. Default: ``False``.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
sparsifier,
|
||||
init_sl=0.0,
|
||||
init_t=0,
|
||||
delta_t=10,
|
||||
total_t=100,
|
||||
initially_zero=False,
|
||||
last_epoch=-1,
|
||||
verbose=False,
|
||||
):
|
||||
self.sparsifier = sparsifier
|
||||
|
||||
self.init_sl = self._make_sure_a_list(init_sl)
|
||||
self.init_t = self._make_sure_a_list(init_t)
|
||||
self.delta_t = self._make_sure_a_list(delta_t)
|
||||
self.total_t = self._make_sure_a_list(total_t)
|
||||
|
||||
self.initially_zero = self._make_sure_a_list(initially_zero)
|
||||
|
||||
super().__init__(sparsifier, last_epoch, verbose)
|
||||
|
||||
@staticmethod
|
||||
def sparsity_compute_fn(s_0, s_f, t, t_0, dt, n, initially_zero=False):
|
||||
r""" "Computes the current level of sparsity.
|
||||
|
||||
Based on https://arxiv.org/pdf/1710.01878.pdf
|
||||
|
||||
Args:
|
||||
s_0: Initial level of sparsity, :math:`s_i`
|
||||
s_f: Target level of sparsity, :math:`s_f`
|
||||
t: Current step, :math:`t`
|
||||
t_0: Initial step, :math:`t_0`
|
||||
dt: Pruning frequency, :math:`\Delta T`
|
||||
n: Pruning steps, :math:`n`
|
||||
initially_zero: Sets the level of sparsity to 0 before t_0.
|
||||
If False, sets to s_0
|
||||
|
||||
Returns:
|
||||
The sparsity level :math:`s_t` at the current step :math:`t`
|
||||
"""
|
||||
if initially_zero and t < t_0:
|
||||
return 0
|
||||
s_t = s_f + (s_0 - s_f) * (1.0 - (t - t_0) / (dt * n)) ** 3
|
||||
s_t = _clamp(s_t, s_0, s_f)
|
||||
return s_t
|
||||
|
||||
def get_sl(self):
|
||||
if not self._get_sl_called_within_step:
|
||||
warnings.warn(
|
||||
"To get the last sparsity level computed by the scheduler, "
|
||||
"please use `get_last_sl()`.",
|
||||
stacklevel=2,
|
||||
)
|
||||
return [
|
||||
self.sparsity_compute_fn(
|
||||
s_0=initial_sparsity,
|
||||
s_f=final_sparsity,
|
||||
t=self.last_epoch,
|
||||
t_0=initial_epoch,
|
||||
dt=delta_epoch,
|
||||
n=interval_epochs,
|
||||
initially_zero=initially_zero,
|
||||
)
|
||||
for initial_sparsity, final_sparsity, initial_epoch, delta_epoch, interval_epochs, initially_zero in zip(
|
||||
self.init_sl,
|
||||
self.base_sl,
|
||||
self.init_t,
|
||||
self.delta_t,
|
||||
self.total_t,
|
||||
self.initially_zero,
|
||||
)
|
||||
]
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
import warnings
|
||||
from collections.abc import Callable
|
||||
|
||||
from torch.ao.pruning.sparsifier.base_sparsifier import BaseSparsifier
|
||||
|
||||
from .base_scheduler import BaseScheduler
|
||||
|
||||
|
||||
__all__ = ["LambdaSL"]
|
||||
|
||||
|
||||
class LambdaSL(BaseScheduler):
|
||||
"""Sets the sparsity level of each parameter group to the final sl
|
||||
times a given function. When last_epoch=-1, sets initial sl as zero.
|
||||
Args:
|
||||
sparsifier (BaseSparsifier): Wrapped sparsifier.
|
||||
sl_lambda (function or list): A function which computes a multiplicative
|
||||
factor given an integer parameter epoch, or a list of such
|
||||
functions, one for each group in sparsifier.param_groups.
|
||||
last_epoch (int): The index of last epoch. Default: -1.
|
||||
verbose (bool): If ``True``, prints a message to stdout for
|
||||
each update. Default: ``False``.
|
||||
Example:
|
||||
>>> # Assuming sparsifier has two groups.
|
||||
>>> lambda1 = lambda epoch: epoch // 30
|
||||
>>> lambda2 = lambda epoch: 0.95**epoch
|
||||
>>> # xdoctest: +SKIP
|
||||
>>> scheduler = LambdaSL(sparsifier, sl_lambda=[lambda1, lambda2])
|
||||
>>> for epoch in range(100):
|
||||
>>> train(...)
|
||||
>>> validate(...)
|
||||
>>> scheduler.step()
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
sparsifier: BaseSparsifier,
|
||||
sl_lambda: Callable[[int], float] | list[Callable[[int], float]],
|
||||
last_epoch: int = -1,
|
||||
verbose: bool = False,
|
||||
) -> None:
|
||||
self.sparsifier = sparsifier
|
||||
|
||||
if not isinstance(sl_lambda, list) and not isinstance(sl_lambda, tuple):
|
||||
self.sl_lambdas = [sl_lambda] * len(sparsifier.groups)
|
||||
else:
|
||||
if len(sl_lambda) != len(sparsifier.groups):
|
||||
raise ValueError(
|
||||
f"Expected {len(sparsifier.groups)} lr_lambdas, but got {len(sl_lambda)}"
|
||||
)
|
||||
self.sl_lambdas = list(sl_lambda)
|
||||
super().__init__(sparsifier, last_epoch, verbose) # type: ignore[no-untyped-call]
|
||||
|
||||
def get_sl(self) -> list[float]:
|
||||
if not self._get_sl_called_within_step:
|
||||
warnings.warn(
|
||||
"To get the last sparsity level computed by the scheduler, "
|
||||
"please use `get_last_sl()`.",
|
||||
stacklevel=2,
|
||||
)
|
||||
return [
|
||||
base_sl * lmbda(self.last_epoch)
|
||||
for lmbda, base_sl in zip(self.sl_lambdas, self.base_sl)
|
||||
]
|
||||
Reference in New Issue
Block a user