[feat] Add flexible step-wise LR scheduler with minimum changes (#256)

* [doc] Add the bug fix information for ipython user

* [fix] Delete the gcc and gnn install because I could check we can build without them

* [feat] Add flexible step-wise LR scheduler with minimum changes

Since we would like to merge this feature promptly, I cut this new branch
from the branch hot-fix-adapt... and narrowed down the scope of this PR.
The subsequent PR addresses the issues, epspecially the format and mypy
typing.

* [fix] Fix flake8 issue and remove unneeded changes because of mis-branching

* [test] Add the tests for the new features

* [fix] Fix flake8 issues and torch.tensor to torch.Tensor in typing check

The intention behind the change from torch.tensor to torch.Tensor is that
since I got an error NoModuleFound `import torch.tensor`.
Plus, the torch.tensor is not a TensorType class, but torch.Tensor is.
Therefore, I changed the torch.tensor to torch.Tensor.

* [feat] Be able to add the step_unit to ConfigSpace

* [fix] Fix pytest issues by adding batch-wise update to incumbent

Since the previous version always use batch-wise update, I added the
step_unit = batch and then avoid the errors I got from pytest.

* [fix] Add step_unit info in the greedy portfolio json file

Since the latest version only supports the batch-wise update,
I just inserted step_unit == "batch" to be able to run greedy portfolio
selection.

* [refactor] Rebase to the latest development and add overridden functions in base_scheduler

* [fix] Fix flake8 and mypy issues

* [fix] Fix flake8 issues

* [test] Add the test for the train step and the lr scheduler check

* [refactor] Change the name  to

* [fix] Fix flake8 issues

* [fix] Disable the step_interval option from the hyperparameter settings

* [fix] Change the default step_interval to Epoch-wise

* [fix] Fix the step timing for ReduceLROnPlateau and fix flake8 issues

* [fix] Add the after-validation option to StepIntervalUnit for ReduceLROnPlateau

* [fix] Fix flake8 issues

* [fix] Fix loss value for epoch-wise scheduler update

* [fix] Delete type check of step_interval and make it property

Since the step_interval should not be modified from outside,
I made it a property of the base_scheduler class.
Furthermore, since we do not have to check the type of step_interval
except the initialization, I deleted the type check from prepare method.

* [fix] Fix a mypy issue

* [fix] Fix a mypy issue

* [fix] Fix mypy issues

* [fix] Fix mypy issues

* [feedback] Address the Ravin's suggestions

Author: nabenabe0928

autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingLR.py

@@ -8,10 +8,10 @@
 import numpy as np
 
 import torch.optim.lr_scheduler
-from torch.optim.lr_scheduler import _LRScheduler
 
 from autoPyTorch.datasets.base_dataset import BaseDatasetPropertiesType
 from autoPyTorch.pipeline.components.setup.lr_scheduler.base_scheduler import BaseLRComponent
+from autoPyTorch.pipeline.components.setup.lr_scheduler.constants import StepIntervalUnit
 from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter
 
 
@@ -26,13 +26,13 @@ class CosineAnnealingLR(BaseLRComponent):
     def __init__(
         self,
         T_max: int,
+        step_interval: Union[str, StepIntervalUnit] = StepIntervalUnit.epoch,
         random_state: Optional[np.random.RandomState] = None
     ):
 
-        super().__init__()
+        super().__init__(step_interval)
         self.T_max = T_max
         self.random_state = random_state
-        self.scheduler = None  # type: Optional[_LRScheduler]
 
     def fit(self, X: Dict[str, Any], y: Any = None) -> BaseLRComponent:
         """
@@ -71,6 +71,8 @@ def get_hyperparameter_search_space(
                                                                      default_value=200,
                                                                      )
     ) -> ConfigurationSpace:
+
         cs = ConfigurationSpace()
         add_hyperparameter(cs, T_max, UniformIntegerHyperparameter)
+
         return cs

autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingWarmRestarts.py

@@ -1,15 +1,18 @@
 from typing import Any, Dict, Optional, Union
 
 from ConfigSpace.configuration_space import ConfigurationSpace
-from ConfigSpace.hyperparameters import UniformFloatHyperparameter, UniformIntegerHyperparameter
+from ConfigSpace.hyperparameters import (
+    UniformFloatHyperparameter,
+    UniformIntegerHyperparameter
+)
 
 import numpy as np
 
 import torch.optim.lr_scheduler
-from torch.optim.lr_scheduler import _LRScheduler
 
 from autoPyTorch.datasets.base_dataset import BaseDatasetPropertiesType
 from autoPyTorch.pipeline.components.setup.lr_scheduler.base_scheduler import BaseLRComponent
+from autoPyTorch.pipeline.components.setup.lr_scheduler.constants import StepIntervalUnit
 from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter
 
 
@@ -30,13 +33,13 @@ def __init__(
         self,
         T_0: int,
         T_mult: int,
-        random_state: Optional[np.random.RandomState] = None
+        step_interval: Union[str, StepIntervalUnit] = StepIntervalUnit.epoch,
+        random_state: Optional[np.random.RandomState] = None,
     ):
-        super().__init__()
+        super().__init__(step_interval)
         self.T_0 = T_0
         self.T_mult = T_mult
         self.random_state = random_state
-        self.scheduler = None  # type: Optional[_LRScheduler]
 
     def fit(self, X: Dict[str, Any], y: Any = None) -> BaseLRComponent:
         """
@@ -78,8 +81,9 @@ def get_hyperparameter_search_space(
         T_mult: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='T_mult',
                                                                       value_range=(1.0, 2.0),
                                                                       default_value=1.0,
-                                                                      ),
+                                                                      )
     ) -> ConfigurationSpace:
+
         cs = ConfigurationSpace()
         add_hyperparameter(cs, T_0, UniformIntegerHyperparameter)
         add_hyperparameter(cs, T_mult, UniformFloatHyperparameter)

autoPyTorch/pipeline/components/setup/lr_scheduler/CyclicLR.py

@@ -10,10 +10,10 @@
 import numpy as np
 
 import torch.optim.lr_scheduler
-from torch.optim.lr_scheduler import _LRScheduler
 
 from autoPyTorch.datasets.base_dataset import BaseDatasetPropertiesType
 from autoPyTorch.pipeline.components.setup.lr_scheduler.base_scheduler import BaseLRComponent
+from autoPyTorch.pipeline.components.setup.lr_scheduler.constants import StepIntervalUnit
 from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter
 
 
@@ -39,16 +39,16 @@ def __init__(
         base_lr: float,
         mode: str,
         step_size_up: int,
+        step_interval: Union[str, StepIntervalUnit] = StepIntervalUnit.epoch,
         max_lr: float = 0.1,
         random_state: Optional[np.random.RandomState] = None
     ):
-        super().__init__()
+        super().__init__(step_interval)
         self.base_lr = base_lr
         self.mode = mode
         self.max_lr = max_lr
         self.step_size_up = step_size_up
         self.random_state = random_state
-        self.scheduler = None  # type: Optional[_LRScheduler]
 
     def fit(self, X: Dict[str, Any], y: Any = None) -> BaseLRComponent:
         """

autoPyTorch/pipeline/components/setup/lr_scheduler/ExponentialLR.py

@@ -8,10 +8,10 @@
 import numpy as np
 
 import torch.optim.lr_scheduler
-from torch.optim.lr_scheduler import _LRScheduler
 
 from autoPyTorch.datasets.base_dataset import BaseDatasetPropertiesType
 from autoPyTorch.pipeline.components.setup.lr_scheduler.base_scheduler import BaseLRComponent
+from autoPyTorch.pipeline.components.setup.lr_scheduler.constants import StepIntervalUnit
 from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter
 
 
@@ -27,13 +27,13 @@ class ExponentialLR(BaseLRComponent):
     def __init__(
         self,
         gamma: float,
+        step_interval: Union[str, StepIntervalUnit] = StepIntervalUnit.epoch,
         random_state: Optional[np.random.RandomState] = None
     ):
 
-        super().__init__()
+        super().__init__(step_interval)
         self.gamma = gamma
         self.random_state = random_state
-        self.scheduler = None  # type: Optional[_LRScheduler]
 
     def fit(self, X: Dict[str, Any], y: Any = None) -> BaseLRComponent:
         """
@@ -72,6 +72,8 @@ def get_hyperparameter_search_space(
                                                                      default_value=0.9,
                                                                      )
     ) -> ConfigurationSpace:
+
         cs = ConfigurationSpace()
         add_hyperparameter(cs, gamma, UniformFloatHyperparameter)
+
         return cs

autoPyTorch/pipeline/components/setup/lr_scheduler/NoScheduler.py

@@ -4,10 +4,9 @@
 
 import numpy as np
 
-from torch.optim.lr_scheduler import _LRScheduler
-
 from autoPyTorch.datasets.base_dataset import BaseDatasetPropertiesType
 from autoPyTorch.pipeline.components.setup.lr_scheduler.base_scheduler import BaseLRComponent
+from autoPyTorch.pipeline.components.setup.lr_scheduler.constants import StepIntervalUnit
 
 
 class NoScheduler(BaseLRComponent):
@@ -17,12 +16,12 @@ class NoScheduler(BaseLRComponent):
     """
     def __init__(
         self,
+        step_interval: Union[str, StepIntervalUnit] = StepIntervalUnit.epoch,
         random_state: Optional[np.random.RandomState] = None
     ):
 
-        super().__init__()
+        super().__init__(step_interval)
         self.random_state = random_state
-        self.scheduler = None  # type: Optional[_LRScheduler]
 
     def fit(self, X: Dict[str, Any], y: Any = None) -> BaseLRComponent:
         """

autoPyTorch/pipeline/components/setup/lr_scheduler/ReduceLROnPlateau.py

@@ -10,10 +10,10 @@
 import numpy as np
 
 import torch.optim.lr_scheduler
-from torch.optim.lr_scheduler import _LRScheduler
 
 from autoPyTorch.datasets.base_dataset import BaseDatasetPropertiesType
 from autoPyTorch.pipeline.components.setup.lr_scheduler.base_scheduler import BaseLRComponent
+from autoPyTorch.pipeline.components.setup.lr_scheduler.constants import StepIntervalUnit
 from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter
 
 
@@ -31,22 +31,26 @@ class ReduceLROnPlateau(BaseLRComponent):
         factor (float): Factor by which the learning rate will be reduced. new_lr = lr * factor.
         patience (int): Number of epochs with no improvement after which learning
             rate will be reduced.
+        step_interval (str): step should be called after validation in the case of ReduceLROnPlateau
         random_state (Optional[np.random.RandomState]): random state
+
+    Reference:
+        https://pytorch.org/docs/stable/generated/torch.optim.lr_scheduler.ReduceLROnPlateau.html#torch.optim.lr_scheduler.ReduceLROnPlateau
     """
 
     def __init__(
         self,
         mode: str,
         factor: float,
         patience: int,
-        random_state: Optional[np.random.RandomState] = None
+        step_interval: Union[str, StepIntervalUnit] = StepIntervalUnit.valid,
+        random_state: Optional[np.random.RandomState] = None,
     ):
-        super().__init__()
+        super().__init__(step_interval)
         self.mode = mode
         self.factor = factor
         self.patience = patience
         self.random_state = random_state
-        self.scheduler = None  # type: Optional[_LRScheduler]
 
     def fit(self, X: Dict[str, Any], y: Any = None) -> BaseLRComponent:
         """
@@ -93,7 +97,7 @@ def get_hyperparameter_search_space(
         factor: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='factor',
                                                                       value_range=(0.01, 0.9),
                                                                       default_value=0.1,
-                                                                      ),
+                                                                      )
     ) -> ConfigurationSpace:
 
         cs = ConfigurationSpace()

autoPyTorch/pipeline/components/setup/lr_scheduler/StepLR.py

@@ -9,10 +9,10 @@
 import numpy as np
 
 import torch.optim.lr_scheduler
-from torch.optim.lr_scheduler import _LRScheduler
 
 from autoPyTorch.datasets.base_dataset import BaseDatasetPropertiesType
 from autoPyTorch.pipeline.components.setup.lr_scheduler.base_scheduler import BaseLRComponent
+from autoPyTorch.pipeline.components.setup.lr_scheduler.constants import StepIntervalUnit
 from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter
 
 
@@ -32,13 +32,13 @@ def __init__(
         self,
         step_size: int,
         gamma: float,
+        step_interval: Union[str, StepIntervalUnit] = StepIntervalUnit.epoch,
         random_state: Optional[np.random.RandomState] = None
     ):
-        super().__init__()
+        super().__init__(step_interval)
         self.gamma = gamma
         self.step_size = step_size
         self.random_state = random_state
-        self.scheduler = None  # type: Optional[_LRScheduler]
 
     def fit(self, X: Dict[str, Any], y: Any = None) -> BaseLRComponent:
         """
@@ -80,7 +80,7 @@ def get_hyperparameter_search_space(
         step_size: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='step_size',
                                                                          value_range=(1, 10),
                                                                          default_value=5,
-                                                                         ),
+                                                                         )
     ) -> ConfigurationSpace:
         cs = ConfigurationSpace()
 

autoPyTorch/pipeline/components/setup/lr_scheduler/__init__.py

@@ -16,6 +16,7 @@
 )
 from autoPyTorch.pipeline.components.setup.lr_scheduler.base_scheduler import BaseLRComponent
 
+
 directory = os.path.split(__file__)[0]
 _schedulers = find_components(__package__,
                               directory,

autoPyTorch/pipeline/components/setup/lr_scheduler/base_scheduler.py

@@ -1,23 +1,39 @@
-from typing import Any, Dict, Optional
+from typing import Any, Dict, Optional, Union
 
 from torch.optim import Optimizer
 from torch.optim.lr_scheduler import _LRScheduler
 
 from autoPyTorch.pipeline.components.setup.base_setup import autoPyTorchSetupComponent
+from autoPyTorch.pipeline.components.setup.lr_scheduler.constants import StepIntervalUnit, StepIntervalUnitChoices
 from autoPyTorch.utils.common import FitRequirement
 
 
 class BaseLRComponent(autoPyTorchSetupComponent):
     """Provide an abstract interface for schedulers
     in Auto-Pytorch"""
 
-    def __init__(self) -> None:
+    def __init__(self, step_interval: Union[str, StepIntervalUnit]):
         super().__init__()
         self.scheduler = None  # type: Optional[_LRScheduler]
+        self._step_interval: StepIntervalUnit
+
+        if isinstance(step_interval, str):
+            if step_interval not in StepIntervalUnitChoices:
+                raise ValueError('step_interval must be either {}, but got {}.'.format(
+                    StepIntervalUnitChoices,
+                    step_interval
+                ))
+            self._step_interval = getattr(StepIntervalUnit, step_interval)
+        else:
+            self._step_interval = step_interval
 
         self.add_fit_requirements([
             FitRequirement('optimizer', (Optimizer,), user_defined=False, dataset_property=False)])
 
+    @property
+    def step_interval(self) -> StepIntervalUnit:
+        return self._step_interval
+
     def transform(self, X: Dict[str, Any]) -> Dict[str, Any]:
         """
         Adds the scheduler into the fit dictionary 'X' and returns it.
@@ -26,7 +42,11 @@ def transform(self, X: Dict[str, Any]) -> Dict[str, Any]:
         Returns:
             (Dict[str, Any]): the updated 'X' dictionary
         """
-        X.update({'lr_scheduler': self.scheduler})
+
+        X.update(
+            lr_scheduler=self.scheduler,
+            step_interval=self.step_interval
+        )
         return X
 
     def get_scheduler(self) -> _LRScheduler:

autoPyTorch/pipeline/components/setup/lr_scheduler/constants.py

@@ -0,0 +1,17 @@
+from enum import Enum
+
+
+class StepIntervalUnit(Enum):
+    """
+    By which interval we perform the step for learning rate schedulers.
+    Attributes:
+        batch (str): We update every batch evaluation
+        epoch (str): We update every epoch
+        valid (str): We update every validation
+    """
+    batch = 'batch'
+    epoch = 'epoch'
+    valid = 'valid'
+
+
+StepIntervalUnitChoices = [step_interval.name for step_interval in StepIntervalUnit]