Merge branch 'master' into refactor/epoch-end-hook-removal

Author: carmocca

src/lightning_app/cli/lightning_cli.py

@@ -389,8 +389,8 @@ def run_app(
     )
 
 
-if RequirementCache("lightning-fabric>=1.9.0.dev0") or RequirementCache("lightning>=1.9.0.dev0"):
-    # lightning.fabric.cli may not be available when installing only standalone lightning-app package
+if RequirementCache("lightning-fabric>=1.9.0") or RequirementCache("lightning>=1.9.0"):
+    # note it is automatically replaced to `from lightning.fabric.cli` when building monolithic/mirror package
     from lightning_fabric.cli import _run_model
 
     run.add_command(_run_model)

src/version.info

@@ -1 +1 @@
-1.9.0
+2.0.0dev

tests/tests_pytorch/accelerators/test_hpu.py

@@ -29,13 +29,6 @@
 from tests_pytorch.helpers.simple_models import ClassificationModel
 
 
-class HPUTestModel(BoringModel):
-    def configure_optimizers(self):
-        opt_a = torch.optim.Adam(self.layer.parameters(), lr=0.001)
-        opt_b = torch.optim.SGD(self.layer.parameters(), lr=0.001)
-        return opt_a, opt_b
-
-
 @RunIf(hpu=True)
 def test_availability():
     assert HPUAccelerator.is_available()
@@ -277,17 +270,26 @@ def test_strategy_params_with_hpu_parallel_strategy():
 
 @RunIf(hpu=True)
 def test_multi_optimizers_with_hpu(tmpdir):
-    class TestModel(HPUTestModel):
-
-        optims = [False, False]
-
-        def training_step(self, batch, batch_idx, optimizer_idx):
-            self.optims[optimizer_idx] = True
-            return super().training_step(batch, batch_idx)
+    class MultiOptimizerModel(BoringModel):
+        def configure_optimizers(self):
+            opt_a = torch.optim.Adam(self.layer.parameters(), lr=0.001)
+            opt_b = torch.optim.SGD(self.layer.parameters(), lr=0.001)
+            return opt_a, opt_b
 
-    model = TestModel()
+        def training_step(self, batch, batch_idx):
+            opt1, opt2 = self.optimizers()
+            loss = self.loss(self.step(batch))
+            opt1.zero_grad()
+            self.manual_backward(loss)
+            opt1.step()
+            loss = self.loss(self.step(batch))
+            opt2.zero_grad()
+            self.manual_backward(loss)
+            opt2.step()
+
+    model = MultiOptimizerModel()
+    model.automatic_optimization = False
     model.val_dataloader = None
-
     trainer = Trainer(
         default_root_dir=tmpdir,
         accelerator="hpu",
@@ -300,11 +302,9 @@ def training_step(self, batch, batch_idx, optimizer_idx):
     )
     trainer.fit(model)
 
-    assert all(model.optims)
-
 
 @RunIf(hpu=True)
-def test_hpu_device_stats_monitor(tmpdir):
+def test_hpu_device_stats_monitor():
 
     hpu_stats = HPUAccelerator().get_device_stats("hpu")
     fields = [

tests/tests_pytorch/callbacks/test_lr_monitor.py

@@ -218,8 +218,27 @@ def test_lr_monitor_multi_lrs(tmpdir, logging_interval: str):
     """Test that learning rates are extracted and logged for multi lr schedulers."""
 
     class CustomBoringModel(BoringModel):
-        def training_step(self, batch, batch_idx, optimizer_idx):
-            return super().training_step(batch, batch_idx)
+        def __init__(self):
+            super().__init__()
+            self.automatic_optimization = False
+
+        def training_step(self, batch, batch_idx):
+            opt1, opt2 = self.optimizers()
+
+            loss = self.loss(self.step(batch))
+            opt1.zero_grad()
+            self.manual_backward(loss)
+            opt1.step()
+
+            loss = self.loss(self.step(batch))
+            opt2.zero_grad()
+            self.manual_backward(loss)
+            opt2.step()
+
+        def on_train_epoch_end(self):
+            scheduler1, scheduler2 = self.lr_schedulers()
+            scheduler1.step()
+            scheduler2.step()
 
         def configure_optimizers(self):
             optimizer1 = optim.Adam(self.parameters(), lr=1e-2)
@@ -262,8 +281,22 @@ def test_lr_monitor_no_lr_scheduler_multi_lrs(tmpdir, logging_interval: str):
     """Test that learning rates are extracted and logged for multi optimizers but no lr scheduler."""
 
     class CustomBoringModel(BoringModel):
-        def training_step(self, batch, batch_idx, optimizer_idx):
-            return super().training_step(batch, batch_idx)
+        def __init__(self):
+            super().__init__()
+            self.automatic_optimization = False
+
+        def training_step(self, batch, batch_idx):
+            opt1, opt2 = self.optimizers()
+
+            loss = self.loss(self.step(batch))
+            opt1.zero_grad()
+            self.manual_backward(loss)
+            opt1.step()
+
+            loss = self.loss(self.step(batch))
+            opt2.zero_grad()
+            self.manual_backward(loss)
+            opt2.step()
 
         def configure_optimizers(self):
             optimizer1 = optim.Adam(self.parameters(), lr=1e-2)
@@ -421,22 +454,46 @@ def test_multiple_optimizers_basefinetuning(tmpdir):
     class TestModel(BoringModel):
         def __init__(self):
             super().__init__()
+            self.automatic_optimization = False
             self.backbone = torch.nn.Sequential(
                 torch.nn.Linear(32, 32), torch.nn.Linear(32, 32), torch.nn.Linear(32, 32), torch.nn.ReLU(True)
             )
             self.layer = torch.nn.Linear(32, 2)
 
-        def training_step(self, batch, batch_idx, optimizer_idx):
-            return super().training_step(batch, batch_idx)
+        def training_step(self, batch, batch_idx):
+            opt1, opt2, opt3 = self.optimizers()
+
+            # optimizer 1
+            loss = self.step(batch)
+            self.manual_backward(loss)
+            opt1.step()
+            opt1.zero_grad()
+
+            # optimizer 2
+            loss = self.step(batch)
+            self.manual_backward(loss)
+            opt2.step()
+            opt2.zero_grad()
+
+            # optimizer 3
+            loss = self.step(batch)
+            self.manual_backward(loss)
+            opt3.step()
+            opt3.zero_grad()
+
+        def on_train_epoch_end(self) -> None:
+            lr_sched1, lr_sched2 = self.lr_schedulers()
+            lr_sched1.step()
+            lr_sched2.step()
 
         def forward(self, x):
             return self.layer(self.backbone(x))
 
         def configure_optimizers(self):
             parameters = list(filter(lambda p: p.requires_grad, self.parameters()))
-            opt = optim.Adam(parameters, lr=0.1)
+            opt = optim.SGD(parameters, lr=0.1)
             opt_2 = optim.Adam(parameters, lr=0.1)
-            opt_3 = optim.Adam(parameters, lr=0.1)
+            opt_3 = optim.AdamW(parameters, lr=0.1)
             optimizers = [opt, opt_2, opt_3]
             schedulers = [
                 optim.lr_scheduler.StepLR(opt, step_size=1, gamma=0.5),
@@ -452,24 +509,24 @@ def on_train_epoch_start(self, trainer, pl_module) -> None:
                 assert num_param_groups == 3
             elif trainer.current_epoch == 1:
                 assert num_param_groups == 4
-                assert list(lr_monitor.lrs) == ["lr-Adam-1", "lr-Adam-2", "lr-Adam/pg1", "lr-Adam/pg2"]
+                assert list(lr_monitor.lrs) == ["lr-Adam", "lr-AdamW", "lr-SGD/pg1", "lr-SGD/pg2"]
             elif trainer.current_epoch == 2:
                 assert num_param_groups == 5
                 assert list(lr_monitor.lrs) == [
-                    "lr-Adam-2",
+                    "lr-AdamW",
+                    "lr-SGD/pg1",
+                    "lr-SGD/pg2",
                     "lr-Adam/pg1",
                     "lr-Adam/pg2",
-                    "lr-Adam-1/pg1",
-                    "lr-Adam-1/pg2",
                 ]
             else:
                 expected = [
-                    "lr-Adam-2",
+                    "lr-AdamW",
+                    "lr-SGD/pg1",
+                    "lr-SGD/pg2",
                     "lr-Adam/pg1",
                     "lr-Adam/pg2",
-                    "lr-Adam-1/pg1",
-                    "lr-Adam-1/pg2",
-                    "lr-Adam-1/pg3",
+                    "lr-Adam/pg3",
                 ]
                 assert list(lr_monitor.lrs) == expected
 
@@ -481,12 +538,12 @@ def freeze_before_training(self, pl_module):
 
         def finetune_function(self, pl_module, epoch: int, optimizer, opt_idx: int):
             """Called when the epoch begins."""
-            if epoch == 1 and opt_idx == 0:
+            if epoch == 1 and isinstance(optimizer, torch.optim.SGD):
                 self.unfreeze_and_add_param_group(pl_module.backbone[0], optimizer, lr=0.1)
-            if epoch == 2 and opt_idx == 1:
+            if epoch == 2 and isinstance(optimizer, torch.optim.Adam):
                 self.unfreeze_and_add_param_group(pl_module.layer, optimizer, lr=0.1)
 
-            if epoch == 3 and opt_idx == 1:
+            if epoch == 3 and isinstance(optimizer, torch.optim.Adam):
                 assert len(optimizer.param_groups) == 2
                 self.unfreeze_and_add_param_group(pl_module.backbone[1], optimizer, lr=0.1)
                 assert len(optimizer.param_groups) == 3
@@ -507,22 +564,22 @@ def finetune_function(self, pl_module, epoch: int, optimizer, opt_idx: int):
     trainer.fit(model)
 
     expected = [0.1, 0.1, 0.1, 0.1, 0.1]
-    assert lr_monitor.lrs["lr-Adam-2"] == expected
+    assert lr_monitor.lrs["lr-AdamW"] == expected
 
     expected = [0.1, 0.05, 0.025, 0.0125, 0.00625]
-    assert lr_monitor.lrs["lr-Adam/pg1"] == expected
+    assert lr_monitor.lrs["lr-SGD/pg1"] == expected
 
     expected = [0.1, 0.05, 0.025, 0.0125]
-    assert lr_monitor.lrs["lr-Adam/pg2"] == expected
+    assert lr_monitor.lrs["lr-SGD/pg2"] == expected
 
     expected = [0.1, 0.05, 0.025, 0.0125, 0.00625]
-    assert lr_monitor.lrs["lr-Adam-1/pg1"] == expected
+    assert lr_monitor.lrs["lr-Adam/pg1"] == expected
 
     expected = [0.1, 0.05, 0.025]
-    assert lr_monitor.lrs["lr-Adam-1/pg2"] == expected
+    assert lr_monitor.lrs["lr-Adam/pg2"] == expected
 
     expected = [0.1, 0.05]
-    assert lr_monitor.lrs["lr-Adam-1/pg3"] == expected
+    assert lr_monitor.lrs["lr-Adam/pg3"] == expected
 
 
 def test_lr_monitor_multiple_param_groups_no_lr_scheduler(tmpdir):