An instance of SaveConfigCallback should only save the config once (#14927)

Author: mauvilsa

src/pytorch_lightning/CHANGELOG.md

@@ -295,6 +295,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 
 
 
+- `SaveConfigCallback` instances should only save the config once to allow having the `overwrite=False` safeguard when using `LightningCLI(..., run=False)` ([#14927](https://github.com/Lightning-AI/lightning/pull/14927))
+
+
 ## [1.7.7] - 2022-09-22
 
 ### Fixed

src/pytorch_lightning/cli.py

@@ -209,8 +209,12 @@ def __init__(
         self.config_filename = config_filename
         self.overwrite = overwrite
         self.multifile = multifile
+        self.already_saved = False
 
     def setup(self, trainer: Trainer, pl_module: LightningModule, stage: str) -> None:
+        if self.already_saved:
+            return
+
         log_dir = trainer.log_dir  # this broadcasts the directory
         assert log_dir is not None
         config_path = os.path.join(log_dir, self.config_filename)
@@ -238,6 +242,10 @@ def setup(self, trainer: Trainer, pl_module: LightningModule, stage: str) -> Non
             self.parser.save(
                 self.config, config_path, skip_none=False, overwrite=self.overwrite, multifile=self.multifile
             )
+            self.already_saved = True
+
+        # broadcast so that all ranks are in sync on future calls to .setup()
+        self.already_saved = trainer.strategy.broadcast(self.already_saved)
 
 
 class LightningCLI:

tests/tests_pytorch/test_cli.py

@@ -249,6 +249,22 @@ def test_lightning_cli_save_config_cases(tmpdir):
         LightningCLI(BoringModel)
 
 
+def test_lightning_cli_save_config_only_once(tmpdir):
+    config_path = tmpdir / "config.yaml"
+    cli_args = [f"--trainer.default_root_dir={tmpdir}", "--trainer.logger=False", "--trainer.max_epochs=1"]
+
+    with mock.patch("sys.argv", ["any.py"] + cli_args):
+        cli = LightningCLI(BoringModel, run=False)
+
+    save_config_callback = next(c for c in cli.trainer.callbacks if isinstance(c, SaveConfigCallback))
+    assert not save_config_callback.overwrite
+    assert not save_config_callback.already_saved
+    cli.trainer.fit(cli.model)
+    assert os.path.isfile(config_path)
+    assert save_config_callback.already_saved
+    cli.trainer.test(cli.model)  # Should not fail because config already saved
+
+
 def test_lightning_cli_config_and_subclass_mode(tmpdir):
     input_config = {
         "fit": {