From 3a96f00f6deb7e67dcdcf52f25e862181f7834f9 Mon Sep 17 00:00:00 2001 From: Yurij Mikhalevich Date: Wed, 14 Dec 2022 21:25:11 +0100 Subject: [PATCH 1/6] fix(cloud): detect and ignore venv --- src/lightning_app/runners/cloud.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/lightning_app/runners/cloud.py b/src/lightning_app/runners/cloud.py index ab5ae29c092a5..88452bac50ae9 100644 --- a/src/lightning_app/runners/cloud.py +++ b/src/lightning_app/runners/cloud.py @@ -230,6 +230,19 @@ def dispatch( else: ignore_functions = None + # Create a default dotignore if it doesn't exist + if not (root / DOT_IGNORE_FILENAME).is_file(): + with open(root / DOT_IGNORE_FILENAME, "w") as f: + f.write("venv/\n") + if (root / "bin" / "activate").is_file() or (root / "pyvenv.cfg").is_file(): + # the user is developing inside venv + f.write( + "bin/\n" + "include/\n" + "lib/\n" + "pyvenv.cfg\n" + ) + repo = LocalSourceCodeDir(path=root, ignore_functions=ignore_functions) self._check_uploaded_folder(root, repo) requirements_file = root / "requirements.txt" @@ -556,15 +569,9 @@ def _check_uploaded_folder(root: Path, repo: LocalSourceCodeDir) -> None: f"Your application folder '{root.absolute()}' is more than {CLOUD_UPLOAD_WARNING} MB. " f"The total size is {round(app_folder_size_in_mb, 2)} MB. {len(files)} files were uploaded.\n" + largest_paths_msg - + "Perhaps you should try running the app in an empty directory." + + "Perhaps you should try running the app in an empty directory.\n" + "You can ignore some files or folders by adding them to `.lightningignore`." ) - if not (root / DOT_IGNORE_FILENAME).is_file(): - warning_msg += ( - "\nIn order to ignore some files or folder, create a `.lightningignore` file and add the paths to" - " ignore. You can also set the `lightningingore` attribute in a Flow or Work." - ) - else: - warning_msg += "\nYou can ignore some files or folders by adding them to `.lightningignore`." logger.warn(warning_msg) From eaa552b0eef4b27092990e00a98601e5f17ca82e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 14 Dec 2022 20:29:36 +0000 Subject: [PATCH 2/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/lightning_app/runners/cloud.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/lightning_app/runners/cloud.py b/src/lightning_app/runners/cloud.py index 88452bac50ae9..49165f9d973af 100644 --- a/src/lightning_app/runners/cloud.py +++ b/src/lightning_app/runners/cloud.py @@ -236,12 +236,7 @@ def dispatch( f.write("venv/\n") if (root / "bin" / "activate").is_file() or (root / "pyvenv.cfg").is_file(): # the user is developing inside venv - f.write( - "bin/\n" - "include/\n" - "lib/\n" - "pyvenv.cfg\n" - ) + f.write("bin/\n" "include/\n" "lib/\n" "pyvenv.cfg\n") repo = LocalSourceCodeDir(path=root, ignore_functions=ignore_functions) self._check_uploaded_folder(root, repo) From bc4913de15dac2cf7aabb9105b312b0442076554 Mon Sep 17 00:00:00 2001 From: Ethan Harris Date: Wed, 14 Dec 2022 20:47:52 +0000 Subject: [PATCH 3/6] Fix test --- tests/tests_app/runners/test_cloud.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/tests_app/runners/test_cloud.py b/tests/tests_app/runners/test_cloud.py index cd9e4c2923d6a..7c66fb1d60a25 100644 --- a/tests/tests_app/runners/test_cloud.py +++ b/tests/tests_app/runners/test_cloud.py @@ -1334,8 +1334,7 @@ def test_check_uploaded_folder(monkeypatch, tmpdir, caplog): assert "The total size is 15.0 MB" in caplog.text assert "3 files were uploaded" in caplog.text assert "files:\n6.0 MB: c.jpg\n5.0 MB: b.txt\n4.0 MB: a.png\nPerhaps" in caplog.text # tests the order - assert "create a `.lightningignore` file" in caplog.text - assert "lightningingore` attribute in a Flow or Work" in caplog.text + assert "adding them to `.lightningignore`." in caplog.text @mock.patch("lightning_app.core.queues.QueuingSystem", MagicMock()) From ef2c5ad55c8dc5fb9fa6cb55165f9aedccb34d91 Mon Sep 17 00:00:00 2001 From: Ethan Harris Date: Wed, 14 Dec 2022 21:00:01 +0000 Subject: [PATCH 4/6] Add test --- tests/tests_app/runners/test_cloud.py | 48 +++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tests/tests_app/runners/test_cloud.py b/tests/tests_app/runners/test_cloud.py index 7c66fb1d60a25..ab789d8eb4332 100644 --- a/tests/tests_app/runners/test_cloud.py +++ b/tests/tests_app/runners/test_cloud.py @@ -1570,6 +1570,54 @@ def run(self): flow.run() +def test_default_lightningignore(monkeypatch, caplog, tmpdir): + mock_client = mock.MagicMock() + mock_client.projects_service_list_memberships.return_value = V1ListMembershipsResponse( + memberships=[V1Membership(name="test-project", project_id="test-project-id")] + ) + mock_client.lightningapp_instance_service_list_lightningapp_instances.return_value = ( + V1ListLightningappInstancesResponse(lightningapps=[]) + ) + mock_client.lightningapp_v2_service_create_lightningapp_release.return_value = V1LightningappRelease( + cluster_id="test" + ) + cloud_backend = mock.MagicMock(client=mock_client) + monkeypatch.setattr(backends, "CloudBackend", mock.MagicMock(return_value=cloud_backend)) + + class MyWork(LightningWork): + def run(self): + pass + + app = LightningApp(MyWork()) + + path = Path(tmpdir) + cloud_runtime = cloud.CloudRuntime(app=app, entrypoint_file=path / "entrypoint.py") + monkeypatch.setattr(LocalSourceCodeDir, "upload", mock.MagicMock()) + + # write some files + write_file_of_size(path / "a.txt", 5 * 1000 * 1000) + write_file_of_size(path / "venv" / "foo.txt", 4 * 1000 * 1000) + + with mock.patch( + "lightning_app.runners.cloud._parse_lightningignore", wraps=_parse_lightningignore + ) as parse_mock, mock.patch( + "lightning_app.source_code.local._copytree", wraps=_copytree + ) as copy_mock, caplog.at_level( + logging.WARN + ): + cloud_runtime.dispatch() + + parse_mock.assert_called_once_with(()) + assert copy_mock.mock_calls[0].kwargs["ignore_functions"][0].args[1] == {"lightning_logs", "foo"} + + assert (path / ".lightningignore").exists() + + assert f"Your application folder '{path.absolute()}' is more than 2 MB" in caplog.text + assert "The total size is 5.0 MB" in caplog.text + assert "2 files were uploaded" # a.txt and .lightningignore + assert "files:\n5.0 MB: a.txt\nPerhaps" in caplog.text # only this file appears + + @pytest.mark.parametrize( "lightning_app_instance, lightning_cloud_url, expected_url", [ From 3b29751b42090be9aa04331584614075f87adb51 Mon Sep 17 00:00:00 2001 From: Ethan Harris Date: Wed, 14 Dec 2022 21:08:23 +0000 Subject: [PATCH 5/6] Fix test --- tests/tests_app/runners/test_cloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests_app/runners/test_cloud.py b/tests/tests_app/runners/test_cloud.py index ab789d8eb4332..770c86dfec4b2 100644 --- a/tests/tests_app/runners/test_cloud.py +++ b/tests/tests_app/runners/test_cloud.py @@ -1608,7 +1608,7 @@ def run(self): cloud_runtime.dispatch() parse_mock.assert_called_once_with(()) - assert copy_mock.mock_calls[0].kwargs["ignore_functions"][0].args[1] == {"lightning_logs", "foo"} + assert copy_mock.mock_calls[0].kwargs["ignore_functions"][0].args[1] == set() assert (path / ".lightningignore").exists() From 5d10181a05f9d8b7b2a3a5295598d0ed423e8169 Mon Sep 17 00:00:00 2001 From: Yurij Mikhalevich Date: Thu, 15 Dec 2022 14:41:12 +0100 Subject: [PATCH 6/6] addressed the PR feedback --- src/lightning_app/runners/cloud.py | 5 +++-- tests/tests_app/runners/test_cloud.py | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/lightning_app/runners/cloud.py b/src/lightning_app/runners/cloud.py index 49165f9d973af..8a7170ee0244a 100644 --- a/src/lightning_app/runners/cloud.py +++ b/src/lightning_app/runners/cloud.py @@ -236,7 +236,7 @@ def dispatch( f.write("venv/\n") if (root / "bin" / "activate").is_file() or (root / "pyvenv.cfg").is_file(): # the user is developing inside venv - f.write("bin/\n" "include/\n" "lib/\n" "pyvenv.cfg\n") + f.write("bin/\ninclude/\nlib/\npyvenv.cfg\n") repo = LocalSourceCodeDir(path=root, ignore_functions=ignore_functions) self._check_uploaded_folder(root, repo) @@ -565,7 +565,8 @@ def _check_uploaded_folder(root: Path, repo: LocalSourceCodeDir) -> None: f"The total size is {round(app_folder_size_in_mb, 2)} MB. {len(files)} files were uploaded.\n" + largest_paths_msg + "Perhaps you should try running the app in an empty directory.\n" - "You can ignore some files or folders by adding them to `.lightningignore`." + + "You can ignore some files or folders by adding them to `.lightningignore`.\n" + + " You can also set the `self.lightningingore` attribute in a Flow or Work." ) logger.warn(warning_msg) diff --git a/tests/tests_app/runners/test_cloud.py b/tests/tests_app/runners/test_cloud.py index 770c86dfec4b2..4d1ebda6ccdc5 100644 --- a/tests/tests_app/runners/test_cloud.py +++ b/tests/tests_app/runners/test_cloud.py @@ -1335,6 +1335,7 @@ def test_check_uploaded_folder(monkeypatch, tmpdir, caplog): assert "3 files were uploaded" in caplog.text assert "files:\n6.0 MB: c.jpg\n5.0 MB: b.txt\n4.0 MB: a.png\nPerhaps" in caplog.text # tests the order assert "adding them to `.lightningignore`." in caplog.text + assert "lightningingore` attribute in a Flow or Work" in caplog.text @mock.patch("lightning_app.core.queues.QueuingSystem", MagicMock()) @@ -1598,6 +1599,8 @@ def run(self): write_file_of_size(path / "a.txt", 5 * 1000 * 1000) write_file_of_size(path / "venv" / "foo.txt", 4 * 1000 * 1000) + assert not (path / ".lightningignore").exists() + with mock.patch( "lightning_app.runners.cloud._parse_lightningignore", wraps=_parse_lightningignore ) as parse_mock, mock.patch(