tests: split examples and pytests (#15774)

split examples and pytests

(cherry picked from commit 952b64b)

Author: Borda

.github/checkgroup.yml

@@ -221,7 +221,6 @@ subprojects:
       - ".github/workflows/ci-app-tests.yml"
       - "src/lightning_app/**"
       - "tests/tests_app/**"
-      - "examples/app_*/**"  # some tests_app tests call examples files
       - "requirements/app/**"
       - "setup.py"
       - ".actions/**"

.github/workflows/ci-app-tests.yml

@@ -11,7 +11,6 @@ on:
       - ".github/workflows/ci-app-tests.yml"
       - "src/lightning_app/**"
       - "tests/tests_app/**"
-      - "examples/app_*/**"  # some tests_app tests call examples files
       - "requirements/app/**"
       - "setup.py"
       - ".actions/**"

tests/tests_app/conftest.py

@@ -3,37 +3,20 @@
 import threading
 from datetime import datetime
 from pathlib import Path
-from subprocess import Popen
 
 import psutil
 import py
 import pytest
-from tests_app import _PROJECT_ROOT
 
 from lightning_app.storage.path import _storage_root_dir
 from lightning_app.utilities.component import _set_context
 from lightning_app.utilities.packaging import cloud_compute
 from lightning_app.utilities.packaging.app_config import _APP_CONFIG_FILENAME
 from lightning_app.utilities.state import AppState
 
-GITHUB_APP_URLS = {
-    "template_react_ui": "https://github.com/Lightning-AI/lightning-template-react.git",
-}
-
 os.environ["LIGHTNING_DISPATCHED"] = "1"
 
 
-def pytest_sessionstart(*_):
-    """Pytest hook that get called after the Session object has been created and before performing collection and
-    entering the run test loop."""
-    for name, url in GITHUB_APP_URLS.items():
-        if not os.path.exists(os.path.join(_PROJECT_ROOT, "examples", name)):
-            path_examples = os.path.join(_PROJECT_ROOT, "examples")
-            Popen(["git", "clone", url, name], cwd=path_examples).wait(timeout=90)
-        else:
-            Popen(["git", "pull", "main"], cwd=os.path.join(_PROJECT_ROOT, "examples", name)).wait(timeout=90)
-
-
 def pytest_sessionfinish(session, exitstatus):
     """Pytest hook that get called after whole test run finished, right before returning the exit status to the
     system."""

tests/tests_app/runners/test_cloud.py

@@ -41,15 +41,15 @@
     V1Work,
 )
 
-from lightning_app import _PROJECT_ROOT, BuildConfig, LightningApp, LightningWork
+from lightning_app import BuildConfig, LightningApp, LightningWork
 from lightning_app.runners import backends, cloud, CloudRuntime
 from lightning_app.runners.cloud import (
     _generate_works_json_gallery,
     _generate_works_json_web,
     _validate_build_spec_and_compute,
 )
 from lightning_app.storage import Drive, Mount
-from lightning_app.testing.helpers import EmptyFlow, EmptyWork
+from lightning_app.testing.helpers import EmptyWork
 from lightning_app.utilities.cloud import _get_project
 from lightning_app.utilities.dependency_caching import get_hash
 from lightning_app.utilities.packaging.cloud_compute import CloudCompute
@@ -1222,16 +1222,6 @@ def test_project_has_sufficient_credits():
         assert cloud_runtime._project_has_sufficient_credits(project) is result
 
 
-@mock.patch(
-    "lightning_app.runners.cloud.load_app_from_file",
-    MagicMock(side_effect=ModuleNotFoundError("Module X not found")),
-)
-def test_load_app_from_file_module_error():
-    empty_app = CloudRuntime.load_app_from_file(os.path.join(_PROJECT_ROOT, "examples", "app_v0", "app.py"))
-    assert isinstance(empty_app, LightningApp)
-    assert isinstance(empty_app.root, EmptyFlow)
-
-
 @pytest.mark.parametrize(
     "lines",
     [

tests/tests_app/utilities/test_load_app.py

@@ -3,17 +3,12 @@
 
 import pytest
 import tests_app.core.scripts
-from tests_app import _PROJECT_ROOT
 
-from lightning_app import LightningApp
 from lightning_app.utilities.exceptions import MisconfigurationException
 from lightning_app.utilities.load_app import extract_metadata_from_app, load_app_from_file
 
 
 def test_load_app_from_file():
-    app = load_app_from_file(os.path.join(_PROJECT_ROOT, "examples", "app_v0", "app.py"))
-    assert isinstance(app, LightningApp)
-
     test_script_dir = os.path.join(os.path.dirname(tests_app.core.__file__), "scripts")
     with pytest.raises(MisconfigurationException, match="There should not be multiple apps instantiated within a file"):
         load_app_from_file(os.path.join(test_script_dir, "two_apps.py"))

tests/tests_app_examples/conftest.py

@@ -1,19 +1,36 @@
 import os
 import shutil
 import threading
+from subprocess import Popen
 
 import psutil
 import pytest
 
+from lightning_app import _PROJECT_ROOT
 from lightning_app.storage.path import _storage_root_dir
 from lightning_app.utilities.component import _set_context
 from lightning_app.utilities.packaging import cloud_compute
 from lightning_app.utilities.packaging.app_config import _APP_CONFIG_FILENAME
 from lightning_app.utilities.state import AppState
 
+GITHUB_APP_URLS = {
+    "template_react_ui": "https://github.com/Lightning-AI/lightning-template-react.git",
+}
+
 os.environ["LIGHTNING_DISPATCHED"] = "1"
 
 
+def pytest_sessionstart(*_):
+    """Pytest hook that get called after the Session object has been created and before performing collection and
+    entering the run test loop."""
+    for name, url in GITHUB_APP_URLS.items():
+        if not os.path.exists(os.path.join(_PROJECT_ROOT, "examples", name)):
+            path_examples = os.path.join(_PROJECT_ROOT, "examples")
+            Popen(["git", "clone", url, name], cwd=path_examples).wait(timeout=90)
+        else:
+            Popen(["git", "pull", "main"], cwd=os.path.join(_PROJECT_ROOT, "examples", name)).wait(timeout=90)
+
+
 def pytest_sessionfinish(session, exitstatus):
     """Pytest hook that get called after whole test run finished, right before returning the exit status to the
     system."""

tests/tests_app/components/serve/test_gradio.py renamed to tests/tests_app_examples/test_gradio.py

@@ -1,12 +1,18 @@
 import os
 from time import sleep
 from typing import Tuple
+from unittest import mock
+from unittest.mock import MagicMock
 
 import pytest
 from tests_app import _PROJECT_ROOT
 
+from lightning_app import LightningApp
+from lightning_app.runners import CloudRuntime
+from lightning_app.testing import EmptyFlow
 from lightning_app.testing.testing import application_testing, LightningTestApp, run_app_in_cloud, wait_for
 from lightning_app.utilities.enum import AppStage
+from lightning_app.utilities.load_app import load_app_from_file
 
 
 class LightningAppTestInt(LightningTestApp):
@@ -78,3 +84,18 @@ def test_v0_app_example_cloud() -> None:
         _,
     ):
         run_v0_app(fetch_logs, view_page)
+
+
+@mock.patch(
+    "lightning_app.runners.cloud.load_app_from_file",
+    MagicMock(side_effect=ModuleNotFoundError("Module X not found")),
+)
+def test_load_app_from_file_module_error():
+    empty_app = CloudRuntime.load_app_from_file(os.path.join(_PATH_EXAMPLES, "app_v0", "app.py"))
+    assert isinstance(empty_app, LightningApp)
+    assert isinstance(empty_app.root, EmptyFlow)
+
+
+def test_load_app_from_file():
+    app = load_app_from_file(os.path.join(_PATH_EXAMPLES, "app_v0", "app.py"))
+    assert isinstance(app, LightningApp)