[App] Add rm one level below project level (#16740)

Co-authored-by: Ethan Harris <[email protected]>
Co-authored-by: Justus Schock <[email protected]>
Co-authored-by: thomas <[email protected]>

Author: 4 people

examples/app_commands_and_api/app.py

@@ -1,8 +1,8 @@
 from command import CustomCommand, CustomConfig
 
 from lightning import LightningFlow
-from lightning_app.api import Get, Post
-from lightning_app.core.app import LightningApp
+from lightning.app.api import Get, Post
+from lightning.app.core.app import LightningApp
 
 
 async def handler():

src/lightning/app/CHANGELOG.md

@@ -9,6 +9,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 
 ### Added
 
+- Added Storage Commands ([#16740](https://github.com/Lightning-AI/lightning/pull/16740))
+  * `rm`: Delete files from your Cloud Platform Filesystem
+
 - Added `lightning connect data` to register data connection to private s3 buckets ([#16738](https://github.com/Lightning-AI/lightning/pull/16738))
 
 

src/lightning/app/cli/commands/cp.py

@@ -57,7 +57,7 @@ def cp(src_path: str, dst_path: str, r: bool = False, recursive: bool = False) -
         if pwd == "/" or len(pwd.split("/")) == 1:
             return _error_and_exit("Uploading files at the project level isn't allowed yet.")
 
-        client = LightningClient()
+        client = LightningClient(retry=False)
 
         src_path, src_remote = _sanitize_path(src_path, pwd)
         dst_path, dst_remote = _sanitize_path(dst_path, pwd)
@@ -87,6 +87,9 @@ def _upload_files(live, client: LightningClient, local_src: str, remote_dst: str
     else:
         project_id = _get_project_id_from_name(remote_dst)
 
+    if len(remote_splits) > 2:
+        remote_dst = os.path.join(*remote_splits[2:])
+
     local_src = Path(local_src).resolve()
     upload_paths = []
 
@@ -101,6 +104,8 @@ def _upload_files(live, client: LightningClient, local_src: str, remote_dst: str
 
     clusters = client.projects_service_list_project_cluster_bindings(project_id)
 
+    live.stop()
+
     for upload_path in upload_paths:
         for cluster in clusters.clusters:
             filename = str(upload_path).replace(str(os.getcwd()), "")[1:]

src/lightning/app/cli/commands/ls.py

@@ -18,6 +18,7 @@
 from typing import Generator, List, Optional
 
 import click
+import lightning_cloud
 import rich
 from lightning_cloud.openapi import Externalv1LightningappInstance
 from rich.console import Console
@@ -255,7 +256,7 @@ def _collect_artifacts(
                     page_token=response.next_page_token,
                     tokens=tokens,
                 )
-        except Exception:
+        except lightning_cloud.openapi.rest.ApiException:
             # Note: This is triggered when the request is wrong.
             # This is currently happening due to looping through the user clusters.
             pass

src/lightning/app/cli/commands/rm.py

@@ -0,0 +1,104 @@
+# Copyright The Lightning AI team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+import click
+import lightning_cloud
+import rich
+
+from lightning.app.cli.commands.ls import _add_colors, _get_prefix
+from lightning.app.cli.commands.pwd import _pwd
+from lightning.app.utilities.app_helpers import Logger
+from lightning.app.utilities.cli_helpers import _error_and_exit
+from lightning.app.utilities.network import LightningClient
+
+logger = Logger(__name__)
+
+
+@click.argument("rm_path", required=True)
+@click.option("-r", required=False, hidden=True)
+@click.option("--recursive", required=False, hidden=True)
+def rm(rm_path: str, r: bool = False, recursive: bool = False) -> None:
+    """Delete files on the Lightning Cloud filesystem."""
+
+    root = _pwd()
+
+    if rm_path in (".", ".."):
+        return _error_and_exit('rm "." and ".." may not be removed')
+
+    if ".." in rm_path:
+        return _error_and_exit('rm ".." or higher may not be removed')
+
+    root = os.path.join(root, rm_path)
+    splits = [split for split in root.split("/") if split != ""]
+
+    if root == "/" or len(splits) == 1:
+        return _error_and_exit("rm at the project level isn't supported")
+
+    client = LightningClient(retry=False)
+    projects = client.projects_service_list_memberships()
+
+    project = [project for project in projects.memberships if project.name == splits[0]]
+
+    # This happens if the user changes cluster and the project doesn't exist.
+    if len(project) == 0:
+        return _error_and_exit(
+            f"There isn't any Lightning Project matching the name {splits[0]}." " HINT: Use `lightning cd`."
+        )
+
+    project_id = project[0].project_id
+
+    # Parallelise calls
+    lit_apps = client.lightningapp_instance_service_list_lightningapp_instances(project_id=project_id, async_req=True)
+    lit_cloud_spaces = client.cloud_space_service_list_cloud_spaces(project_id=project_id, async_req=True)
+
+    lit_apps = lit_apps.get().lightningapps
+    lit_cloud_spaces = lit_cloud_spaces.get().cloudspaces
+
+    lit_ressources = [lit_resource for lit_resource in lit_cloud_spaces if lit_resource.name == splits[1]]
+
+    if len(lit_ressources) == 0:
+
+        lit_ressources = [lit_resource for lit_resource in lit_apps if lit_resource.name == splits[1]]
+
+        if len(lit_ressources) == 0:
+            _error_and_exit(f"There isn't any Lightning Ressource matching the name {splits[1]}.")
+
+    lit_resource = lit_ressources[0]
+
+    prefix = "/".join(splits[2:])
+    prefix = _get_prefix(prefix, lit_resource)
+
+    clusters = client.projects_service_list_project_cluster_bindings(project_id)
+    succeeded = False
+
+    for cluster in clusters.clusters:
+        try:
+            client.lightningapp_instance_service_delete_project_artifact(
+                project_id=project_id,
+                cluster_id=cluster.cluster_id,
+                filename=prefix,
+            )
+            succeeded = True
+            break
+        except lightning_cloud.openapi.rest.ApiException:
+            pass
+
+    prefix = os.path.join(*splits)
+
+    if succeeded:
+        rich.print(_add_colors(f"Successfuly deleted `{prefix}`.", color="green"))
+    else:
+        return _error_and_exit(f"No file or folder named `{prefix}` was found.")

src/lightning/app/cli/lightning_cli.py

@@ -38,6 +38,7 @@
 from lightning.app.cli.commands.logs import logs
 from lightning.app.cli.commands.ls import ls
 from lightning.app.cli.commands.pwd import pwd
+from lightning.app.cli.commands.rm import rm
 from lightning.app.cli.connect.app import (
     _list_app_commands,
     _retrieve_connection_to_an_app,
@@ -144,6 +145,7 @@ def disconnect() -> None:
 _main.command(hidden=True)(cd)
 _main.command(hidden=True)(cp)
 _main.command(hidden=True)(pwd)
+_main.command(hidden=True)(rm)
 show.command()(logs)
 
 

src/lightning/app/testing/testing.py

@@ -282,7 +282,7 @@ def run_app_in_cloud(
     token = res.json()["token"]
 
     # 3. Disconnect from the App if any.
-    Popen("lightning disconnect", shell=True).wait()
+    Popen("lightning logout", shell=True).wait()
 
     # 4. Launch the application in the cloud from the Lightning CLI.
     with tempfile.TemporaryDirectory() as tmpdir:
@@ -392,6 +392,9 @@ def run_app_in_cloud(
 
         admin_page.locator(f'[data-cy="{name}"]').click()
 
+        app_url = admin_page.url
+        admin_page.goto(app_url + "/logs")
+
         client = LightningClient()
         project_id = _get_project(client).project_id
 

src/lightning/app/utilities/cli_helpers.py

@@ -357,6 +357,6 @@ def _check_environment_and_redirect():
     return
 
 
-def _error_and_exit(msg: str) -> str:
+def _error_and_exit(msg: str) -> None:
     rich.print(f"[red]ERROR[/red]: {msg}")
     sys.exit(0)

tests/integrations_app/public/test_commands_and_api.py

@@ -23,7 +23,7 @@ def test_commands_and_api_example_cloud() -> None:
         cmd_2 = "python -m lightning command with client --name=this"
         cmd_3 = "python -m lightning command without client --name=is"
         cmd_4 = "python -m lightning command without client --name=awesome"
-        cmd_5 = "lightning disconnect app"
+        cmd_5 = "lightning logout"
         process = Popen(" && ".join([cmd_1, cmd_2, cmd_3, cmd_4, cmd_5]), shell=True)
         process.wait()
 

tests/tests_app/cli/test_rm.py

@@ -0,0 +1,109 @@
+import os
+import sys
+from unittest.mock import MagicMock
+
+import pytest
+from lightning_cloud.openapi import (
+    Externalv1LightningappInstance,
+    V1LightningappInstanceArtifact,
+    V1ListCloudSpacesResponse,
+    V1ListLightningappInstanceArtifactsResponse,
+    V1ListLightningappInstancesResponse,
+    V1ListMembershipsResponse,
+    V1Membership,
+)
+
+from lightning.app.cli.commands import cd, ls, rm
+
+
+@pytest.mark.skipif(sys.platform == "win32", reason="not supported on windows yet")
+def test_rm(monkeypatch):
+    """This test validates rm behaves as expected."""
+
+    if os.path.exists(cd._CD_FILE):
+        os.remove(cd._CD_FILE)
+
+    client = MagicMock()
+    client.projects_service_list_memberships.return_value = V1ListMembershipsResponse(
+        memberships=[
+            V1Membership(name="project-0", project_id="project-id-0"),
+            V1Membership(name="project-1", project_id="project-id-1"),
+            V1Membership(name="project 2", project_id="project-id-2"),
+        ]
+    )
+
+    client.lightningapp_instance_service_list_lightningapp_instances().get.return_value = (
+        V1ListLightningappInstancesResponse(
+            lightningapps=[
+                Externalv1LightningappInstance(
+                    name="app-name-0",
+                    id="app-id-0",
+                ),
+                Externalv1LightningappInstance(
+                    name="app-name-1",
+                    id="app-id-1",
+                ),
+                Externalv1LightningappInstance(
+                    name="app name 2",
+                    id="app-id-1",
+                ),
+            ]
+        )
+    )
+
+    client.cloud_space_service_list_cloud_spaces().get.return_value = V1ListCloudSpacesResponse(cloudspaces=[])
+
+    clusters = MagicMock()
+    clusters.clusters = [MagicMock()]
+    client.projects_service_list_project_cluster_bindings.return_value = clusters
+
+    def fn(*args, prefix, **kwargs):
+        splits = [split for split in prefix.split("/") if split != ""]
+        if len(splits) == 2:
+            return V1ListLightningappInstanceArtifactsResponse(
+                artifacts=[
+                    V1LightningappInstanceArtifact(filename="file_1.txt"),
+                    V1LightningappInstanceArtifact(filename="folder_1/file_2.txt"),
+                    V1LightningappInstanceArtifact(filename="folder_2/folder_3/file_3.txt"),
+                    V1LightningappInstanceArtifact(filename="folder_2/file_4.txt"),
+                ]
+            )
+        elif splits[-1] == "folder_1":
+            return V1ListLightningappInstanceArtifactsResponse(
+                artifacts=[V1LightningappInstanceArtifact(filename="file_2.txt")]
+            )
+        elif splits[-1] == "folder_2":
+            return V1ListLightningappInstanceArtifactsResponse(
+                artifacts=[
+                    V1LightningappInstanceArtifact(filename="folder_3/file_3.txt"),
+                    V1LightningappInstanceArtifact(filename="file_4.txt"),
+                ]
+            )
+        elif splits[-1] == "folder_3":
+            return V1ListLightningappInstanceArtifactsResponse(
+                artifacts=[
+                    V1LightningappInstanceArtifact(filename="file_3.txt"),
+                ]
+            )
+
+    client.lightningapp_instance_service_list_project_artifacts = fn
+
+    client.lightningapp_instance_service_delete_project_artifact = MagicMock()
+
+    monkeypatch.setattr(rm, "LightningClient", MagicMock(return_value=client))
+    monkeypatch.setattr(ls, "LightningClient", MagicMock(return_value=client))
+
+    assert ls.ls() == ["project-0", "project-1", "project 2"]
+    assert "/project-0" == cd.cd("project-0", verify=False)
+
+    assert f"/project-0{os.sep}app-name-1" == cd.cd("app-name-1", verify=False)
+
+    assert f"/project-0{os.sep}app-name-1{os.sep}folder_1" == cd.cd("folder_1", verify=False)
+
+    rm.rm("file_2.txt")
+
+    kwargs = client.lightningapp_instance_service_delete_project_artifact._mock_call_args.kwargs
+    assert kwargs["project_id"] == "project-id-0"
+    assert kwargs["filename"] == "/lightningapps/app-id-1/folder_1/file_2.txt"
+
+    os.remove(cd._CD_FILE)