From 0fc05e9090e53ec41feb91f3692cdd3ba6e84ac8 Mon Sep 17 00:00:00 2001 From: thomas Date: Tue, 13 Dec 2022 12:32:10 +0000 Subject: [PATCH 01/20] update --- src/lightning_app/cli/commands/connection.py | 245 ++++++++++-------- src/lightning_app/cli/lightning_cli.py | 2 - .../components/database/server.py | 6 +- 3 files changed, 148 insertions(+), 105 deletions(-) diff --git a/src/lightning_app/cli/commands/connection.py b/src/lightning_app/cli/commands/connection.py index ee0bf7edc5d67..0b19191295a7f 100644 --- a/src/lightning_app/cli/commands/connection.py +++ b/src/lightning_app/cli/commands/connection.py @@ -3,11 +3,14 @@ import shutil import sys from subprocess import Popen +from threading import Event, Thread +from time import sleep, time from typing import List, Optional, Tuple import click import psutil from lightning_utilities.core.imports import package_available +from rich.progress import Progress from lightning_app.utilities.cli_helpers import _LightningAppOpenAPIRetriever from lightning_app.utilities.cloud import _get_project @@ -16,7 +19,7 @@ from lightning_app.utilities.network import LightningClient _HOME = os.path.expanduser("~") -_PPID = str(psutil.Process(os.getpid()).ppid()) +_PPID = os.getenv("LIGHTNING_CONNECT_PPID", str(psutil.Process(os.getpid()).ppid())) _LIGHTNING_CONNECTION = os.path.join(_HOME, ".lightning", "lightning_connection") _LIGHTNING_CONNECTION_FOLDER = os.path.join(_LIGHTNING_CONNECTION, _PPID) @@ -51,47 +54,65 @@ def connect(app_name_or_id: str, yes: bool = False): elif app_name_or_id.startswith("localhost"): - if app_name_or_id != "localhost": - raise Exception("You need to pass localhost to connect to the local Lightning App.") + with Progress() as progress_bar: + total = 30 + event = Event() + connecting = progress_bar.add_task("[green]Connecting...", total=total) - retriever = _LightningAppOpenAPIRetriever(None) + thread = Thread(target=update_progresss, args=[event, progress_bar, connecting, total], daemon=True) + thread.start() - if retriever.api_commands is None: - raise Exception(f"The commands weren't found. Is your app {app_name_or_id} running ?") + if app_name_or_id != "localhost": + raise Exception("You need to pass localhost to connect to the local Lightning App.") - commands_folder = os.path.join(_LIGHTNING_CONNECTION_FOLDER, "commands") - if not os.path.exists(commands_folder): - os.makedirs(commands_folder) + retriever = _LightningAppOpenAPIRetriever(None) + + if retriever.api_commands is None: + raise Exception(f"The commands weren't found. Is your app {app_name_or_id} running ?") - _write_commands_metadata(retriever.api_commands) + commands_folder = os.path.join(_LIGHTNING_CONNECTION_FOLDER, "commands") + if not os.path.exists(commands_folder): + os.makedirs(commands_folder) - with open(os.path.join(commands_folder, "openapi.json"), "w") as f: - json.dump(retriever.openapi, f) + _write_commands_metadata(retriever.api_commands) - _install_missing_requirements(retriever, yes) + with open(os.path.join(commands_folder, "openapi.json"), "w") as f: + json.dump(retriever.openapi, f) - for command_name, metadata in retriever.api_commands.items(): - if "cls_path" in metadata: - target_file = os.path.join(commands_folder, f"{command_name.replace(' ','_')}.py") - _download_command( - command_name, - metadata["cls_path"], - metadata["cls_name"], - None, - target_file=target_file, - ) - repr_command_name = command_name.replace("_", " ") - click.echo(f"Storing `{repr_command_name}` at {target_file}") - else: - with open(os.path.join(commands_folder, f"{command_name}.txt"), "w") as f: - f.write(command_name) + _install_missing_requirements(retriever) + for command_name, metadata in retriever.api_commands.items(): + if "cls_path" in metadata: + target_file = os.path.join(commands_folder, f"{command_name.replace(' ','_')}.py") + _download_command( + command_name, + metadata["cls_path"], + metadata["cls_name"], + None, + target_file=target_file, + ) + else: + with open(os.path.join(commands_folder, f"{command_name}.txt"), "w") as f: + f.write(command_name) + + event.set() + thread.join() + + click.echo(" ") click.echo(f"You can review all the downloaded commands at {commands_folder}") + click.echo(" ") with open(connected_file, "w") as f: f.write(app_name_or_id + "\n") - click.echo("You are connected to the local Lightning App.") + click.echo("Use ``lightning --help`` in your terminal to get the following information:") + click.echo(" ") + Popen( + f"LIGHTNING_CONNECT_PPID={_PPID} {sys.executable} -m lightning --help", + shell=True, + stdout=sys.stdout, + stderr=sys.stderr, + ).wait() elif matched_connection_path: @@ -101,76 +122,89 @@ def connect(app_name_or_id: str, yes: bool = False): commands = os.path.join(_LIGHTNING_CONNECTION_FOLDER, "commands") shutil.copytree(matched_commands, commands) shutil.copy(matched_connected_file, connected_file) - copied_files = [el for el in os.listdir(commands) if os.path.splitext(el)[1] == ".py"] - click.echo("Found existing connection, reusing cached commands") - for target_file in copied_files: - pretty_command_name = os.path.splitext(target_file)[0].replace("_", " ") - click.echo(f"Storing `{pretty_command_name}` at {os.path.join(commands, target_file)}") - click.echo(f"You can review all the commands at {commands}") click.echo(" ") - click.echo(f"You are connected to the cloud Lightning App: {app_name_or_id}.") + click.echo(f"You can review all the downloaded commands at {commands_folder}") + click.echo(" ") + click.echo("Use ``lightning --help`` in your terminal to get the following information:") + click.echo(" ") + Popen( + f"LIGHTNING_CONNECT_PPID={_PPID} {sys.executable} -m lightning --help", + shell=True, + stdout=sys.stdout, + stderr=sys.stderr, + ).wait() else: + with Progress() as progress_bar: + total = 30 + event = Event() + connecting = progress_bar.add_task("[green]Connecting...", total=total) + + thread = Thread(target=update_progresss, args=[event, progress_bar, connecting, total], daemon=True) + thread.start() + + retriever = _LightningAppOpenAPIRetriever(app_name_or_id) + + if not retriever.api_commands: + client = LightningClient() + project = _get_project(client) + apps = client.lightningapp_instance_service_list_lightningapp_instances(project_id=project.project_id) + click.echo( + "We didn't find a matching App. Here are the available Apps that could be " + f"connected to {[app.name for app in apps.lightningapps]}." + ) + return - retriever = _LightningAppOpenAPIRetriever(app_name_or_id) - - if not retriever.api_commands: - client = LightningClient() - project = _get_project(client) - apps = client.lightningapp_instance_service_list_lightningapp_instances(project_id=project.project_id) - click.echo( - "We didn't find a matching App. Here are the available Apps that could be " - f"connected to {[app.name for app in apps.lightningapps]}." - ) - return - - _install_missing_requirements(retriever, yes) - - if not yes: - yes = click.confirm( - f"The Lightning App `{app_name_or_id}` provides a command-line (CLI). " - "Do you want to proceed and install its CLI ?" - ) - click.echo(" ") - - if yes: - commands_folder = os.path.join(_LIGHTNING_CONNECTION_FOLDER, "commands") - if not os.path.exists(commands_folder): - os.makedirs(commands_folder) - - _write_commands_metadata(retriever.api_commands) + _install_missing_requirements(retriever) - for command_name, metadata in retriever.api_commands.items(): - if "cls_path" in metadata: - target_file = os.path.join(commands_folder, f"{command_name}.py") - _download_command( - command_name, - metadata["cls_path"], - metadata["cls_name"], - retriever.app_id, - target_file=target_file, - ) - pretty_command_name = command_name.replace("_", " ") - click.echo(f"Storing `{pretty_command_name}` at {target_file}") - else: - with open(os.path.join(commands_folder, f"{command_name}.txt"), "w") as f: - f.write(command_name) - - click.echo(f"You can review all the downloaded commands at {commands_folder}") + if not yes: + yes = click.confirm( + f"The Lightning App `{app_name_or_id}` provides a command-line (CLI). " + "Do you want to proceed and install its CLI ?" + ) + click.echo(" ") + + if yes: + commands_folder = os.path.join(_LIGHTNING_CONNECTION_FOLDER, "commands") + if not os.path.exists(commands_folder): + os.makedirs(commands_folder) + + _write_commands_metadata(retriever.api_commands) + + for command_name, metadata in retriever.api_commands.items(): + if "cls_path" in metadata: + target_file = os.path.join(commands_folder, f"{command_name}.py") + _download_command( + command_name, + metadata["cls_path"], + metadata["cls_name"], + retriever.app_id, + target_file=target_file, + ) + else: + with open(os.path.join(commands_folder, f"{command_name}.txt"), "w") as f: + f.write(command_name) + + event.set() + thread.join() - click.echo(" ") - click.echo("The client interface has been successfully installed. ") - click.echo("You can now run the following commands:") - for command in retriever.api_commands: - pretty_command_name = command.replace("_", " ") - click.echo(f" lightning {pretty_command_name}") + click.echo(" ") + click.echo(f"You can review all the downloaded commands at {commands_folder}") + click.echo(" ") with open(connected_file, "w") as f: f.write(retriever.app_name + "\n") f.write(retriever.app_id + "\n") + + click.echo("Use ``lightning --help`` in your terminal to get the following information:") click.echo(" ") - click.echo(f"You are connected to the cloud Lightning App: {app_name_or_id}.") + Popen( + f"LIGHTNING_CONNECT_PPID={_PPID} {sys.executable} -m lightning --help", + shell=True, + stdout=sys.stdout, + stderr=sys.stderr, + ).wait() def disconnect(logout: bool = False): @@ -259,7 +293,6 @@ def _list_app_commands(echo: bool = True) -> List[str]: def _install_missing_requirements( retriever: _LightningAppOpenAPIRetriever, - yes_global: bool = False, fail_if_missing: bool = False, ): requirements = set() @@ -281,20 +314,15 @@ def _install_missing_requirements( sys.exit(0) for req in missing_requirements: - if not yes_global: - yes = click.confirm( - f"The Lightning App CLI `{retriever.app_id}` requires `{req}`. Do you want to install it ?" - ) - else: - print(f"Installing missing `{req}` requirement.") - yes = yes_global - if yes: - std_out_out = get_logfile("output.log") - with open(std_out_out, "wb") as stdout: - Popen( - f"{sys.executable} -m pip install {req}", shell=True, stdout=stdout, stderr=sys.stderr - ).wait() - print() + std_out_out = get_logfile("output.log") + with open(std_out_out, "wb") as stdout: + Popen( + f"{sys.executable} -m pip install {req}", + shell=True, + stdout=stdout, + stderr=stdout, + ).wait() + os.remove(std_out_out) def _clean_lightning_connection(): @@ -332,3 +360,16 @@ def _scan_lightning_connections(app_name_or_id): return connection_path return None + + +def update_progresss(exit_event, progress_bar, task, total): + t0 = time() + while not exit_event.is_set(): + sleep(0.5) + progress_bar.update(task, advance=0.5) + + # Note: Not required but make progress feel more natural + remaning = total - (time() - t0) + for _ in range(10): + progress_bar.update(task, advance=remaning / 10.0) + sleep(0.2) diff --git a/src/lightning_app/cli/lightning_cli.py b/src/lightning_app/cli/lightning_cli.py index 87b0ef91755b1..f17d4775784d9 100644 --- a/src/lightning_app/cli/lightning_cli.py +++ b/src/lightning_app/cli/lightning_cli.py @@ -77,8 +77,6 @@ def main() -> None: else: message = f"You are connected to the cloud Lightning App: {app_name}." - click.echo(" ") - if (len(sys.argv) > 1 and sys.argv[1] in ["-h", "--help"]) or len(sys.argv) == 1: _list_app_commands() else: diff --git a/src/lightning_app/components/database/server.py b/src/lightning_app/components/database/server.py index 6d187e4cda133..0a93bf94f985b 100644 --- a/src/lightning_app/components/database/server.py +++ b/src/lightning_app/components/database/server.py @@ -14,6 +14,7 @@ from lightning_app.components.database.utilities import _create_database, _Delete, _Insert, _SelectAll, _Update from lightning_app.core.work import LightningWork from lightning_app.storage import Drive +from lightning_app.utilities.app_helpers import Logger from lightning_app.utilities.imports import _is_sqlmodel_available from lightning_app.utilities.packaging.build_config import BuildConfig @@ -23,6 +24,9 @@ SQLModel = object +logger = Logger(__name__) + + # Required to avoid Uvicorn Server overriding Lightning App signal handlers. # Discussions: https://github.com/encode/uvicorn/discussions/1708 class _DatabaseUvicornServer(uvicorn.Server): @@ -167,7 +171,7 @@ def store_database(self): drive = Drive("lit://database", component_name=self.name, root_folder=tmpdir) drive.put(os.path.basename(tmp_db_filename)) - print("Stored the database to the Drive.") + logger.debug("Stored the database to the Drive.") except Exception: print(traceback.print_exc()) From 1176694ec38cb49ade691ed18a2b223a7c1fbeab Mon Sep 17 00:00:00 2001 From: thomas Date: Tue, 13 Dec 2022 14:38:22 +0000 Subject: [PATCH 02/20] update --- src/lightning_app/cli/commands/connection.py | 130 ++++++++++--------- src/lightning_app/runners/multiprocess.py | 2 +- src/lightning_app/utilities/cli_helpers.py | 4 + src/lightning_app/utilities/commands/base.py | 12 +- src/lightning_app/utilities/frontend.py | 1 + 5 files changed, 83 insertions(+), 66 deletions(-) diff --git a/src/lightning_app/cli/commands/connection.py b/src/lightning_app/cli/commands/connection.py index 0b19191295a7f..d9f59895e8bb5 100644 --- a/src/lightning_app/cli/commands/connection.py +++ b/src/lightning_app/cli/commands/connection.py @@ -10,7 +10,7 @@ import click import psutil from lightning_utilities.core.imports import package_available -from rich.progress import Progress +from rich.progress import Progress, Task from lightning_app.utilities.cli_helpers import _LightningAppOpenAPIRetriever from lightning_app.utilities.cloud import _get_project @@ -22,11 +22,11 @@ _PPID = os.getenv("LIGHTNING_CONNECT_PPID", str(psutil.Process(os.getpid()).ppid())) _LIGHTNING_CONNECTION = os.path.join(_HOME, ".lightning", "lightning_connection") _LIGHTNING_CONNECTION_FOLDER = os.path.join(_LIGHTNING_CONNECTION, _PPID) +_PROGRESS_TOTAL = 60 @click.argument("app_name_or_id", required=True) -@click.option("-y", "--yes", required=False, is_flag=True, help="Whether to download the commands automatically.") -def connect(app_name_or_id: str, yes: bool = False): +def connect(app_name_or_id: str): """Connect to a Lightning App.""" from lightning_app.utilities.commands.base import _download_command @@ -50,16 +50,14 @@ def connect(app_name_or_id: str, yes: bool = False): click.echo(f"You are already connected to the cloud Lightning App: {app_name_or_id}.") else: disconnect() - connect(app_name_or_id, yes) + connect(app_name_or_id) elif app_name_or_id.startswith("localhost"): with Progress() as progress_bar: - total = 30 event = Event() - connecting = progress_bar.add_task("[green]Connecting...", total=total) - - thread = Thread(target=update_progresss, args=[event, progress_bar, connecting, total], daemon=True) + connecting = progress_bar.add_task("[magenta]Setting things up for you...", total=_PROGRESS_TOTAL) + thread = Thread(target=update_progresss, args=[event, progress_bar, connecting], daemon=True) thread.start() if app_name_or_id != "localhost": @@ -98,15 +96,15 @@ def connect(app_name_or_id: str, yes: bool = False): event.set() thread.join() - click.echo(" ") - click.echo(f"You can review all the downloaded commands at {commands_folder}") - click.echo(" ") - with open(connected_file, "w") as f: f.write(app_name_or_id + "\n") - click.echo("Use ``lightning --help`` in your terminal to get the following information:") + click.echo( + "The lightning CLI will now respond to the commands exposed by App." + " Use ``lightning --help`` in your terminal to get the following information:" + ) click.echo(" ") + Popen( f"LIGHTNING_CONNECT_PPID={_PPID} {sys.executable} -m lightning --help", shell=True, @@ -123,11 +121,12 @@ def connect(app_name_or_id: str, yes: bool = False): shutil.copytree(matched_commands, commands) shutil.copy(matched_connected_file, connected_file) + click.echo( + "The lightning CLI will now respond to the commands exposed by App." + " Use ``lightning --help`` in your terminal to get the following information:" + ) click.echo(" ") - click.echo(f"You can review all the downloaded commands at {commands_folder}") - click.echo(" ") - click.echo("Use ``lightning --help`` in your terminal to get the following information:") - click.echo(" ") + Popen( f"LIGHTNING_CONNECT_PPID={_PPID} {sys.executable} -m lightning --help", shell=True, @@ -137,11 +136,9 @@ def connect(app_name_or_id: str, yes: bool = False): else: with Progress() as progress_bar: - total = 30 event = Event() - connecting = progress_bar.add_task("[green]Connecting...", total=total) - - thread = Thread(target=update_progresss, args=[event, progress_bar, connecting, total], daemon=True) + connecting = progress_bar.add_task("[magenta]Setting things up for you...", total=_PROGRESS_TOTAL) + thread = Thread(target=update_progresss, args=[event, progress_bar, connecting], daemon=True) thread.start() retriever = _LightningAppOpenAPIRetriever(app_name_or_id) @@ -158,47 +155,39 @@ def connect(app_name_or_id: str, yes: bool = False): _install_missing_requirements(retriever) - if not yes: - yes = click.confirm( - f"The Lightning App `{app_name_or_id}` provides a command-line (CLI). " - "Do you want to proceed and install its CLI ?" - ) - click.echo(" ") - - if yes: - commands_folder = os.path.join(_LIGHTNING_CONNECTION_FOLDER, "commands") - if not os.path.exists(commands_folder): - os.makedirs(commands_folder) - - _write_commands_metadata(retriever.api_commands) - - for command_name, metadata in retriever.api_commands.items(): - if "cls_path" in metadata: - target_file = os.path.join(commands_folder, f"{command_name}.py") - _download_command( - command_name, - metadata["cls_path"], - metadata["cls_name"], - retriever.app_id, - target_file=target_file, - ) - else: - with open(os.path.join(commands_folder, f"{command_name}.txt"), "w") as f: - f.write(command_name) + commands_folder = os.path.join(_LIGHTNING_CONNECTION_FOLDER, "commands") + if not os.path.exists(commands_folder): + os.makedirs(commands_folder) + + _write_commands_metadata(retriever.api_commands) + + for command_name, metadata in retriever.api_commands.items(): + if "cls_path" in metadata: + target_file = os.path.join(commands_folder, f"{command_name}.py") + _download_command( + command_name, + metadata["cls_path"], + metadata["cls_name"], + retriever.app_id, + target_file=target_file, + ) + else: + with open(os.path.join(commands_folder, f"{command_name}.txt"), "w") as f: + f.write(command_name) event.set() thread.join() - click.echo(" ") - click.echo(f"You can review all the downloaded commands at {commands_folder}") - click.echo(" ") - with open(connected_file, "w") as f: f.write(retriever.app_name + "\n") f.write(retriever.app_id + "\n") - click.echo("Use ``lightning --help`` in your terminal to get the following information:") + click.echo( + "The lightning CLI will now respond to the commands exposed by App." + " Use ``lightning --help`` in your terminal to get the following information:" + ) click.echo(" ") + Popen( f"LIGHTNING_CONNECT_PPID={_PPID} {sys.executable} -m lightning --help", shell=True, @@ -278,16 +267,32 @@ def _list_app_commands(echo: bool = True) -> List[str]: click.echo("The current Lightning App doesn't have commands.") return [] + app_info = metadata[command_names[0]].get("app_info", None) + + title, description = "Lightning", None + if app_info: + title = app_info["title"] + description = app_info["description"] + on_after_connect = app_info["on_after_connect"] + if echo: - click.echo("Usage: lightning [OPTIONS] COMMAND [ARGS]...") - click.echo("") - click.echo(" --help Show this message and exit.") + click.echo(f"{title} App") + if description: + click.echo("") + click.echo("Description:") + if description.endswith("\n"): + description = description[:-2] + click.echo(f" {description}") click.echo("") - click.echo("Lightning App Commands") + click.echo("Commands:") max_length = max(len(n) for n in command_names) for command_name in command_names: padding = (max_length + 1 - len(command_name)) * " " click.echo(f" {command_name}{padding}{metadata[command_name].get('description', '')}") + if "LIGHTNING_CONNECT_PPID" in os.environ and on_after_connect: + if on_after_connect.endswith("\n"): + on_after_connect = on_after_connect[:-2] + click.echo(on_after_connect) return command_names @@ -362,14 +367,15 @@ def _scan_lightning_connections(app_name_or_id): return None -def update_progresss(exit_event, progress_bar, task, total): +def update_progresss(exit_event: Event, progress_bar: Progress, task: Task) -> None: t0 = time() while not exit_event.is_set(): sleep(0.5) progress_bar.update(task, advance=0.5) - # Note: Not required but make progress feel more natural - remaning = total - (time() - t0) - for _ in range(10): - progress_bar.update(task, advance=remaning / 10.0) + # Note: This is required to make progress feel more naturally progressing. + remaning = _PROGRESS_TOTAL - (time() - t0) + num_updates = 10 + for _ in range(num_updates): + progress_bar.update(task, advance=remaning / float(num_updates)) sleep(0.2) diff --git a/src/lightning_app/runners/multiprocess.py b/src/lightning_app/runners/multiprocess.py index 673e8601043d7..e5d34fb76800f 100644 --- a/src/lightning_app/runners/multiprocess.py +++ b/src/lightning_app/runners/multiprocess.py @@ -82,7 +82,7 @@ def dispatch(self, *args: Any, open_ui: bool = True, **kwargs: Any): if is_overridden("configure_commands", self.app.root): commands = _prepare_commands(self.app) - apis += _commands_to_api(commands) + apis += _commands_to_api(commands, info=self.app.info) kwargs = dict( apis=apis, diff --git a/src/lightning_app/utilities/cli_helpers.py b/src/lightning_app/utilities/cli_helpers.py index caa414e163ffc..0ec6eabd3022c 100644 --- a/src/lightning_app/utilities/cli_helpers.py +++ b/src/lightning_app/utilities/cli_helpers.py @@ -69,6 +69,7 @@ def _get_metadata_from_openapi(paths: Dict, path: str): cls_name = paths[path]["post"].get("cls_name", None) description = paths[path]["post"].get("description", None) requirements = paths[path]["post"].get("requirements", None) + app_info = paths[path]["post"].get("app_info", None) metadata = {"tag": tag, "parameters": {}} @@ -84,6 +85,9 @@ def _get_metadata_from_openapi(paths: Dict, path: str): if description: metadata["requirements"] = requirements + if app_info: + metadata["app_info"] = app_info + if not parameters: return metadata diff --git a/src/lightning_app/utilities/commands/base.py b/src/lightning_app/utilities/commands/base.py index 53aefe5725194..947456d8e4e1c 100644 --- a/src/lightning_app/utilities/commands/base.py +++ b/src/lightning_app/utilities/commands/base.py @@ -5,6 +5,7 @@ import shutil import sys import traceback +from dataclasses import asdict from getpass import getuser from importlib.util import module_from_spec, spec_from_file_location from tempfile import gettempdir @@ -16,6 +17,7 @@ from lightning_app.api.http_methods import Post from lightning_app.api.request_types import _APIRequest, _CommandRequest, _RequestResponse +from lightning_app.utilities import frontend from lightning_app.utilities.app_helpers import is_overridden, Logger from lightning_app.utilities.cloud import _get_project from lightning_app.utilities.network import LightningClient @@ -250,7 +252,7 @@ def _process_requests(app, requests: List[Union[_APIRequest, _CommandRequest]]) app.api_response_queue.put(responses) -def _collect_open_api_extras(command) -> Dict: +def _collect_open_api_extras(command, info) -> Dict: if not isinstance(command, ClientCommand): if command.__doc__ is not None: return {"description": command.__doc__} @@ -263,10 +265,14 @@ def _collect_open_api_extras(command) -> Dict: } if command.requirements: extras.update({"requirements": command.requirements}) + if info: + extras.update({"app_info": asdict(info)}) return extras -def _commands_to_api(commands: List[Dict[str, Union[Callable, ClientCommand]]]) -> List: +def _commands_to_api( + commands: List[Dict[str, Union[Callable, ClientCommand]]], info: Optional[frontend.AppInfo] = None +) -> List: """Convert user commands to API endpoint.""" api = [] for command in commands: @@ -278,7 +284,7 @@ def _commands_to_api(commands: List[Dict[str, Union[Callable, ClientCommand]]]) v.method if isinstance(v, ClientCommand) else v, method_name=k, tags=["app_client_command"] if isinstance(v, ClientCommand) else ["app_command"], - openapi_extra=_collect_open_api_extras(v), + openapi_extra=_collect_open_api_extras(v, info), ) ) return api diff --git a/src/lightning_app/utilities/frontend.py b/src/lightning_app/utilities/frontend.py index 315c119935b6f..e581b24f8368d 100644 --- a/src/lightning_app/utilities/frontend.py +++ b/src/lightning_app/utilities/frontend.py @@ -12,6 +12,7 @@ class AppInfo: image: Optional[str] = None # ensure the meta tags are correct or the UI might fail to load. meta_tags: Optional[List[str]] = None + on_after_connect: Optional[str] = None def update_index_file(ui_root: str, info: Optional[AppInfo] = None, root_path: str = "") -> None: From f1a35d6a3de6655818fa18fb1660a5eba4bd20ed Mon Sep 17 00:00:00 2001 From: thomas Date: Tue, 13 Dec 2022 14:59:26 +0000 Subject: [PATCH 03/20] update --- tests/tests_app/cli/test_connect.py | 53 ++++------------------------- 1 file changed, 7 insertions(+), 46 deletions(-) diff --git a/tests/tests_app/cli/test_connect.py b/tests/tests_app/cli/test_connect.py index a8924ab375db2..1b479b94921a1 100644 --- a/tests/tests_app/cli/test_connect.py +++ b/tests/tests_app/cli/test_connect.py @@ -1,6 +1,5 @@ import json import os -import sys from unittest.mock import MagicMock import click @@ -37,11 +36,8 @@ def monkeypatch_connection(monkeypatch, tmpdir, ppid): def test_connect_disconnect_local(tmpdir, monkeypatch): disconnect() - ppid = str(psutil.Process(os.getpid()).ppid()) - connection_path = monkeypatch_connection(monkeypatch, tmpdir, ppid=ppid) - with pytest.raises(Exception, match="The commands weren't found. Is your app localhost running ?"): - connect("localhost", True) + connect("localhost") with open(os.path.join(os.path.dirname(__file__), "jsons/connect_1.json")) as f: data = json.load(f) @@ -64,23 +60,14 @@ def fn(msg): response.status_code = 200 response.json.return_value = data monkeypatch.setattr(cli_helpers.requests, "get", MagicMock(return_value=response)) - connect("localhost", True) + connect("localhost") assert _retrieve_connection_to_an_app() == ("localhost", None) command_path = _resolve_command_path("nested_command") assert not os.path.exists(command_path) command_path = _resolve_command_path("command_with_client") assert os.path.exists(command_path) - s = "/" if sys.platform != "win32" else "\\" - command_folder_path = f"{connection_path}{s}commands" - expected = [ - f"Storing `command with client` at {command_folder_path}{s}command_with_client.py", - f"You can review all the downloaded commands at {command_folder_path}", - "You are connected to the local Lightning App.", - ] - assert messages == expected - messages = [] - connect("localhost", True) + connect("localhost") assert messages == ["You are connected to the local Lightning App."] messages = [] @@ -101,8 +88,6 @@ def test_connect_disconnect_cloud(tmpdir, monkeypatch): ppid_1 = str(psutil.Process(os.getpid()).ppid()) ppid_2 = "222" - connection_path = monkeypatch_connection(monkeypatch, tmpdir, ppid=ppid_1) - target_file = _resolve_command_path("command_with_client") if os.path.exists(target_file): @@ -153,7 +138,7 @@ def fn(msg): with open(data["paths"]["/command/command_with_client"]["post"]["cls_path"], "rb") as f: response.content = f.read() - connect("example", True) + connect("example") assert _retrieve_connection_to_an_app() == ("example", "1234") commands = _list_app_commands() assert commands == ["command with client", "command without client", "nested command"] @@ -161,39 +146,15 @@ def fn(msg): assert not os.path.exists(command_path) command_path = _resolve_command_path("command_with_client") assert os.path.exists(command_path) - s = "/" if sys.platform != "win32" else "\\" - command_folder_path = f"{connection_path}{s}commands" - expected = [ - f"Storing `command with client` at {command_folder_path}{s}command_with_client.py", - f"You can review all the downloaded commands at {command_folder_path}", - " ", - "The client interface has been successfully installed. ", - "You can now run the following commands:", - " lightning command without client", - " lightning command with client", - " lightning nested command", - " ", - "You are connected to the cloud Lightning App: example.", - "Usage: lightning [OPTIONS] COMMAND [ARGS]...", - "", - " --help Show this message and exit.", - "", - "Lightning App Commands", - " command with client A command with a client.", - " command without client A command without a client.", - " nested command A nested command.", - ] - assert messages == expected - messages = [] - connect("example", True) + connect("example") assert messages == ["You are already connected to the cloud Lightning App: example."] _ = monkeypatch_connection(monkeypatch, tmpdir, ppid=ppid_2) messages = [] - connect("example", True) - assert messages[0] == "Found existing connection, reusing cached commands" + connect("example") + assert "The lightning CLI will now respond to the commands exposed by App." in messages[0] messages = [] disconnect() From 0d09985d8b3acd3cf39389c8b7b07e719ddf799f Mon Sep 17 00:00:00 2001 From: thomas Date: Tue, 13 Dec 2022 15:28:33 +0000 Subject: [PATCH 04/20] update --- src/lightning_app/cli/commands/connection.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/src/lightning_app/cli/commands/connection.py b/src/lightning_app/cli/commands/connection.py index d9f59895e8bb5..04519f906c4f9 100644 --- a/src/lightning_app/cli/commands/connection.py +++ b/src/lightning_app/cli/commands/connection.py @@ -99,10 +99,7 @@ def connect(app_name_or_id: str): with open(connected_file, "w") as f: f.write(app_name_or_id + "\n") - click.echo( - "The lightning CLI will now respond to the commands exposed by App." - " Use ``lightning --help`` in your terminal to get the following information:" - ) + click.echo("The lightning CLI can now respond to app commands. Use 'lightning --help' to see them.") click.echo(" ") Popen( @@ -121,10 +118,7 @@ def connect(app_name_or_id: str): shutil.copytree(matched_commands, commands) shutil.copy(matched_connected_file, connected_file) - click.echo( - "The lightning CLI will now respond to the commands exposed by App." - " Use ``lightning --help`` in your terminal to get the following information:" - ) + click.echo("The lightning CLI can now respond to app commands. Use 'lightning --help' to see them.") click.echo(" ") Popen( @@ -182,10 +176,7 @@ def connect(app_name_or_id: str): f.write(retriever.app_name + "\n") f.write(retriever.app_id + "\n") - click.echo( - "The lightning CLI will now respond to the commands exposed by App." - " Use ``lightning --help`` in your terminal to get the following information:" - ) + click.echo("The lightning CLI can now respond to app commands. Use 'lightning --help' to see them.") click.echo(" ") Popen( From d5a48138acf35aab5c7d4fed8102b8a46751326a Mon Sep 17 00:00:00 2001 From: thomas Date: Tue, 13 Dec 2022 15:42:38 +0000 Subject: [PATCH 05/20] update --- src/lightning_app/cli/commands/connection.py | 8 ++++---- tests/tests_app/cli/test_connect.py | 5 ++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/lightning_app/cli/commands/connection.py b/src/lightning_app/cli/commands/connection.py index 04519f906c4f9..45c5212885c20 100644 --- a/src/lightning_app/cli/commands/connection.py +++ b/src/lightning_app/cli/commands/connection.py @@ -99,7 +99,7 @@ def connect(app_name_or_id: str): with open(connected_file, "w") as f: f.write(app_name_or_id + "\n") - click.echo("The lightning CLI can now respond to app commands. Use 'lightning --help' to see them.") + click.echo("The lightning CLI now respond to app commands. Use 'lightning --help' to see them.") click.echo(" ") Popen( @@ -118,7 +118,7 @@ def connect(app_name_or_id: str): shutil.copytree(matched_commands, commands) shutil.copy(matched_connected_file, connected_file) - click.echo("The lightning CLI can now respond to app commands. Use 'lightning --help' to see them.") + click.echo("The lightning CLI now respond to app commands. Use 'lightning --help' to see them.") click.echo(" ") Popen( @@ -176,7 +176,7 @@ def connect(app_name_or_id: str): f.write(retriever.app_name + "\n") f.write(retriever.app_id + "\n") - click.echo("The lightning CLI can now respond to app commands. Use 'lightning --help' to see them.") + click.echo("The lightning CLI now respond to app commands. Use 'lightning --help' to see them.") click.echo(" ") Popen( @@ -260,7 +260,7 @@ def _list_app_commands(echo: bool = True) -> List[str]: app_info = metadata[command_names[0]].get("app_info", None) - title, description = "Lightning", None + title, description, on_after_connect = "Lightning", None, None if app_info: title = app_info["title"] description = app_info["description"] diff --git a/tests/tests_app/cli/test_connect.py b/tests/tests_app/cli/test_connect.py index 1b479b94921a1..6b9d8e97ff1d7 100644 --- a/tests/tests_app/cli/test_connect.py +++ b/tests/tests_app/cli/test_connect.py @@ -154,18 +154,17 @@ def fn(msg): messages = [] connect("example") - assert "The lightning CLI will now respond to the commands exposed by App." in messages[0] + assert "The lightning CLI now respond to app commands" in messages[0] messages = [] disconnect() - print(messages) assert messages == ["You are disconnected from the cloud Lightning App: example."] _ = monkeypatch_connection(monkeypatch, tmpdir, ppid=ppid_1) messages = [] disconnect() - assert messages == ["You are disconnected from the cloud Lightning App: example."] + assert "You aren't connected to any Lightning App" in messages[0] messages = [] disconnect() From f8cf7bc6b996525e792aa93783a1d8c505a749fb Mon Sep 17 00:00:00 2001 From: Luca Antiga Date: Tue, 13 Dec 2022 23:17:03 +0100 Subject: [PATCH 06/20] Update src/lightning_app/cli/commands/connection.py --- src/lightning_app/cli/commands/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lightning_app/cli/commands/connection.py b/src/lightning_app/cli/commands/connection.py index 45c5212885c20..701844ddff370 100644 --- a/src/lightning_app/cli/commands/connection.py +++ b/src/lightning_app/cli/commands/connection.py @@ -66,7 +66,7 @@ def connect(app_name_or_id: str): retriever = _LightningAppOpenAPIRetriever(None) if retriever.api_commands is None: - raise Exception(f"The commands weren't found. Is your app {app_name_or_id} running ?") + raise Exception(f"Connection wasn't successful. Is your app {app_name_or_id} running?") commands_folder = os.path.join(_LIGHTNING_CONNECTION_FOLDER, "commands") if not os.path.exists(commands_folder): From ce643fadafe428788857ebb6f64648da7bf93a91 Mon Sep 17 00:00:00 2001 From: Luca Antiga Date: Tue, 13 Dec 2022 23:17:13 +0100 Subject: [PATCH 07/20] Update src/lightning_app/cli/commands/connection.py --- src/lightning_app/cli/commands/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lightning_app/cli/commands/connection.py b/src/lightning_app/cli/commands/connection.py index 701844ddff370..cfea2138a992c 100644 --- a/src/lightning_app/cli/commands/connection.py +++ b/src/lightning_app/cli/commands/connection.py @@ -99,7 +99,7 @@ def connect(app_name_or_id: str): with open(connected_file, "w") as f: f.write(app_name_or_id + "\n") - click.echo("The lightning CLI now respond to app commands. Use 'lightning --help' to see them.") + click.echo("The lightning CLI now responds to app commands. Use 'lightning --help' to see them.") click.echo(" ") Popen( From b41aee826e6a59059dfff6ad6328b57649bfc3f1 Mon Sep 17 00:00:00 2001 From: Luca Antiga Date: Tue, 13 Dec 2022 23:17:25 +0100 Subject: [PATCH 08/20] Update src/lightning_app/cli/commands/connection.py --- src/lightning_app/cli/commands/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lightning_app/cli/commands/connection.py b/src/lightning_app/cli/commands/connection.py index cfea2138a992c..cb92d2045cdea 100644 --- a/src/lightning_app/cli/commands/connection.py +++ b/src/lightning_app/cli/commands/connection.py @@ -118,7 +118,7 @@ def connect(app_name_or_id: str): shutil.copytree(matched_commands, commands) shutil.copy(matched_connected_file, connected_file) - click.echo("The lightning CLI now respond to app commands. Use 'lightning --help' to see them.") + click.echo("The lightning CLI now responds to app commands. Use 'lightning --help' to see them.") click.echo(" ") Popen( From c712054803892d85d15f79f034d6f3ab419d7818 Mon Sep 17 00:00:00 2001 From: Luca Antiga Date: Tue, 13 Dec 2022 23:17:33 +0100 Subject: [PATCH 09/20] Update src/lightning_app/cli/commands/connection.py --- src/lightning_app/cli/commands/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lightning_app/cli/commands/connection.py b/src/lightning_app/cli/commands/connection.py index cb92d2045cdea..962270b4e824d 100644 --- a/src/lightning_app/cli/commands/connection.py +++ b/src/lightning_app/cli/commands/connection.py @@ -142,7 +142,7 @@ def connect(app_name_or_id: str): project = _get_project(client) apps = client.lightningapp_instance_service_list_lightningapp_instances(project_id=project.project_id) click.echo( - "We didn't find a matching App. Here are the available Apps that could be " + "We didn't find a matching App. Here are the available Apps that you can" f"connected to {[app.name for app in apps.lightningapps]}." ) return From 9f58db1df849123fea687a0c3b454ac35aca436b Mon Sep 17 00:00:00 2001 From: Luca Antiga Date: Tue, 13 Dec 2022 23:17:41 +0100 Subject: [PATCH 10/20] Update src/lightning_app/cli/commands/connection.py --- src/lightning_app/cli/commands/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lightning_app/cli/commands/connection.py b/src/lightning_app/cli/commands/connection.py index 962270b4e824d..ace62919776cf 100644 --- a/src/lightning_app/cli/commands/connection.py +++ b/src/lightning_app/cli/commands/connection.py @@ -143,7 +143,7 @@ def connect(app_name_or_id: str): apps = client.lightningapp_instance_service_list_lightningapp_instances(project_id=project.project_id) click.echo( "We didn't find a matching App. Here are the available Apps that you can" - f"connected to {[app.name for app in apps.lightningapps]}." + f"connect to {[app.name for app in apps.lightningapps]}." ) return From 99ca0e1cbfcefe10c4c42ead2e329e9b049ad90d Mon Sep 17 00:00:00 2001 From: Luca Antiga Date: Tue, 13 Dec 2022 23:17:48 +0100 Subject: [PATCH 11/20] Update tests/tests_app/cli/test_connect.py --- tests/tests_app/cli/test_connect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests_app/cli/test_connect.py b/tests/tests_app/cli/test_connect.py index 6b9d8e97ff1d7..cae57a84a440e 100644 --- a/tests/tests_app/cli/test_connect.py +++ b/tests/tests_app/cli/test_connect.py @@ -154,7 +154,7 @@ def fn(msg): messages = [] connect("example") - assert "The lightning CLI now respond to app commands" in messages[0] + assert "The lightning CLI now responds to app commands" in messages[0] messages = [] disconnect() From d78f8c044234679682c987ea634214417cede01f Mon Sep 17 00:00:00 2001 From: Luca Antiga Date: Tue, 13 Dec 2022 23:18:02 +0100 Subject: [PATCH 12/20] Update src/lightning_app/cli/commands/connection.py --- src/lightning_app/cli/commands/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lightning_app/cli/commands/connection.py b/src/lightning_app/cli/commands/connection.py index ace62919776cf..c5ed2d6698a29 100644 --- a/src/lightning_app/cli/commands/connection.py +++ b/src/lightning_app/cli/commands/connection.py @@ -176,7 +176,7 @@ def connect(app_name_or_id: str): f.write(retriever.app_name + "\n") f.write(retriever.app_id + "\n") - click.echo("The lightning CLI now respond to app commands. Use 'lightning --help' to see them.") + click.echo("The lightning CLI now responds to app commands. Use 'lightning --help' to see them.") click.echo(" ") Popen( From 91a9d9d46c5882aa5e8b54ae914df570952b4821 Mon Sep 17 00:00:00 2001 From: Luca Antiga Date: Tue, 13 Dec 2022 23:32:05 +0100 Subject: [PATCH 13/20] Update tests/tests_app/cli/test_connect.py --- tests/tests_app/cli/test_connect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests_app/cli/test_connect.py b/tests/tests_app/cli/test_connect.py index cae57a84a440e..adbd55385815e 100644 --- a/tests/tests_app/cli/test_connect.py +++ b/tests/tests_app/cli/test_connect.py @@ -36,7 +36,7 @@ def monkeypatch_connection(monkeypatch, tmpdir, ppid): def test_connect_disconnect_local(tmpdir, monkeypatch): disconnect() - with pytest.raises(Exception, match="The commands weren't found. Is your app localhost running ?"): + with pytest.raises(Exception, match="Connection wasn't successful. Is your app localhost running ?"): connect("localhost") with open(os.path.join(os.path.dirname(__file__), "jsons/connect_1.json")) as f: From 6effceff34d5fd8108668386c90fe84a2fda9732 Mon Sep 17 00:00:00 2001 From: thomas Date: Tue, 13 Dec 2022 22:51:15 +0000 Subject: [PATCH 14/20] update --- tests/tests_examples_app/public/test_commands_and_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests_examples_app/public/test_commands_and_api.py b/tests/tests_examples_app/public/test_commands_and_api.py index a6a015d02a84a..290cf0d49b4cb 100644 --- a/tests/tests_examples_app/public/test_commands_and_api.py +++ b/tests/tests_examples_app/public/test_commands_and_api.py @@ -25,7 +25,7 @@ def test_commands_and_api_example_cloud() -> None: # 2: Connect to the App and send the first & second command with the client # Requires to be run within the same process. - cmd_1 = f"python -m lightning connect {app_id} -y" + cmd_1 = f"python -m lightning connect {app_id}" cmd_2 = "python -m lightning command with client --name=this" cmd_3 = "python -m lightning command without client --name=is" cmd_4 = "lightning disconnect" From e91f412592a5d95d837d26548a09b71763b0fa16 Mon Sep 17 00:00:00 2001 From: thomas Date: Tue, 13 Dec 2022 22:52:43 +0000 Subject: [PATCH 15/20] update --- src/lightning_app/CHANGELOG.md | 2 ++ src/lightning_app/cli/commands/connection.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/lightning_app/CHANGELOG.md b/src/lightning_app/CHANGELOG.md index fbddc4210a869..7ce0b8bee6f6c 100644 --- a/src/lightning_app/CHANGELOG.md +++ b/src/lightning_app/CHANGELOG.md @@ -10,6 +10,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Added `Lightning{Flow,Work}.lightningignores` attributes to programmatically ignore files before uploading to the cloud ([#15818](https://github.com/Lightning-AI/lightning/pull/15818)) +- Added a progres bar while connecting to an app through the CLI ([#16035](https://github.com/Lightning-AI/lightning/pull/16035)) + ### Changed diff --git a/src/lightning_app/cli/commands/connection.py b/src/lightning_app/cli/commands/connection.py index c5ed2d6698a29..3adf8394b6cb4 100644 --- a/src/lightning_app/cli/commands/connection.py +++ b/src/lightning_app/cli/commands/connection.py @@ -22,7 +22,7 @@ _PPID = os.getenv("LIGHTNING_CONNECT_PPID", str(psutil.Process(os.getpid()).ppid())) _LIGHTNING_CONNECTION = os.path.join(_HOME, ".lightning", "lightning_connection") _LIGHTNING_CONNECTION_FOLDER = os.path.join(_LIGHTNING_CONNECTION, _PPID) -_PROGRESS_TOTAL = 60 +_PROGRESS_TOTAL = 100 @click.argument("app_name_or_id", required=True) From 7aa5c61114d44d1d407cfb9ae4d0dea2224f1d40 Mon Sep 17 00:00:00 2001 From: Luca Antiga Date: Wed, 14 Dec 2022 11:33:59 +0100 Subject: [PATCH 16/20] Add docstring for connect --- src/lightning_app/cli/commands/connection.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/lightning_app/cli/commands/connection.py b/src/lightning_app/cli/commands/connection.py index 3adf8394b6cb4..45281297db59c 100644 --- a/src/lightning_app/cli/commands/connection.py +++ b/src/lightning_app/cli/commands/connection.py @@ -27,7 +27,25 @@ @click.argument("app_name_or_id", required=True) def connect(app_name_or_id: str): - """Connect to a Lightning App.""" + """Connect your local terminal to a lightning app running locally or in the cloud. + + After connecting, the lightning CLI will respond to commands exposed by the app. + + Example: + + # connect to an app named pizza-cooker-123 + lightning connect pizza-cooker-123 --yes + + # this will now show the commands exposed by pizza-cooker-123 + lightning --help + + # while connected, you can run the cook-pizza command exposed by pizza-cooker-123 + # BTW, this should arguably generate an exception :-) + lightning cook-pizza --flavor pineapple + + # once done, disconnect and go back to the standard lightning CLI commands + lightning disconnect + """ from lightning_app.utilities.commands.base import _download_command _clean_lightning_connection() From b10033935a62dff357df9e54aca3257910f1eb9c Mon Sep 17 00:00:00 2001 From: Luca Antiga Date: Wed, 14 Dec 2022 12:00:23 +0100 Subject: [PATCH 17/20] Fix line breaks --- src/lightning_app/cli/commands/connection.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/lightning_app/cli/commands/connection.py b/src/lightning_app/cli/commands/connection.py index 45281297db59c..5d928a9d9a691 100644 --- a/src/lightning_app/cli/commands/connection.py +++ b/src/lightning_app/cli/commands/connection.py @@ -27,22 +27,23 @@ @click.argument("app_name_or_id", required=True) def connect(app_name_or_id: str): - """Connect your local terminal to a lightning app running locally or in the cloud. + """Connect your local terminal to a running lightning app. After connecting, the lightning CLI will respond to commands exposed by the app. Example: + \b # connect to an app named pizza-cooker-123 lightning connect pizza-cooker-123 --yes - + \b # this will now show the commands exposed by pizza-cooker-123 lightning --help - - # while connected, you can run the cook-pizza command exposed by pizza-cooker-123 - # BTW, this should arguably generate an exception :-) + \b + # while connected, you can run the cook-pizza command exposed + # by pizza-cooker-123.BTW, this should arguably generate an exception :-) lightning cook-pizza --flavor pineapple - + \b # once done, disconnect and go back to the standard lightning CLI commands lightning disconnect """ From 2fdbb59564fd2240e40651e995003e8ec14409cb Mon Sep 17 00:00:00 2001 From: Luca Antiga Date: Wed, 14 Dec 2022 13:38:35 +0100 Subject: [PATCH 18/20] Remove thread and make progress bar iterate properly --- src/lightning_app/cli/commands/connection.py | 45 ++++++-------------- 1 file changed, 14 insertions(+), 31 deletions(-) diff --git a/src/lightning_app/cli/commands/connection.py b/src/lightning_app/cli/commands/connection.py index 5d928a9d9a691..d29ed5f440b95 100644 --- a/src/lightning_app/cli/commands/connection.py +++ b/src/lightning_app/cli/commands/connection.py @@ -3,14 +3,12 @@ import shutil import sys from subprocess import Popen -from threading import Event, Thread -from time import sleep, time from typing import List, Optional, Tuple import click import psutil from lightning_utilities.core.imports import package_available -from rich.progress import Progress, Task +from rich.progress import Progress from lightning_app.utilities.cli_helpers import _LightningAppOpenAPIRetriever from lightning_app.utilities.cloud import _get_project @@ -22,7 +20,6 @@ _PPID = os.getenv("LIGHTNING_CONNECT_PPID", str(psutil.Process(os.getpid()).ppid())) _LIGHTNING_CONNECTION = os.path.join(_HOME, ".lightning", "lightning_connection") _LIGHTNING_CONNECTION_FOLDER = os.path.join(_LIGHTNING_CONNECTION, _PPID) -_PROGRESS_TOTAL = 100 @click.argument("app_name_or_id", required=True) @@ -35,7 +32,7 @@ def connect(app_name_or_id: str): \b # connect to an app named pizza-cooker-123 - lightning connect pizza-cooker-123 --yes + lightning connect pizza-cooker-123 \b # this will now show the commands exposed by pizza-cooker-123 lightning --help @@ -74,10 +71,7 @@ def connect(app_name_or_id: str): elif app_name_or_id.startswith("localhost"): with Progress() as progress_bar: - event = Event() - connecting = progress_bar.add_task("[magenta]Setting things up for you...", total=_PROGRESS_TOTAL) - thread = Thread(target=update_progresss, args=[event, progress_bar, connecting], daemon=True) - thread.start() + connecting = progress_bar.add_task("[magenta]Setting things up for you...", total=1.0) if app_name_or_id != "localhost": raise Exception("You need to pass localhost to connect to the local Lightning App.") @@ -87,6 +81,10 @@ def connect(app_name_or_id: str): if retriever.api_commands is None: raise Exception(f"Connection wasn't successful. Is your app {app_name_or_id} running?") + increment = 1 / (1 + len(retriever.api_commands)) + + progress_bar.update(connecting, advance=increment) + commands_folder = os.path.join(_LIGHTNING_CONNECTION_FOLDER, "commands") if not os.path.exists(commands_folder): os.makedirs(commands_folder) @@ -112,8 +110,7 @@ def connect(app_name_or_id: str): with open(os.path.join(commands_folder, f"{command_name}.txt"), "w") as f: f.write(command_name) - event.set() - thread.join() + progress_bar.update(connecting, advance=increment) with open(connected_file, "w") as f: f.write(app_name_or_id + "\n") @@ -149,10 +146,7 @@ def connect(app_name_or_id: str): else: with Progress() as progress_bar: - event = Event() - connecting = progress_bar.add_task("[magenta]Setting things up for you...", total=_PROGRESS_TOTAL) - thread = Thread(target=update_progresss, args=[event, progress_bar, connecting], daemon=True) - thread.start() + connecting = progress_bar.add_task("[magenta]Setting things up for you...", total=1.0) retriever = _LightningAppOpenAPIRetriever(app_name_or_id) @@ -166,6 +160,10 @@ def connect(app_name_or_id: str): ) return + increment = 1 / (1 + len(retriever.api_commands)) + + progress_bar.update(connecting, advance=increment) + _install_missing_requirements(retriever) commands_folder = os.path.join(_LIGHTNING_CONNECTION_FOLDER, "commands") @@ -188,8 +186,7 @@ def connect(app_name_or_id: str): with open(os.path.join(commands_folder, f"{command_name}.txt"), "w") as f: f.write(command_name) - event.set() - thread.join() + progress_bar.update(connecting, advance=increment) with open(connected_file, "w") as f: f.write(retriever.app_name + "\n") @@ -375,17 +372,3 @@ def _scan_lightning_connections(app_name_or_id): return connection_path return None - - -def update_progresss(exit_event: Event, progress_bar: Progress, task: Task) -> None: - t0 = time() - while not exit_event.is_set(): - sleep(0.5) - progress_bar.update(task, advance=0.5) - - # Note: This is required to make progress feel more naturally progressing. - remaning = _PROGRESS_TOTAL - (time() - t0) - num_updates = 10 - for _ in range(num_updates): - progress_bar.update(task, advance=remaning / float(num_updates)) - sleep(0.2) From 37af6b82e23251fb13b70adf3b373df7874755f6 Mon Sep 17 00:00:00 2001 From: thomas Date: Wed, 14 Dec 2022 14:58:57 +0000 Subject: [PATCH 19/20] update --- src/lightning_app/cli/commands/connection.py | 16 ++++++++-------- src/lightning_app/utilities/frontend.py | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/lightning_app/cli/commands/connection.py b/src/lightning_app/cli/commands/connection.py index d29ed5f440b95..f5600947ab427 100644 --- a/src/lightning_app/cli/commands/connection.py +++ b/src/lightning_app/cli/commands/connection.py @@ -276,11 +276,11 @@ def _list_app_commands(echo: bool = True) -> List[str]: app_info = metadata[command_names[0]].get("app_info", None) - title, description, on_after_connect = "Lightning", None, None + title, description, on_connect_end = "Lightning", None, None if app_info: - title = app_info["title"] - description = app_info["description"] - on_after_connect = app_info["on_after_connect"] + title = app_info.get("title") + description = app_info.get("description") + on_connect_end = app_info.get("on_connect_end") if echo: click.echo(f"{title} App") @@ -296,10 +296,10 @@ def _list_app_commands(echo: bool = True) -> List[str]: for command_name in command_names: padding = (max_length + 1 - len(command_name)) * " " click.echo(f" {command_name}{padding}{metadata[command_name].get('description', '')}") - if "LIGHTNING_CONNECT_PPID" in os.environ and on_after_connect: - if on_after_connect.endswith("\n"): - on_after_connect = on_after_connect[:-2] - click.echo(on_after_connect) + if "LIGHTNING_CONNECT_PPID" in os.environ and on_connect_end: + if on_connect_end.endswith("\n"): + on_connect_end = on_connect_end[:-2] + click.echo(on_connect_end) return command_names diff --git a/src/lightning_app/utilities/frontend.py b/src/lightning_app/utilities/frontend.py index e581b24f8368d..470036436a63c 100644 --- a/src/lightning_app/utilities/frontend.py +++ b/src/lightning_app/utilities/frontend.py @@ -12,7 +12,7 @@ class AppInfo: image: Optional[str] = None # ensure the meta tags are correct or the UI might fail to load. meta_tags: Optional[List[str]] = None - on_after_connect: Optional[str] = None + on_connect_end: Optional[str] = None def update_index_file(ui_root: str, info: Optional[AppInfo] = None, root_path: str = "") -> None: From c03a202cf9dbba161a4c66bcd54a32e5838cb29a Mon Sep 17 00:00:00 2001 From: thomas Date: Wed, 14 Dec 2022 16:23:06 +0000 Subject: [PATCH 20/20] update --- examples/app_installation_commands/app.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/examples/app_installation_commands/app.py b/examples/app_installation_commands/app.py index 087d84b1335b2..f69df99ad9e82 100644 --- a/examples/app_installation_commands/app.py +++ b/examples/app_installation_commands/app.py @@ -13,9 +13,14 @@ def run(self): print("lmdb successfully installed") print("accessing a module in a Work or Flow body works!") - @property - def ready(self) -> bool: - return True + +class RootFlow(L.LightningFlow): + def __init__(self, work): + super().__init__() + self.work = work + + def run(self): + self.work.run() print(f"accessing an object in main code body works!: version={lmdb.version()}") @@ -24,4 +29,4 @@ def ready(self) -> bool: # run on a cloud machine compute = L.CloudCompute("cpu") worker = YourComponent(cloud_compute=compute) -app = L.LightningApp(worker) +app = L.LightningApp(RootFlow(worker))