Merge branch 'master' into ci/drop-1.9

Author: carmocca

.actions/setup_tools.py

@@ -232,8 +232,10 @@ def create_mirror_package(src_folder: str, lit_pkg_mapping: dict) -> None:
     ...     {"pytorch": "pytorch_lightning", "app": "lightning_app", "lite": "lightning_lite"}
     ... )
     """
-    for lit_name, pkg_name in lit_pkg_mapping.items():
-        copy_adjusted_modules(src_folder, pkg_name, lit_name, lit_pkg_mapping)
+    mapping = lit_pkg_mapping.copy()
+    mapping.pop("lightning", None)  # pop this key to avoid replacing `lightning` to `lightning.lightning`
+    for lit_name, pkg_name in mapping.items():
+        copy_adjusted_modules(src_folder, pkg_name, lit_name, mapping)
 
 
 def _download_frontend(pkg_path: str):
@@ -272,19 +274,3 @@ def _load_aggregate_requirements(req_dir: str = "requirements", freeze_requireme
     requires = list(chain(*requires))
     with open(os.path.join(req_dir, "base.txt"), "w") as fp:
         fp.writelines([ln + os.linesep for ln in requires])
-
-
-def set_actual_version_from_src(req_path: str, src_root: str, pkg_name: str) -> None:
-    """Setting actual version from source code for a given package."""
-    with open(req_path, encoding="utf-8") as fo:
-        lines = fo.readlines()
-    ver = parse_version_from_file(os.path.join(src_root, pkg_name.replace("-", "_")))
-    for i, ln in enumerate(lines):
-        reqs = list(parse_requirements([ln]))
-        if not reqs:
-            continue
-        if reqs[0].name == pkg_name:
-            lines[i] = f"{pkg_name}=={ver}{os.linesep}"
-
-    with open(req_path, "w", encoding="utf-8") as fw:
-        fw.writelines(lines)

.azure/app-cloud-e2e.yml

@@ -75,7 +75,7 @@ jobs:
           name: "commands_and_api"
         # 'App: quick_start':  # TODO: Failed during installation
         #   name: "quick_start"
-    timeoutInMinutes: "30"
+    timeoutInMinutes: "20"
     cancelTimeoutInMinutes: "2"
     # values: https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#workspace
     workspace:

.azure/gpu-benchmark.yml

@@ -51,16 +51,6 @@ jobs:
         echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/cu${cuda_ver}/torch_stable.html"
       displayName: 'set env. vars'
 
-    - bash: |
-        pip install -e . --find-links ${TORCH_URL}
-        git checkout -- setup.py MANIFEST.in
-      env:
-        PACKAGE_NAME: lite
-        FREEZE_REQUIREMENTS: 1
-      # Lite shall have pin version, so to keep development aligned we need to install lite from source not from pypi
-      # also installing from source not from on-the-fly created package tp prevent accidental interaction with cache
-      displayName: 'Install Lite pkg'
-
     - bash: |
         pip install -e .[strategies] --find-links ${TORCH_URL}
         pip list

.azure/gpu-tests-pytorch.yml

@@ -93,16 +93,6 @@ jobs:
         python ./requirements/pytorch/adjust-versions.py requirements/pytorch/examples.txt ${PYTORCH_VERSION}
       displayName: 'Adjust dependencies'
 
-    - bash: |
-        pip install -e . --find-links ${TORCH_URL}
-        git checkout -- setup.py MANIFEST.in
-      env:
-        PACKAGE_NAME: "lite"
-        FREEZE_REQUIREMENTS: "1"
-      # Lite shall have pin version, so to keep development aligned we need to install lite from source not from pypi
-      # also installing from source not from on-the-fly created package to prevent accidental interaction with cache
-      displayName: 'Install Lite pkg'
-
     - bash: pip install -e .[strategies] -r requirements/pytorch/devel.txt -r requirements/pytorch/examples.txt --find-links ${TORCH_URL}
       env:
         PACKAGE_NAME: "pytorch"

.azure/hpu-tests.yml

@@ -58,16 +58,6 @@ jobs:
         sudo pip install pip -U
       displayName: 'Instance HW info'
 
-    - bash: |
-        pip install -e .
-        git checkout -- setup.py MANIFEST.in
-      env:
-        PACKAGE_NAME: "lite"
-        FREEZE_REQUIREMENTS: "1"
-      # Lite shall have pin version, so to keep development aligned we need to install lite from source not from pypi
-      # also installing from source not from on-the-fly created package to prevent accidental interaction with cache
-      displayName: 'Install Lite pkg'
-
     - bash: |
         set -e
         pip --version

.azure/ipu-tests.yml

@@ -63,16 +63,6 @@ jobs:
         done
       displayName: "Reset IPU devices"
 
-    - bash: |
-        pip install -e .
-        git checkout -- setup.py MANIFEST.in
-      env:
-        PACKAGE_NAME: "lite"
-        FREEZE_REQUIREMENTS: "1"
-      # Lite shall have pin version, so to keep development aligned we need to install lite from source not from pypi
-      # also installing from source not from on-the-fly created package tp prevent accidental interaction with cache
-      displayName: 'Install Lite pkg'
-
     - bash: |
         python ./requirements/pytorch/adjust-versions.py requirements/pytorch/extra.txt
         python ./requirements/pytorch/adjust-versions.py requirements/pytorch/examples.txt

.github/actions/pkg-check/action.yml

@@ -39,13 +39,13 @@ runs:
     shell: bash
 
   - name: Unzip packages
-    if: ${{ inputs.pkg-name != '' }}
+    if: ${{ inputs.pkg-name != 'lightning' }}
     working-directory: dist
     run: for file in `ls *.gz`; do tar -xzf $file; done
     shell: bash
 
   - name: Check single pkg/folder
-    if: ${{ inputs.pkg-name != '' }}
+    if: ${{ inputs.pkg-name != 'lightning' }}
     working-directory: dist
     run: |
       import os, glob, pathlib, shutil

.github/actions/pkg-install/action.yml

@@ -1,42 +1,35 @@
-name: Install and check package
-description: installing and validationg the package
+name: Install and validate the package
+description: Install and validate the package
 
 inputs:
   pkg-name:
-    description: package name for import
-    required: false
-    default: ""
+    description: Package name to import
+    required: true
   pip-flags:
-    description: additional pil install flags
+    description: Additional pip install flags
     required: false
     default: ""
 
 runs:
   using: "composite"
   steps:
-  - name: Determine package name
-    if: ${{ inputs.pkg-import == '' }}
-    working-directory: ./dist
-    run: python -c "import glob ; ls = glob.glob('*.tar.gz') ; name = '_'.join(ls[0].split('-')[:-1]) ; print(f'PKG_NAME={name}')" >> $GITHUB_ENV
-    shell: bash
-
-  - name: Pass package name
-    if: ${{ inputs.pkg-import != '' }}
-    run: echo "PKG_NAME=${{ inputs.pkg-name }}" >> $GITHUB_ENV
+  - name: Choose package import
+    run: |
+      python -c "print('PKG_IMPORT=' + {'app': 'lightning_app', 'lite': 'lightning_lite', 'pytorch': 'pytorch_lightning', 'lightning': 'lightning'}['${{matrix.pkg-name}}'])" >> $GITHUB_ENV
     shell: bash
 
   - name: Install package - archive
     working-directory: ./dist
     run: |
       pip install *.tar.gz ${{ inputs.pip-flags }}
       pip list | grep lightning
-      python -c "import ${PKG_NAME} ; print(${PKG_NAME}.__version__)"
+      python -c "import ${{ env.PKG_IMPORT }}; print(${{ env.PKG_IMPORT }}.__version__)"
     shell: bash
 
   - name: Install package - wheel
     working-directory: ./dist
     run: |
       pip install *.whl ${{ inputs.pip-flags }}
       pip list | grep lightning
-      python -c "import ${PKG_NAME} ; print(${PKG_NAME}.__version__)"
+      python -c "import ${{ env.PKG_IMPORT }}; print(${{ env.PKG_IMPORT }}.__version__)"
     shell: bash

.github/checkgroup.yml

@@ -48,7 +48,6 @@ subprojects:
       - "pl-cpu (windows-2022, lightning, 3.7, 1.10, oldest)"
       - "make-doctest (pytorch)"
       - "make-html (pytorch)"
-      - "mypy"
       - "pytorch-lightning (GPUs)"
       - "pytorch-lightning (HPUs)"
       - "pytorch-lightning (IPUs)"
@@ -58,7 +57,7 @@ subprojects:
       # TODO: since this job has intermittent availability, it cannot be required or it will block all PL PRs from forks
       #- "test-on-tpus"
 
-  - id: "pytorch_lightning: CPU"
+  - id: "pytorch_lightning: CPU workflow"
     paths:
       - ".github/workflows/ci-pytorch-tests.yml"
     checks:
@@ -83,7 +82,7 @@ subprojects:
       - "pl-cpu (windows-2022, lightning, 3.10, 1.13)"
       - "pl-cpu (windows-2022, lightning, 3.7, 1.10, oldest)"
 
-  - id: "pytorch_lightning: Slow"
+  - id: "pytorch_lightning: Slow workflow"
     paths:
       - ".github/workflows/ci-pytorch-tests-slow.yml"
     checks:
@@ -110,7 +109,7 @@ subprojects:
     checks:
       - "pytorch-lightning (IPUs)"
 
-  - id: "pytorch-lightning: TPU"
+  - id: "pytorch-lightning: TPU workflow"
     paths:
       - ".github/workflows/tpu-tests.yml"
     checks:
@@ -176,7 +175,6 @@ subprojects:
       - "lite-cpu (ubuntu-20.04, lightning, 3.8, 1.12)"
       - "lite-cpu (windows-2022, lightning, 3.8, 1.12)"
       - "lightning-lite (GPUs)"
-      - "mypy"
       # Lite also requires PL checks as it depends on Lite
       - "pl-cpu (macOS-11, pytorch, 3.8, 1.11)"
       - "pl-cpu (macOS-11, pytorch, 3.9, 1.12)"
@@ -244,8 +242,7 @@ subprojects:
       - "requirements/app/**"
       - "src/lightning_app/**"
       - "tests/tests_app/**"
-      - "examples/app_*"  # some tests_app tests call examples files
-      - ".github/workflows/ci-app-tests.yml"
+      - "examples/app_*/**"  # some tests_app tests call examples files
       - "setup.py"
       - ".actions/**"
     checks:
@@ -262,13 +259,26 @@ subprojects:
       - "app-pytest (windows-2022, app, 3.8, oldest)"
       - "app-pytest (windows-2022, lightning, 3.8, latest)"
 
+  - id: "lightning_app: Tests workflow"
+    paths:
+      - ".github/workflows/ci-app-tests.yml"
+    checks:
+      - "app-pytest (macOS-11, app, 3.8, latest)"
+      - "app-pytest (macOS-11, app, 3.8, oldest)"
+      - "app-pytest (macOS-11, lightning, 3.9, latest)"
+      - "app-pytest (ubuntu-20.04, app, 3.8, latest)"
+      - "app-pytest (ubuntu-20.04, app, 3.8, oldest)"
+      - "app-pytest (ubuntu-20.04, lightning, 3.9, latest)"
+      - "app-pytest (windows-2022, app, 3.8, latest)"
+      - "app-pytest (windows-2022, app, 3.8, oldest)"
+      - "app-pytest (windows-2022, lightning, 3.8, latest)"
+
   - id: "lightning_app: Examples"
     paths:
       - "requirements/app/**"
       - "src/lightning_app/**"
       - "tests/tests_app_examples/**"
-      - "examples/app_*"
-      - ".github/workflows/ci-app-examples.yml"
+      - "examples/app_*/**"
       - "setup.py"
       - ".actions/**"
     checks:
@@ -282,6 +292,20 @@ subprojects:
       - "app-examples (windows-2022, app, 3.9, oldest)"
       - "app-examples (windows-2022, lightning, 3.9, latest)"
 
+  - id: "lightning_app: Examples workflow"
+    paths:
+      - ".github/workflows/ci-app-examples.yml"
+    checks:
+      - "app-examples (macOS-11, app, 3.9, latest)"
+      - "app-examples (macOS-11, app, 3.9, oldest)"
+      - "app-examples (macOS-11, lightning, 3.9, latest)"
+      - "app-examples (ubuntu-20.04, app, 3.9, latest)"
+      - "app-examples (ubuntu-20.04, app, 3.9, oldest)"
+      - "app-examples (ubuntu-20.04, lightning, 3.9, latest)"
+      - "app-examples (windows-2022, app, 3.9, latest)"
+      - "app-examples (windows-2022, app, 3.9, oldest)"
+      - "app-examples (windows-2022, lightning, 3.9, latest)"
+
   - id: "lightning_app: Azure"
     paths:
       - ".azure/app-cloud-e2e.yml"
@@ -303,7 +327,10 @@ subprojects:
   - id: "mypy"
     paths:
       - ".github/workflows/code-checks.yml"
+      - "requirements/**"
+      - "src/**"
       - "pyproject.toml"  # includes mypy config
+      - ".actions/**"
     checks:
       - "mypy"
 
@@ -321,21 +348,21 @@ subprojects:
       - "install-pkg (ubuntu-22.04, lite, 3.10)"
       - "install-pkg (ubuntu-22.04, pytorch, 3.7)"
       - "install-pkg (ubuntu-22.04, pytorch, 3.10)"
-      - "install-pkg (ubuntu-22.04, 3.7)"
-      - "install-pkg (ubuntu-22.04, 3.10)"
+      - "install-pkg (ubuntu-22.04, lightning, 3.7)"
+      - "install-pkg (ubuntu-22.04, lightning, 3.10)"
       - "install-pkg (macOS-12, app, 3.7)"
       - "install-pkg (macOS-12, app, 3.10)"
       - "install-pkg (macOS-12, lite, 3.7)"
       - "install-pkg (macOS-12, lite, 3.10)"
       - "install-pkg (macOS-12, pytorch, 3.7)"
       - "install-pkg (macOS-12, pytorch, 3.10)"
-      - "install-pkg (macOS-12, 3.7)"
-      - "install-pkg (macOS-12, 3.10)"
+      - "install-pkg (macOS-12, lightning, 3.7)"
+      - "install-pkg (macOS-12, lightning, 3.10)"
       - "install-pkg (windows-2022, app, 3.7)"
       - "install-pkg (windows-2022, app, 3.10)"
       - "install-pkg (windows-2022, lite, 3.7)"
       - "install-pkg (windows-2022, lite, 3.10)"
       - "install-pkg (windows-2022, pytorch, 3.7)"
       - "install-pkg (windows-2022, pytorch, 3.10)"
-      - "install-pkg (windows-2022, 3.7)"
-      - "install-pkg (windows-2022, 3.10)"
+      - "install-pkg (windows-2022, lightning, 3.7)"
+      - "install-pkg (windows-2022, lightning, 3.10)"

.github/workflows/ci-app-examples.yml

@@ -6,13 +6,14 @@ on:
     branches: [master, "release/*"]
   pull_request:
     branches: [master, "release/*"]
-    types: [opened, reopened, ready_for_review, synchronize]  # add `ready_for_review` since draft is skipped
+    types: [opened, reopened, ready_for_review, synchronize]  # added `ready_for_review` since draft is skipped
     paths:
       - ".github/workflows/ci-app-examples.yml"
       - "src/lightning_app/**"
       - "tests/tests_app_examples/**"
       - "examples/app_*"
-      - "requirements/app/**"
+      - "requirements/app/*"
+      - "!requirements/app/docs.txt"
       - "setup.py"
       - ".actions/**"
 
@@ -98,8 +99,7 @@ jobs:
       - name: Adjust examples
         if: ${{ matrix.pkg-name != 'lightning' }}
         run: |
-          python .actions/assistant.py copy_replace_imports --source_dir="./examples" --source_import="lightning.app" --target_import="lightning_app"
-          python .actions/assistant.py copy_replace_imports --source_dir="./examples" --source_import="lightning" --target_import="lightning_app"
+          python .actions/assistant.py copy_replace_imports --source_dir="./examples" --source_import="lightning.app,lightning" --target_import="lightning_app,lightning_app"
 
       - name: Switch coverage scope
         run: python -c "print('COVERAGE_SCOPE=' + str('lightning' if '${{matrix.pkg-name}}' == 'lightning' else 'lightning_app'))" >> $GITHUB_ENV