LAI: creating mirror package

.azure/app-cloud-e2e.yml

@@ -100,6 +100,7 @@ jobs:
         path: $(pip_cache_dir)
       displayName: Cache pip
 
+    # TODO: we are testing it as `lightning`, so add also version for `lightning_app`
     - bash: git restore . && python -m pip install -e . -r requirements/app/test.txt -r requirements/app/ui.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html
       displayName: 'Install Lightning & dependencies'
 
@@ -125,6 +126,16 @@ jobs:
       condition: eq(variables['name'], 'quick_start')
       displayName: 'Install Quick Start'
 
+    - bash: python -m pip install -q -r .actions/requirements.txt
+      displayName: 'Install assistant dependencies'
+
+    # Fix imports to use `lightning` instead of `lightning_app` since we install lightning only ATM
+    # TODO: fixme when installing `lightning_app` as well
+    - bash: |
+        python .actions/assistant.py copy_replace_imports --source_dir="./examples" --source_import="lightning_app" --target_import="lightning.app"
+        python .actions/assistant.py copy_replace_imports --source_dir="./tests" --source_import="lightning_app" --target_import="lightning.app"
+      displayName: 'Adjust examples'
+
     - bash: |
         pip --version
         pip list
@@ -144,12 +155,12 @@ jobs:
         TEST_APP_NAME: $(name)
         HAR_LOCATION: './artifacts/hars'
         SLOW_MO: '50'
-        # LAI_USER: $(LAI_USER)
-        # LAI_PASS: $(LAI_PASS)
-        LIGHTNING_USER_ID: $(LIGHTNING_USER_ID_PROD)
-        LIGHTNING_API_KEY: $(LIGHTNING_API_KEY_PROD)
+        LAI_USER: $(LAI_USER)  # for STAGING
+        LAI_PASS: $(LAI_PASS)  # for STAGING
+        LIGHTNING_USER_ID: $(LIGHTNING_USER_ID_STAGING)
+        LIGHTNING_API_KEY: $(LIGHTNING_API_KEY_STAGING)
         LIGHTNING_USERNAME: $(LIGHTNING_USERNAME)
-        LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL_PROD)
+        LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL_STAGING)
         LIGHTNING_DEBUG: '1'
       displayName: 'Run the tests'
 
@@ -162,12 +173,12 @@ jobs:
         time python -c "from lightning.app import testing; testing.delete_cloud_lightning_apps()"
       condition: always()
       env:
-        # LAI_USER: $(LAI_USER)
-        # LAI_PASS: $(LAI_PASS)
-        LIGHTNING_USER_ID: $(LIGHTNING_USER_ID_PROD)
-        LIGHTNING_API_KEY: $(LIGHTNING_API_KEY_PROD)
+        LAI_USER: $(LAI_USER)  # for STAGING
+        LAI_PASS: $(LAI_PASS)  # for STAGING
+        LIGHTNING_USER_ID: $(LIGHTNING_USER_ID_STAGING)
+        LIGHTNING_API_KEY: $(LIGHTNING_API_KEY_STAGING)
         LIGHTNING_USERNAME: $(LIGHTNING_USERNAME)
-        LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL_PROD)
+        LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL_STAGING)
         PR_NUMBER: $(local_id)
         TEST_APP_NAME: $(name)
         # GRID_USER_ID: $(LIGHTNING_USER_ID)  # TODO: clarify the meaning

.github/actions/pkg-check/action.yml

@@ -5,6 +5,7 @@ inputs:
   pkg-name:
     description: package name inside lightning.*
     required: true
+    default: ""
   nb-dirs:
     description: nb of packages in the wrap/distribution
     required: false
@@ -21,10 +22,7 @@ runs:
   - name: Source check
     env:
       PACKAGE_NAME: ${{ inputs.pkg-name }}
-    run: |
-      python setup.py check --metadata --strict
-      # TODO: fix reimporting (F811) which is aftefact after rplacing var assigne with import in meta package
-      flake8 src/lightning/ --ignore E402,F401,E501,W391,E303,F811
+    run: python setup.py check --metadata --strict
     shell: bash
 
   - name: Create package
@@ -40,10 +38,6 @@ runs:
       # python setup.py clean
     shell: bash
 
-  - name: copy/export pkg
-    run: cp dist/* pypi/
-    shell: bash
-
   - name: Unzip packages
     if: ${{ inputs.pkg-name != '' }}
     working-directory: dist

.github/actions/pkg-install/action.yml

@@ -25,15 +25,15 @@ runs:
     run: echo "PKG_NAME=${{ inputs.pkg-name }}" >> $GITHUB_ENV
     shell: bash
 
-  - name: Install | Uninstall package - archive
+  - name: Install package - archive
     working-directory: ./dist
     run: |
       pip install *.tar.gz ${{ inputs.pip-flags }}
       pip list | grep lightning
       python -c "import ${PKG_NAME} ; print(${PKG_NAME}.__version__)"
     shell: bash
 
-  - name: Install | Uninstall package - wheel
+  - name: Install package - wheel
     working-directory: ./dist
     run: |
       pip install *.whl ${{ inputs.pip-flags }}

.github/checkgroup.yml

@@ -30,16 +30,23 @@ subprojects:
       - "pl-cpu (macOS-11, pytorch, 3.10, 1.12)"
       - "pl-cpu (macOS-11, pytorch, 3.7, 1.9, oldest)"
       - "pl-cpu (macOS-11, pytorch, 3.10, 1.13, pre)"
-      - "pl-cpu (ubuntu-20.04, pytorch, 3.8, 1.11)"
+      - "pl-cpu (ubuntu-20.04, pytorch, 3.8, 1.10)"
+      - "pl-cpu (ubuntu-20.04, pytorch, 3.9, 1.11)"
+      - "pl-cpu (ubuntu-20.04, pytorch, 3.10, 1.11)"
       - "pl-cpu (ubuntu-20.04, pytorch, 3.10, 1.12)"
       - "pl-cpu (ubuntu-20.04, pytorch, 3.7, 1.9, oldest)"
       - "pl-cpu (ubuntu-20.04, pytorch, 3.9, 1.13, pre)"
-      - "pl-cpu (windows-2022, pytorch, 3.8, 1.9)"
-      - "pl-cpu (windows-2022, pytorch, 3.9, 1.10)"
+      - "pl-cpu (windows-2022, pytorch, 3.9, 1.11)"
       - "pl-cpu (windows-2022, pytorch, 3.10, 1.11)"
       - "pl-cpu (windows-2022, pytorch, 3.10, 1.12)"
       - "pl-cpu (windows-2022, pytorch, 3.7, 1.9, oldest)"
       - "pl-cpu (windows-2022, pytorch, 3.8, 1.13, pre)"
+      - "pl-cpu (macOS-11, lightning, 3.10, 1.12)"
+      #- "pl-cpu (macOS-11, lightning, 3.7, 1.9, oldest)"
+      - "pl-cpu (ubuntu-20.04, lightning, 3.10, 1.12)"
+      #- "pl-cpu (ubuntu-20.04, lightning, 3.7, 1.9, oldest)"
+      - "pl-cpu (windows-2022, lightning, 3.10, 1.12)"
+      #- "pl-cpu (windows-2022, lightning, 3.7, 1.9, oldest)"
       - "make-doctest (pytorch)"
       - "make-html (pytorch)"
       - "mypy"
@@ -56,21 +63,28 @@ subprojects:
     paths:
       - ".github/workflows/ci-pytorch-tests.yml"
     checks:
-      - "pl-cpu (macOS-11, pytorch, 3.9, 1.11)"
       - "pl-cpu (macOS-11, pytorch, 3.8, 1.10)"
+      - "pl-cpu (macOS-11, pytorch, 3.9, 1.11)"
       - "pl-cpu (macOS-11, pytorch, 3.10, 1.12)"
       - "pl-cpu (macOS-11, pytorch, 3.7, 1.9, oldest)"
       - "pl-cpu (macOS-11, pytorch, 3.10, 1.13, pre)"
-      - "pl-cpu (ubuntu-20.04, pytorch, 3.8, 1.11)"
+      - "pl-cpu (ubuntu-20.04, pytorch, 3.8, 1.10)"
+      - "pl-cpu (ubuntu-20.04, pytorch, 3.9, 1.11)"
+      - "pl-cpu (ubuntu-20.04, pytorch, 3.10, 1.11)"
       - "pl-cpu (ubuntu-20.04, pytorch, 3.10, 1.12)"
       - "pl-cpu (ubuntu-20.04, pytorch, 3.7, 1.9, oldest)"
       - "pl-cpu (ubuntu-20.04, pytorch, 3.9, 1.13, pre)"
-      - "pl-cpu (windows-2022, pytorch, 3.8, 1.9)"
-      - "pl-cpu (windows-2022, pytorch, 3.9, 1.10)"
+      - "pl-cpu (windows-2022, pytorch, 3.9, 1.11)"
       - "pl-cpu (windows-2022, pytorch, 3.10, 1.11)"
       - "pl-cpu (windows-2022, pytorch, 3.10, 1.12)"
       - "pl-cpu (windows-2022, pytorch, 3.7, 1.9, oldest)"
       - "pl-cpu (windows-2022, pytorch, 3.8, 1.13, pre)"
+      - "pl-cpu (macOS-11, lightning, 3.10, 1.12)"
+      #- "pl-cpu (macOS-11, lightning, 3.7, 1.9, oldest)"
+      - "pl-cpu (ubuntu-20.04, lightning, 3.10, 1.12)"
+      #- "pl-cpu (ubuntu-20.04, lightning, 3.7, 1.9, oldest)"
+      - "pl-cpu (windows-2022, lightning, 3.10, 1.12)"
+      #- "pl-cpu (windows-2022, lightning, 3.7, 1.9, oldest)"
 
   - id: "pytorch_lightning: Slow"
     paths:
@@ -165,24 +179,34 @@ subprojects:
       - "lite-cpu (windows-2022, lite, 3.10, 1.12)"
       - "lite-cpu (windows-2022, lite, 3.7, 1.9, oldest)"
       - "lite-cpu (windows-2022, lite, 3.8, 1.13, pre)"
+      - "lite-cpu (macOS-11, lightning, 3.8, 1.12)"
+      - "lite-cpu (ubuntu-20.04, lightning, 3.8, 1.12)"
+      - "lite-cpu (windows-2022, lightning, 3.8, 1.12)"
       - "lightning-lite (GPUs)"
       - "mypy"
       # Lite also requires PL checks as it depends on Lite
-      - "pl-cpu (macOS-11, pytorch, 3.9, 1.11)"
       - "pl-cpu (macOS-11, pytorch, 3.8, 1.10)"
+      - "pl-cpu (macOS-11, pytorch, 3.9, 1.11)"
       - "pl-cpu (macOS-11, pytorch, 3.10, 1.12)"
       - "pl-cpu (macOS-11, pytorch, 3.7, 1.9, oldest)"
       - "pl-cpu (macOS-11, pytorch, 3.10, 1.13, pre)"
-      - "pl-cpu (ubuntu-20.04, pytorch, 3.8, 1.11)"
+      - "pl-cpu (ubuntu-20.04, pytorch, 3.8, 1.10)"
+      - "pl-cpu (ubuntu-20.04, pytorch, 3.9, 1.11)"
+      - "pl-cpu (ubuntu-20.04, pytorch, 3.10, 1.11)"
       - "pl-cpu (ubuntu-20.04, pytorch, 3.10, 1.12)"
       - "pl-cpu (ubuntu-20.04, pytorch, 3.7, 1.9, oldest)"
       - "pl-cpu (ubuntu-20.04, pytorch, 3.9, 1.13, pre)"
-      - "pl-cpu (windows-2022, pytorch, 3.8, 1.9)"
-      - "pl-cpu (windows-2022, pytorch, 3.9, 1.10)"
+      - "pl-cpu (windows-2022, pytorch, 3.9, 1.11)"
       - "pl-cpu (windows-2022, pytorch, 3.10, 1.11)"
       - "pl-cpu (windows-2022, pytorch, 3.10, 1.12)"
       - "pl-cpu (windows-2022, pytorch, 3.7, 1.9, oldest)"
       - "pl-cpu (windows-2022, pytorch, 3.8, 1.13, pre)"
+      - "pl-cpu (macOS-11, lightning, 3.10, 1.12)"
+      #- "pl-cpu (macOS-11, lightning, 3.7, 1.9, oldest)"
+      - "pl-cpu (ubuntu-20.04, lightning, 3.10, 1.12)"
+      #- "pl-cpu (ubuntu-20.04, lightning, 3.7, 1.9, oldest)"
+      - "pl-cpu (windows-2022, lightning, 3.10, 1.12)"
+      #- "pl-cpu (windows-2022, lightning, 3.7, 1.9, oldest)"
       - "make-doctest (pytorch)"
       - "make-html (pytorch)"
       - "pytorch-lightning (GPUs)"
@@ -210,6 +234,9 @@ subprojects:
       - "lite-cpu (windows-2022, lite, 3.10, 1.12)"
       - "lite-cpu (windows-2022, lite, 3.7, 1.9, oldest)"
       - "lite-cpu (windows-2022, lite, 3.8, 1.13, pre)"
+      - "lite-cpu (macOS-11, lightning, 3.8, 1.12)"
+      - "lite-cpu (ubuntu-20.04, lightning, 3.8, 1.12)"
+      - "lite-cpu (windows-2022, lightning, 3.8, 1.12)"
       - "lightning-lite (GPUs)"
 
   - id: "lightning_lite: Azure GPU"
@@ -235,18 +262,24 @@ subprojects:
       - "App.cloud-e2e"
       - "make-doctest (app)"
       - "make-html (app)"
-      - "app-examples (macOS-11, app, 3.8, latest)"
-      - "app-examples (macOS-11, app, 3.8, oldest)"
-      - "app-examples (ubuntu-20.04, app, 3.8, latest)"
-      - "app-examples (ubuntu-20.04, app, 3.8, oldest)"
-      - "app-examples (windows-2022, app, 3.8, latest)"
-      - "app-examples (windows-2022, app, 3.8, oldest)"
+      - "app-examples (macOS-11, app, 3.9, latest)"
+      - "app-examples (macOS-11, app, 3.9, oldest)"
+      - "app-examples (macOS-11, lightning, 3.9, latest)"
+      - "app-examples (ubuntu-20.04, app, 3.9, latest)"
+      - "app-examples (ubuntu-20.04, app, 3.9, oldest)"
+      - "app-examples (ubuntu-20.04, lightning, 3.9, latest)"
+      - "app-examples (windows-2022, app, 3.9, latest)"
+      - "app-examples (windows-2022, app, 3.9, oldest)"
+      - "app-examples (windows-2022, lightning, 3.9, latest)"
       - "app-pytest (macOS-11, app, 3.8, latest)"
       - "app-pytest (macOS-11, app, 3.8, oldest)"
+      - "app-pytest (macOS-11, lightning, 3.9, latest)"
       - "app-pytest (ubuntu-20.04, app, 3.8, latest)"
       - "app-pytest (ubuntu-20.04, app, 3.8, oldest)"
+      - "app-pytest (ubuntu-20.04, lightning, 3.9, latest)"
       - "app-pytest (windows-2022, app, 3.8, latest)"
       - "app-pytest (windows-2022, app, 3.8, oldest)"
+      - "app-pytest (windows-2022, lightning, 3.9, latest)"
 
   - id: "lightning_app: Azure"
     paths:
@@ -287,15 +320,21 @@ subprojects:
       - "install-pkg (ubuntu-22.04, lite, 3.10)"
       - "install-pkg (ubuntu-22.04, pytorch, 3.7)"
       - "install-pkg (ubuntu-22.04, pytorch, 3.10)"
+      - "install-pkg (ubuntu-22.04, 3.7)"
+      - "install-pkg (ubuntu-22.04, 3.10)"
       - "install-pkg (macOS-12, app, 3.7)"
       - "install-pkg (macOS-12, app, 3.10)"
       - "install-pkg (macOS-12, lite, 3.7)"
       - "install-pkg (macOS-12, lite, 3.10)"
       - "install-pkg (macOS-12, pytorch, 3.7)"
       - "install-pkg (macOS-12, pytorch, 3.10)"
+      - "install-pkg (macOS-12, 3.7)"
+      - "install-pkg (macOS-12, 3.10)"
       - "install-pkg (windows-2022, app, 3.7)"
       - "install-pkg (windows-2022, app, 3.10)"
       - "install-pkg (windows-2022, lite, 3.7)"
       - "install-pkg (windows-2022, lite, 3.10)"
       - "install-pkg (windows-2022, pytorch, 3.7)"
       - "install-pkg (windows-2022, pytorch, 3.10)"
+      - "install-pkg (windows-2022, 3.7)"
+      - "install-pkg (windows-2022, 3.10)"

.github/workflows/ci-app-examples.yml

@@ -32,12 +32,17 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-20.04, macOS-11, windows-2022]
-        pkg-name: ["app"]  # TODO: add "LAI" # "LAI" installs the monolithic package
-        python-version: [3.8]
+        pkg-name: ["app"]
+        python-version: ["3.9"]
         requires: ["oldest", "latest"]
+        include:
+          # "lightning" installs the monolithic package
+          - {os: "macOS-11", pkg-name: "lightning", python-version: "3.9", requires: "latest"}
+          - {os: "ubuntu-20.04", pkg-name: "lightning", python-version: "3.9", requires: "latest"}
+          - {os: "windows-2022", pkg-name: "lightning", python-version: "3.9", requires: "latest"}
 
     # Timeout: https://stackoverflow.com/a/59076067/4521646
-    timeout-minutes: 10
+    timeout-minutes: 15
 
     steps:
       - uses: actions/checkout@v3
@@ -89,11 +94,17 @@ jobs:
         run: pip install -e .
 
       - name: Adjust tests
-        if: ${{ matrix.pkg-name == 'LAI' }}
+        if: ${{ matrix.pkg-name == 'lightning' }}
         run: python .actions/assistant.py copy_replace_imports --source_dir="./tests" --source_import="lightning_app" --target_import="lightning.app"
 
-      - name: switch coverage scope
-        run: python -c "print('COVERAGE_SCOPE=' + str('lightning' if '${{matrix.pkg-name}}' == 'LAI' else 'lightning_app'))" >> $GITHUB_ENV
+      - name: Adjust examples
+        if: ${{ matrix.pkg-name != 'lightning' }}
+        run: |
+          python .actions/assistant.py copy_replace_imports --source_dir="./examples" --source_import="lightning.app" --target_import="lightning_app"
+          python .actions/assistant.py copy_replace_imports --source_dir="./examples" --source_import="lightning" --target_import="lightning_app"
+
+      - name: Switch coverage scope
+        run: python -c "print('COVERAGE_SCOPE=' + str('lightning' if '${{matrix.pkg-name}}' == 'lightning' else 'lightning_app'))" >> $GITHUB_ENV
 
       - name: Tests
         working-directory: ./tests

.github/workflows/ci-app-tests.yml

@@ -33,12 +33,17 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-20.04, macOS-11, windows-2022]
-        pkg-name: ["app"]  # TODO: add "LAI" # "LAI" installs the monolithic package
-        python-version: [3.8]
+        pkg-name: ["app"]
+        python-version: ["3.8"]
         requires: ["oldest", "latest"]
+        include:
+          # "lightning" installs the monolithic package
+          - {os: "macOS-11", pkg-name: "lightning", python-version: "3.9", requires: "latest"}
+          - {os: "ubuntu-20.04", pkg-name: "lightning", python-version: "3.9", requires: "latest"}
+          - {os: "windows-2022", pkg-name: "lightning", python-version: "3.9", requires: "latest"}
 
     # Timeout: https://stackoverflow.com/a/59076067/4521646
-    timeout-minutes: 25
+    timeout-minutes: 30
 
     steps:
       - uses: actions/checkout@v3
@@ -54,21 +59,6 @@ jobs:
         if: ${{ matrix.requires == 'oldest' }}
         run: python .actions/assistant.py replace_oldest_ver
 
-      - name: switch PyTorch URL
-        run: python -c "print('TORCH_URL=https://download.pytorch.org/whl/' + str('test/cpu/torch_test.html' if '${{matrix.release}}' == 'pre' else 'cpu/torch_stable.html'))" >> $GITHUB_ENV
-
-      - name: Install package
-        env:
-          PACKAGE_NAME: ${{ matrix.pkg-name }}
-        run: |
-          pip install -e . pytest --upgrade --find-links ${TORCH_URL}
-          pip list
-
-      - name: DocTests App
-        if: ${{ matrix.pkg-name != 'LAI' }}
-        working-directory: src
-        run: python -m pytest lightning_app --ignore-glob="**/cli/*-template/**"
-
       - name: Get pip cache
         id: pip-cache
         run: |
@@ -81,6 +71,15 @@ jobs:
           key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ matrix.pkg-name }}-${{ matrix.requires }}-${{ hashFiles('requirements/app/base.txt') }}
           restore-keys: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ matrix.pkg-name }}-${{ matrix.requires }}-
 
+      - name: Switch PyTorch URL
+        run: python -c "print('TORCH_URL=https://download.pytorch.org/whl/' + str('test/cpu/torch_test.html' if '${{matrix.release}}' == 'pre' else 'cpu/torch_stable.html'))" >> $GITHUB_ENV
+
+      - name: Install package
+        env:
+          PACKAGE_NAME: ${{ matrix.pkg-name }}
+        run: |
+          pip install -e . pytest --upgrade --find-links ${TORCH_URL}
+          pip list
 
       - name: Install dependencies
         env:  # TODO: drop this when we will be using regular releases for testing
@@ -98,17 +97,17 @@ jobs:
         run: npm install -g yarn
 
       - name: Adjust tests
-        if: ${{ matrix.pkg-name == 'LAI' }}
+        if: ${{ matrix.pkg-name == 'lightning' }}
         run: python .actions/assistant.py copy_replace_imports --source_dir="./tests" --source_import="lightning_app" --target_import="lightning.app"
 
       - name: Adjust examples
-        if: ${{ matrix.pkg-name != 'LAI' }}
+        if: ${{ matrix.pkg-name != 'lightning' }}
         run: |
           python .actions/assistant.py copy_replace_imports --source_dir="./examples" --source_import="lightning.app" --target_import="lightning_app"
           python .actions/assistant.py copy_replace_imports --source_dir="./examples" --source_import="lightning" --target_import="lightning_app"
 
-      - name: switch coverage scope
-        run: python -c "print('COVERAGE_SCOPE=' + str('lightning' if '${{matrix.pkg-name}}' == 'LAI' else 'lightning_app'))" >> $GITHUB_ENV
+      - name: Switch coverage scope
+        run: python -c "print('COVERAGE_SCOPE=' + str('lightning' if '${{matrix.pkg-name}}' == 'lightning' else 'lightning_app'))" >> $GITHUB_ENV
 
       - name: Tests
         working-directory: ./tests