From 3e473323ca1f63a3cf4681d9e44fa9e779598090 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sat, 6 Sep 2025 17:15:41 +0200 Subject: [PATCH] Improve cache usage for prek hooks Previously we were only using prek cache from canary builds, but since we are starting to use several separate prek runs, it makes sense to install prek hooks only once and store them in cache and reuse even in the same build. This PR removes "only-canary" prek cache preparation - now all builds including all PRs from fork preapare prek cache once and upload them as artifacts - then restoring prek cache for every prek run should be much faster. --- .github/actions/install-prek/action.yml | 19 +++++++- .github/workflows/basic-tests.yml | 58 ++----------------------- .github/workflows/ci-amd.yml | 14 ++++-- .github/workflows/ci-arm.yml | 13 ++++-- .github/workflows/ci-image-checks.yml | 47 +------------------- 5 files changed, 44 insertions(+), 107 deletions(-) diff --git a/.github/actions/install-prek/action.yml b/.github/actions/install-prek/action.yml index e29294d4ede9c..d4be15daebc68 100644 --- a/.github/actions/install-prek/action.yml +++ b/.github/actions/install-prek/action.yml @@ -31,6 +31,9 @@ inputs: skip-prek-hooks: description: "Skip some prek hooks from installation" default: "" + save-cache: + description: "Whether to save prek cache" + required: true platform: description: 'Platform for the build - linux/amd64 or linux/arm64' required: true @@ -61,7 +64,7 @@ runs: uses: apache/infrastructure-actions/stash/restore@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468 with: # yamllint disable rule:line-length - key: cache-prek-v6-${{ inputs.platform }}-${{ inputs.python-version }}-${{ hashFiles('.pre-commit-config.yaml') }} + key: cache-prek-v6-${{ inputs.platform }}-${{ inputs.python-version }}-${{inputs.skip-prek-hooks}}-${{ hashFiles('.pre-commit-config.yaml') }} path: /tmp/ id: restore-prek-cache - name: "Check if prek cache tarball exists" @@ -91,3 +94,17 @@ runs: working-directory: ${{ github.workspace }} env: SKIP: ${{ inputs.skip-prek-hooks }} + - name: "Prepare .tar file from prek cache" + run: | + tar -C ~ -czf /tmp/cache-prek.tar.gz .cache/prek + shell: bash + if: inputs.save-cache == 'true' + - name: "Save prek cache" + uses: apache/infrastructure-actions/stash/save@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468 + with: + # yamllint disable rule:line-length + key: cache-prek-v6-${{ inputs.platform }}-${{ inputs.python-version }}-${{ inputs.skip-prek-hooks }}-${{ hashFiles('.pre-commit-config.yaml') }} + path: /tmp/cache-prek.tar.gz + if-no-files-found: 'error' + retention-days: '2' + if: inputs.save-cache == 'true' diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index af053ead94be6..0f61212f4683f 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -183,58 +183,6 @@ jobs: retention-days: '2' if: steps.restore-eslint-cache-simple-am-ui.outputs.stash-hit != 'true' - install-prek: - timeout-minutes: 5 - name: "Install prek for cache" - runs-on: ${{ fromJSON(inputs.runners) }} - env: - PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" - SKIP: ${{ inputs.skip-prek-hooks }} - steps: - - name: "Cleanup repo" - shell: bash - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - persist-credentials: false - - name: "Install Breeze" - uses: ./.github/actions/breeze - id: breeze - - name: "Install prek" - uses: ./.github/actions/install-prek - id: prek - with: - python-version: ${{steps.breeze.outputs.host-python-version}} - skip-prek-hooks: ${{ inputs.skip-prek-hooks }} - platform: ${{ inputs.platform }} - - check-translation-completness: - timeout-minutes: 15 - name: "Check translation completeness" - runs-on: ${{ fromJSON(inputs.runners) }} - needs: install-prek - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - persist-credentials: false - - name: "Install prek" - uses: ./.github/actions/install-prek - id: prek - with: - python-version: ${{steps.breeze.outputs.host-python-version}} - skip-prek-hooks: ${{ inputs.skip-prek-hooks }} - platform: ${{ inputs.platform }} - - name: "Check translation completeness" - run: > - prek --show-diff-on-failure --color always - --hook-stage manual --verbose --all-files - check-translations-completeness - env: - SKIP: ${{ inputs.skip-prek-hooks }} - COLUMNS: "202" - # Those checks are run if no image needs to be built for checks. This is for simple changes that # Do not touch any of the python code or any of the important files that might require building # The CI Docker image and they can be run entirely using the prek virtual environments on host @@ -242,7 +190,6 @@ jobs: timeout-minutes: 30 name: "Static checks: basic checks only" runs-on: ${{ fromJSON(inputs.runners) }} - needs: install-prek if: inputs.basic-checks-only == 'true' steps: - name: "Cleanup repo" @@ -264,6 +211,7 @@ jobs: python-version: ${{ steps.breeze.outputs.host-python-version }} skip-prek-hooks: ${{ inputs.skip-prek-hooks }} platform: ${{ inputs.platform }} + save-cache: false - name: Fetch incoming commit ${{ github.sha }} with its parent uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: @@ -295,7 +243,6 @@ jobs: timeout-minutes: 45 name: "Upgrade checks" runs-on: ${{ fromJSON(inputs.runners) }} - needs: install-prek env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" if: inputs.canary-run == 'true' @@ -316,9 +263,10 @@ jobs: uses: ./.github/actions/install-prek id: prek with: - python-version: ${{steps.breeze.outputs.host-python-version}} + python-version: ${{ steps.breeze.outputs.host-python-version }} skip-prek-hooks: ${{ inputs.skip-prek-hooks }} platform: ${{ inputs.platform }} + save-cache: false - name: "Autoupdate all prek hooks" run: prek autoupdate --freeze - name: "Autoupdate Lucas-C hooks to bleeding edge" diff --git a/.github/workflows/ci-amd.yml b/.github/workflows/ci-amd.yml index bebcf3a000ab8..584426f80b5e5 100644 --- a/.github/workflows/ci-amd.yml +++ b/.github/workflows/ci-amd.yml @@ -167,7 +167,14 @@ jobs: env: PR_LABELS: ${{ steps.source-run-info.outputs.pr-labels }} GITHUB_CONTEXT: ${{ toJson(github) }} - + - name: "Install and cache prek" + uses: ./.github/actions/install-prek + id: prek + with: + python-version: ${{ steps.breeze.outputs.host-python-version }} + skip-prek-hooks: ${{ needs.build-info.outputs.skip-prek-hooks }} + platform: "linux/amd64" + save-cache: true run-pin-versions-hook: name: "Run pin-versions hook" needs: [build-info] @@ -184,6 +191,7 @@ jobs: # octopin needs python 3.11 python-version: "3.11" platform: "linux/amd64" + save-cache: false - name: "Run pin-versions" run: > prek -c .github/.pre-commit-config.yaml --all-files --verbose --hook-stage manual @@ -201,8 +209,8 @@ jobs: run-www-tests: ${{needs.build-info.outputs.run-www-tests}} run-api-codegen: ${{needs.build-info.outputs.run-api-codegen}} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" - basic-checks-only: ${{needs.build-info.outputs.basic-checks-only}} - skip-prek-hooks: ${{needs.build-info.outputs.skip-prek-hooks}} + basic-checks-only: ${{ needs.build-info.outputs.basic-checks-only }} + skip-prek-hooks: ${{ needs.build-info.outputs.skip-prek-hooks }} canary-run: ${{needs.build-info.outputs.canary-run}} latest-versions-only: ${{needs.build-info.outputs.latest-versions-only}} use-uv: ${{needs.build-info.outputs.use-uv}} diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml index bd3300ce5a699..493c55afc8cfb 100644 --- a/.github/workflows/ci-arm.yml +++ b/.github/workflows/ci-arm.yml @@ -159,7 +159,14 @@ jobs: env: PR_LABELS: ${{ steps.source-run-info.outputs.pr-labels }} GITHUB_CONTEXT: ${{ toJson(github) }} - + - name: "Install and cache prek" + uses: ./.github/actions/install-prek + id: prek + with: + python-version: ${{ steps.breeze.outputs.host-python-version }} + skip-prek-hooks: ${{ needs.build-info.outputs.skip-prek-hooks }} + platform: "linux/arm64" + save-cache: true basic-tests: name: "Basic tests" needs: [build-info] @@ -170,8 +177,8 @@ jobs: run-www-tests: ${{needs.build-info.outputs.run-www-tests}} run-api-codegen: ${{needs.build-info.outputs.run-api-codegen}} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" - basic-checks-only: ${{needs.build-info.outputs.basic-checks-only}} - skip-prek-hooks: ${{needs.build-info.outputs.skip-prek-hooks}} + basic-checks-only: ${{ needs.build-info.outputs.basic-checks-only }} + skip-prek-hooks: ${{ needs.build-info.outputs.skip-prek-hooks }} canary-run: ${{needs.build-info.outputs.canary-run}} latest-versions-only: ${{needs.build-info.outputs.latest-versions-only}} use-uv: ${{needs.build-info.outputs.use-uv}} diff --git a/.github/workflows/ci-image-checks.yml b/.github/workflows/ci-image-checks.yml index 8bfff2f0007ff..02bb49ec41367 100644 --- a/.github/workflows/ci-image-checks.yml +++ b/.github/workflows/ci-image-checks.yml @@ -122,54 +122,10 @@ on: # yamllint disable-line rule:truthy permissions: contents: read jobs: - install-prek: - timeout-minutes: 5 - name: "Install prek for cache (only canary runs)" - runs-on: ${{ fromJSON(inputs.runners) }} - env: - PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" - if: inputs.basic-checks-only == 'false' - steps: - - name: "Cleanup repo" - shell: bash - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - if: inputs.canary-run == 'true' - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - persist-credentials: false - if: inputs.canary-run == 'true' - - name: "Install Breeze" - uses: ./.github/actions/breeze - id: breeze - if: inputs.canary-run == 'true' - - name: "Install prek" - uses: ./.github/actions/install-prek - id: prek - with: - python-version: ${{steps.breeze.outputs.host-python-version}} - platform: ${{ inputs.platform }} - if: inputs.canary-run == 'true' - - name: "Prepare .tar file from prek cache" - run: | - tar -C ~ -czf /tmp/cache-prek.tar.gz .cache/prek - shell: bash - if: inputs.canary-run == 'true' - - name: "Save prek cache" - uses: apache/infrastructure-actions/stash/save@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468 - with: - # yamllint disable rule:line-length - key: cache-prek-v6-${{ inputs.platform }}-${{ steps.breeze.outputs.host-python-version }}-${{ hashFiles('.pre-commit-config.yaml') }} - path: /tmp/cache-prek.tar.gz - if-no-files-found: 'error' - retention-days: '2' - if: inputs.canary-run == 'true' - static-checks: timeout-minutes: 45 name: "Static checks" runs-on: ${{ fromJSON(inputs.runners) }} - needs: install-prek env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" UPGRADE_TO_NEWER_DEPENDENCIES: "${{ inputs.upgrade-to-newer-dependencies }}" @@ -196,6 +152,7 @@ jobs: with: python-version: ${{steps.breeze.outputs.host-python-version}} platform: ${{ inputs.platform }} + save-cache: false - name: "Static checks" run: prek --all-files --show-diff-on-failure --color always env: @@ -210,7 +167,6 @@ jobs: timeout-minutes: 45 name: "MyPy checks" runs-on: ${{ fromJSON(inputs.runners) }} - needs: install-prek if: inputs.run-mypy == 'true' strategy: fail-fast: false @@ -240,6 +196,7 @@ jobs: with: python-version: ${{steps.breeze.outputs.host-python-version}} platform: ${{ inputs.platform }} + save-cache: false - name: "MyPy checks for ${{ matrix.mypy-check }}" run: prek --color always --verbose --hook-stage manual "$MYPY_CHECK" --all-files env: