Improve cache usage for prek hooks

Previously we were only using prek cache from canary builds, but
since we are starting to use several separate prek runs, it makes
sense to install prek hooks only once and store them in cache and
reuse even in the same build.

This PR removes "only-canary" prek cache preparation - now all
builds including all PRs from fork preapare prek cache once and
upload them as artifacts - then restoring prek cache for every
prek run should be much faster.

Author: potiuk

.github/actions/install-prek/action.yml

@@ -31,6 +31,9 @@ inputs:
   skip-prek-hooks:
     description: "Skip some prek hooks from installation"
     default: ""
+  save-cache:
+    description: "Whether to save prek cache"
+    required: true
   platform:
     description: 'Platform for the build - linux/amd64 or linux/arm64'
     required: true
@@ -61,7 +64,7 @@ runs:
       uses: apache/infrastructure-actions/stash/restore@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
       with:
         # yamllint disable rule:line-length
-        key: cache-prek-v6-${{ inputs.platform }}-${{ inputs.python-version }}-${{ hashFiles('.pre-commit-config.yaml') }}
+        key: cache-prek-v6-${{ inputs.platform }}-${{ inputs.python-version }}-${{inputs.skip-prek-hooks}}-${{ hashFiles('.pre-commit-config.yaml') }}
         path: /tmp/
       id: restore-prek-cache
     - name: "Check if prek cache tarball exists"
@@ -91,3 +94,17 @@ runs:
       working-directory: ${{ github.workspace }}
       env:
         SKIP: ${{ inputs.skip-prek-hooks }}
+    - name: "Prepare .tar file from prek cache"
+      run: |
+        tar -C ~ -czf /tmp/cache-prek.tar.gz .cache/prek
+      shell: bash
+      if: inputs.save-cache == 'true'
+    - name: "Save prek cache"
+      uses: apache/infrastructure-actions/stash/save@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
+      with:
+        # yamllint disable rule:line-length
+        key: cache-prek-v6-${{ inputs.platform }}-${{ steps.breeze.outputs.host-python-version }}-${{inputs.skip-prek-hooks}}-${{ hashFiles('.pre-commit-config.yaml') }}
+        path: /tmp/cache-prek.tar.gz
+        if-no-files-found: 'error'
+        retention-days: '2'
+      if: inputs.save-cache == 'true'

.github/workflows/basic-tests.yml

@@ -183,40 +183,13 @@ jobs:
           retention-days: '2'
         if: steps.restore-eslint-cache-simple-am-ui.outputs.stash-hit != 'true'
 
-  install-prek:
-    timeout-minutes: 5
-    name: "Install prek for cache"
-    runs-on: ${{ fromJSON(inputs.runners) }}
-    env:
-      PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
-      SKIP: ${{ inputs.skip-prek-hooks }}
-    steps:
-      - name: "Cleanup repo"
-        shell: bash
-        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
-      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
-        with:
-          persist-credentials: false
-      - name: "Install Breeze"
-        uses: ./.github/actions/breeze
-        id: breeze
-      - name: "Install prek"
-        uses: ./.github/actions/install-prek
-        id: prek
-        with:
-          python-version: ${{steps.breeze.outputs.host-python-version}}
-          skip-prek-hooks: ${{ inputs.skip-prek-hooks }}
-          platform: ${{ inputs.platform }}
-
   # Those checks are run if no image needs to be built for checks. This is for simple changes that
   # Do not touch any of the python code or any of the important files that might require building
   # The CI Docker image and they can be run entirely using the prek virtual environments on host
   static-checks-basic-checks-only:
     timeout-minutes: 30
     name: "Static checks: basic checks only"
     runs-on: ${{ fromJSON(inputs.runners) }}
-    needs: install-prek
     if: inputs.basic-checks-only == 'true'
     steps:
       - name: "Cleanup repo"
@@ -238,6 +211,7 @@ jobs:
           python-version: ${{ steps.breeze.outputs.host-python-version }}
           skip-prek-hooks: ${{ inputs.skip-prek-hooks }}
           platform: ${{ inputs.platform }}
+          save-cache: false
       - name: Fetch incoming commit ${{ github.sha }} with its parent
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
         with:
@@ -269,7 +243,6 @@ jobs:
     timeout-minutes: 45
     name: "Upgrade checks"
     runs-on: ${{ fromJSON(inputs.runners) }}
-    needs: install-prek
     env:
       PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
     if: inputs.canary-run == 'true'
@@ -290,9 +263,10 @@ jobs:
         uses: ./.github/actions/install-prek
         id: prek
         with:
-          python-version: ${{steps.breeze.outputs.host-python-version}}
+          python-version: ${{ steps.breeze.outputs.host-python-version }}
           skip-prek-hooks: ${{ inputs.skip-prek-hooks }}
           platform: ${{ inputs.platform }}
+          save-cache: false
       - name: "Autoupdate all prek hooks"
         run: prek autoupdate --freeze
       - name: "Autoupdate Lucas-C hooks to bleeding edge"

.github/workflows/ci-amd.yml

@@ -167,6 +167,14 @@ jobs:
         env:
           PR_LABELS: ${{ steps.source-run-info.outputs.pr-labels }}
           GITHUB_CONTEXT: ${{ toJson(github) }}
+      - name: "Install and cache prek"
+        uses: ./.github/actions/install-prek
+        id: prek
+        with:
+          python-version: ${{ steps.breeze.outputs.host-python-version }}
+          skip-prek-hooks: ${{ needs.build-info.outputs.skip-prek-hooks }}
+          platform: ${{ inputs.platform }}
+          save-cache: true
 
   run-pin-versions-hook:
     name: "Run pin-versions hook"
@@ -184,6 +192,7 @@ jobs:
           # octopin needs python 3.11
           python-version: "3.11"
           platform: "linux/amd64"
+          save-cache: false
       - name: "Run pin-versions"
         run: >
           prek -c .github/.pre-commit-config.yaml --all-files --verbose --hook-stage manual
@@ -201,8 +210,8 @@ jobs:
       run-www-tests: ${{needs.build-info.outputs.run-www-tests}}
       run-api-codegen: ${{needs.build-info.outputs.run-api-codegen}}
       default-python-version: "${{ needs.build-info.outputs.default-python-version }}"
-      basic-checks-only: ${{needs.build-info.outputs.basic-checks-only}}
-      skip-prek-hooks: ${{needs.build-info.outputs.skip-prek-hooks}}
+      basic-checks-only: ${{ needs.build-info.outputs.basic-checks-only }}
+      skip-prek-hooks: ${{ needs.build-info.outputs.skip-prek-hooks }}
       canary-run: ${{needs.build-info.outputs.canary-run}}
       latest-versions-only: ${{needs.build-info.outputs.latest-versions-only}}
       use-uv: ${{needs.build-info.outputs.use-uv}}

.github/workflows/ci-arm.yml

@@ -170,8 +170,8 @@ jobs:
       run-www-tests: ${{needs.build-info.outputs.run-www-tests}}
       run-api-codegen: ${{needs.build-info.outputs.run-api-codegen}}
       default-python-version: "${{ needs.build-info.outputs.default-python-version }}"
-      basic-checks-only: ${{needs.build-info.outputs.basic-checks-only}}
-      skip-prek-hooks: ${{needs.build-info.outputs.skip-prek-hooks}}
+      basic-checks-only: ${{ needs.build-info.outputs.basic-checks-only }}
+      skip-prek-hooks: ${{ needs.build-info.outputs.skip-prek-hooks }}
       canary-run: ${{needs.build-info.outputs.canary-run}}
       latest-versions-only: ${{needs.build-info.outputs.latest-versions-only}}
       use-uv: ${{needs.build-info.outputs.use-uv}}

.github/workflows/ci-image-checks.yml

@@ -122,54 +122,10 @@ on:  # yamllint disable-line rule:truthy
 permissions:
   contents: read
 jobs:
-  install-prek:
-    timeout-minutes: 5
-    name: "Install prek for cache (only canary runs)"
-    runs-on: ${{ fromJSON(inputs.runners) }}
-    env:
-      PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
-    if: inputs.basic-checks-only == 'false'
-    steps:
-      - name: "Cleanup repo"
-        shell: bash
-        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
-        if: inputs.canary-run == 'true'
-      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
-        with:
-          persist-credentials: false
-        if: inputs.canary-run == 'true'
-      - name: "Install Breeze"
-        uses: ./.github/actions/breeze
-        id: breeze
-        if: inputs.canary-run == 'true'
-      - name: "Install prek"
-        uses: ./.github/actions/install-prek
-        id: prek
-        with:
-          python-version: ${{steps.breeze.outputs.host-python-version}}
-          platform: ${{ inputs.platform }}
-        if: inputs.canary-run == 'true'
-      - name: "Prepare .tar file from prek cache"
-        run: |
-          tar -C ~ -czf /tmp/cache-prek.tar.gz .cache/prek
-        shell: bash
-        if: inputs.canary-run == 'true'
-      - name: "Save prek cache"
-        uses: apache/infrastructure-actions/stash/save@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
-        with:
-          # yamllint disable rule:line-length
-          key: cache-prek-v6-${{ inputs.platform }}-${{ steps.breeze.outputs.host-python-version }}-${{ hashFiles('.pre-commit-config.yaml') }}
-          path: /tmp/cache-prek.tar.gz
-          if-no-files-found: 'error'
-          retention-days: '2'
-        if: inputs.canary-run == 'true'
-
   static-checks:
     timeout-minutes: 45
     name: "Static checks"
     runs-on: ${{ fromJSON(inputs.runners) }}
-    needs: install-prek
     env:
       PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
       UPGRADE_TO_NEWER_DEPENDENCIES: "${{ inputs.upgrade-to-newer-dependencies }}"
@@ -196,6 +152,7 @@ jobs:
         with:
           python-version: ${{steps.breeze.outputs.host-python-version}}
           platform: ${{ inputs.platform }}
+          save-cache: false
       - name: "Static checks"
         run: prek --all-files --show-diff-on-failure --color always
         env:
@@ -210,7 +167,6 @@ jobs:
     timeout-minutes: 45
     name: "MyPy checks"
     runs-on: ${{ fromJSON(inputs.runners) }}
-    needs: install-prek
     if: inputs.run-mypy == 'true'
     strategy:
       fail-fast: false
@@ -240,6 +196,7 @@ jobs:
         with:
           python-version: ${{steps.breeze.outputs.host-python-version}}
           platform: ${{ inputs.platform }}
+          save-cache: false
       - name: "MyPy checks for ${{ matrix.mypy-check }}"
         run: prek --color always --verbose --hook-stage manual "$MYPY_CHECK" --all-files
         env: