ci: remove macos from PR test suite (#5142)

## Changes Made

We currently use MacOS to run unit tests in PRs because we had an issue
with getting Rust code coverage in Ubuntu 22.04. The latest Rust
toolchain no longer has this issue, so this PR updates the rust
toolchain to the latest nightly and moves the code coverage tests to
Ubuntu.

The vast majority of the diff here is just linting and lifetime fixes
due to the upgrade to the latest nightly Rust + 2024 edition. The rest
of the changes are as follows:
- moving the code coverage tests in
`.github/workflows/pr-test-suite.yml` to Ubuntu as described above
- it was failing due to out of disk space so I also added a disk space
remover action
- update our Rust versions in `rust-toolchain.toml` and `Cargo.toml`
- use `hashbrown::HashMap` instead of `std::HashMap` because
`raw_entry_mut` was removed from the standard library

## Related Issues

#3801

## Checklist

- [x] Documented in API Docs (if applicable)
- [x] Documented in User Guide (if applicable)
- [x] If adding a new documentation page, doc is added to
`docs/mkdocs.yml` navigation
- [x] Documentation builds and is formatted properly (tag @/ccmao1130
for docs review)

Author: kevinzwang

.github/workflows/pr-test-suite.yml

@@ -71,30 +71,32 @@ jobs:
   unit-test:
     needs: skipcheck
     if: ${{ needs.skipcheck.outputs.skip == 'false' }}
-    runs-on: ${{ matrix.os }}
-    timeout-minutes: 45
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
     strategy:
       fail-fast: false
       matrix:
         python-version: ['3.9', '3.12']
         daft-runner: [ray, native]
         pyarrow-version: [8.0.0, 19.0.1]
         flotilla: [1, 0]
-        os: [ubuntu-latest, macos-latest]
         exclude:
         - daft-runner: native
           flotilla: 0
         - daft-runner: ray
           pyarrow-version: 8.0.0
-        - python-version: '3.9'
-          os: macos-latest
-        - pyarrow-version: 8.0.0
-          os: macos-latest
         - python-version: '3.12'
           pyarrow-version: 8.0.0
-        - os: macos-latest
-          flotilla: 0
     steps:
+    - name: Free Disk Space (Ubuntu) # only run on ubuntu
+      uses: jlumbroso/free-disk-space@main
+      with:
+        tool-cache: false
+
+    - run: |
+        sudo apt-get update
+        sudo apt-get install ffmpeg libsm6 libxext6 -y
+
     - uses: actions/checkout@v4
     - uses: moonrepo/setup-rust@v1
       with:
@@ -135,27 +137,7 @@ jobs:
         source .venv/bin/activate
         uv pip install pyarrow==${{ matrix.pyarrow-version }}
 
-    # Rust code coverage does not work on ubuntu-latest, so we only run it on macOS
-    # For more info: https://github.com/Eventual-Inc/Daft/issues/3801
-    - name: Build library and Test with pytest (Linux)
-      if: ${{ (runner.os == 'Linux') }}
-      run: |
-        source .venv/bin/activate
-        maturin develop --uv
-        pytest --ignore tests/integration --collect-only -qq # run this to ensure no weird imports that result in `Collector` errors
-        pytest --cov=daft --ignore tests/integration --durations=0 | ./tools/capture-durations.sh "pytest_output.txt"
-        python tools/aggregate_test_durations.py pytest_output.txt
-        coverage combine -a --data-file='.coverage' || true
-        mkdir -p report-output
-        coverage xml -o ./report-output/coverage-${{ join(matrix.*, '-') }}.xml
-      env:
-        CARGO_TARGET_DIR: ./target
-
-        DAFT_RUNNER: ${{ matrix.daft-runner }}
-        DAFT_FLOTILLA: ${{ matrix.flotilla }}
-
-    - name: Build library and Test with pytest with code coverage (macOS)
-      if: ${{ (runner.os == 'macOS') }}
+    - name: Build library and Test with pytest
       run: |
         source .venv/bin/activate
         cargo llvm-cov clean --workspace
@@ -180,7 +162,6 @@ jobs:
         DAFT_FLOTILLA: ${{ matrix.flotilla }}
 
     - name: Upload coverage report
-      if: ${{ (runner.os == 'macOS') }}
       uses: actions/upload-artifact@v4
       with:
         name: coverage-reports-unit-tests-${{ join(matrix.*, '-') }}

.pre-commit-config.yaml

@@ -73,7 +73,7 @@ repos:
   - id: fmt
     name: Rust code Formatting
     description: Format files with cargo fmt.
-    entry: cargo fmt
+    entry: cargo fmt --manifest-path Cargo.toml
     language: system
     types: [rust]
     args: [--]

Cargo.lock

@@ -118,7 +118,7 @@ crate-type = ["cdylib"]
 name = "daft"
 
 [package]
-edition = "2021"
+edition = "2024"
 name = "daft"
 publish = false
 version = "0.3.0-dev0"
@@ -278,6 +278,7 @@ daft-sql = {path = "src/daft-sql"}
 derive_builder = "0.20.2"
 educe = "0.6.0"
 futures = "0.3.30"
+hashbrown = "0.16"
 html-escape = "0.2.13"
 image = {version = "0.25.5", default-features = false}
 indexmap = "2.9.0"
@@ -454,5 +455,5 @@ wildcard_imports = "allow"
 zero_sized_map_values = "allow"
 
 [workspace.package]
-edition = "2021"
+edition = "2024"
 version = "0.3.0-dev0"

Cargo.toml

@@ -1,4 +1,4 @@
 [toolchain]
-channel = "nightly-2025-01-19"
+channel = "nightly-2025-09-03"
 components = ["rustfmt", "clippy"]
 profile = "minimal"

rust-toolchain.toml

@@ -125,7 +125,7 @@ impl Bitmap {
     ///
     /// The returned tuple contains:
     /// * `.1`: The byte slice, truncated to the start of the first bit. So the start of the slice
-    ///       is within the first 8 bits.
+    ///   is within the first 8 bits.
     /// * `.2`: The start offset in bits on a range `0 <= offsets < 8`.
     /// * `.3`: The length in number of bits.
     #[inline]

src/arrow2/src/bitmap/immutable.rs

@@ -48,6 +48,7 @@ pub trait FixedLengthEncoding: Copy {
 
     fn encode(self) -> Self::Encoded;
 
+    #[allow(dead_code)]
     fn decode(encoded: Self::Encoded) -> Self;
 }
 

src/arrow2/src/compute/sort/row/fixed.rs

@@ -36,17 +36,16 @@ use std::{
     sync::Arc,
 };
 
-use crate::{
-    array::{Array, BinaryArray, BooleanArray, DictionaryArray, PrimitiveArray, Utf8Array},
-    datatypes::PhysicalType,
-    error::*,
-};
-use crate::{compute::sort::SortOptions, datatypes::DataType};
-
 use self::{
     dictionary::{compute_dictionary_mapping, encode_dictionary},
     interner::OrderPreservingInterner,
 };
+use crate::{
+    array::{Array, BinaryArray, BooleanArray, DictionaryArray, PrimitiveArray, Utf8Array},
+    compute::sort::SortOptions,
+    datatypes::{DataType, PhysicalType},
+    error::*,
+};
 
 mod dictionary;
 mod fixed;
@@ -378,7 +377,7 @@ impl<'a> Eq for Row<'a> {}
 impl<'a> PartialOrd for Row<'a> {
     #[inline]
     fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
-        Some(self.data.cmp(other.data))
+        Some(self.cmp(other))
     }
 }
 

src/arrow2/src/compute/sort/row/mod.rs

@@ -19,12 +19,8 @@ impl From<parquet2::error::Error> for Error {
                     .to_string();
                 Error::ExternalFormat(message)
             }
-            parquet2::error::Error::Transport(msg) => {
-                Error::Io(std::io::Error::new(std::io::ErrorKind::Other, msg))
-            }
-            parquet2::error::Error::IoError(msg) => {
-                Error::Io(std::io::Error::new(std::io::ErrorKind::Other, msg))
-            }
+            parquet2::error::Error::Transport(msg) => Error::Io(std::io::Error::other(msg)),
+            parquet2::error::Error::IoError(msg) => Error::Io(std::io::Error::other(msg)),
             _ => Error::ExternalFormat(error.to_string()),
         }
     }

src/arrow2/src/io/parquet/mod.rs

@@ -6,11 +6,12 @@ use parquet2::{
     read::levels::get_bit_width,
 };
 
+use super::{
+    super::Pages,
+    utils::{DecodedState, MaybeNext, PageState},
+};
 use crate::{array::Array, bitmap::MutableBitmap, error::Result};
 
-use super::utils::{DecodedState, MaybeNext};
-use super::{super::Pages, utils::PageState};
-
 /// trait describing deserialized repetition and definition levels
 pub trait Nested: std::fmt::Debug + Send + Sync {
     fn inner(&mut self) -> (Vec<i64>, Option<MutableBitmap>);
@@ -455,8 +456,7 @@ pub(super) fn extend<'a, D: NestedDecoder<'a>>(
 /// * `decoded`          - The state of our decoded values.
 /// * `decoder`          - The decoder for the leaf-level type.
 /// * `additional`       - The number of top-level rows to read for the current chunk. This is the
-///                        min of `chunk size - number of rows existing in the current chunk` and
-///                        `rows_remaining`.
+///   min of `chunk size - number of rows existing in the current chunk` and `rows_remaining`.
 #[allow(clippy::too_many_arguments)]
 fn extend_offsets2<'a, D: NestedDecoder<'a>>(
     page: &mut NestedPage<'a>,