feat: basic read_huggingface functionality (#4996)

## Changes Made

Adds `daft.read_huggingface`

## Related Issues

#2841

## Checklist

- [x] Documented in API Docs (if applicable)
- [x] Documented in User Guide (if applicable)
- [x] If adding a new documentation page, doc is added to
`docs/mkdocs.yml` navigation
- [x] Documentation builds and is formatted properly (tag @/ccmao1130
for docs review)

Author: kevinzwang

daft/__init__.py

@@ -90,6 +90,7 @@ def refresh_logger() -> None:
     read_lance,
     read_video_frames,
     read_warc,
+    read_huggingface,
 )
 from daft.series import Series
 from daft.session import (
@@ -206,6 +207,7 @@ def refresh_logger() -> None:
     "read_csv",
     "read_deltalake",
     "read_hudi",
+    "read_huggingface",
     "read_iceberg",
     "read_json",
     "read_lance",

daft/daft/__init__.pyi

@@ -703,6 +703,22 @@ class UnityConfig:
         """Replaces values if provided, returning a new UnityConfig."""
         ...
 
+class HuggingFaceConfig:
+    """I/O configuration for accessing Hugging Face datasets.
+
+    Args:
+        token (str, optional): Your Hugging Face access token, generated from https://huggingface.co/settings/tokens.
+        anonymous (bool, optional): Whether or not to use "anonymous mode", which will access Hugging Face without any credentials. Defaults to False.
+    """
+
+    token: str | None
+    anonymous: bool
+
+    def __init__(self, token: str | None = None, anonymous: bool | None = None): ...
+    def replace(self, token: str | None = None, anonymous: bool | None = None) -> HuggingFaceConfig:
+        """Replaces values if provided, returning a new HuggingFaceConfig."""
+        ...
+
 class IOConfig:
     """Configuration for the native I/O layer, e.g. credentials for accessing cloud storage systems."""
 
@@ -711,6 +727,7 @@ class IOConfig:
     gcs: GCSConfig
     http: HTTPConfig
     unity: UnityConfig
+    hf: HuggingFaceConfig
 
     def __init__(
         self,
@@ -719,6 +736,7 @@ class IOConfig:
         gcs: GCSConfig | None = None,
         http: HTTPConfig | None = None,
         unity: UnityConfig | None = None,
+        hf: HuggingFaceConfig | None = None,
     ): ...
     def replace(
         self,
@@ -727,6 +745,7 @@ class IOConfig:
         gcs: GCSConfig | None = None,
         http: HTTPConfig | None = None,
         unity: UnityConfig | None = None,
+        hf: HuggingFaceConfig | None = None,
     ) -> IOConfig:
         """Replaces values if provided, returning a new IOConfig."""
         ...

daft/io/__init__.py

@@ -8,6 +8,7 @@
     S3Config,
     S3Credentials,
     UnityConfig,
+    HuggingFaceConfig,
 )
 from daft.io._csv import read_csv
 from daft.io.delta_lake._deltalake import read_deltalake
@@ -18,6 +19,7 @@
 from daft.io._parquet import read_parquet
 from daft.io._sql import read_sql
 from daft.io._warc import read_warc
+from daft.io._huggingface import read_huggingface
 from daft.io._range import _range
 from daft.io.catalog import DataCatalogTable, DataCatalogType
 from daft.io.file_path import from_glob_path
@@ -34,6 +36,7 @@
     "DataSourceTask",
     "GCSConfig",
     "HTTPConfig",
+    "HuggingFaceConfig",
     "IOConfig",
     "S3Config",
     "S3Credentials",
@@ -43,6 +46,7 @@
     "read_csv",
     "read_deltalake",
     "read_hudi",
+    "read_huggingface",
     "read_iceberg",
     "read_json",
     "read_lance",

daft/io/_huggingface.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from daft.api_annotations import PublicAPI
+
+from ._parquet import read_parquet
+
+if TYPE_CHECKING:
+    from daft.daft import IOConfig
+    from daft.dataframe import DataFrame
+
+
+@PublicAPI
+def read_huggingface(repo: str, io_config: IOConfig | None = None) -> DataFrame:
+    """Create a DataFrame from a Hugging Face dataset.
+
+    Currently supports all public datasets and all private Parquet datasets. See [the Hugging Face docs](https://huggingface.co/docs/dataset-viewer/en/parquet) for more details.
+
+    Args:
+        repo (str): repository to read in the form `username/dataset_name`
+        io_config (IOConfig): Config to use when reading data
+    """
+    return read_parquet(f"hf://datasets/{repo}", io_config=io_config)

daft/udf/generator.py

@@ -45,9 +45,9 @@ For authenticated datasets:
 === "🐍 Python"
 
     ```python
-    from daft.io import IOConfig, HTTPConfig
+    from daft.io import IOConfig, HuggingFaceConfig
 
-    io_config = IoConfig(http=HTTPConfig(bearer_token="your_token"))
+    io_config = IOConfig(hf=HuggingFaceConfig(token="your_token"))
     df = daft.read_parquet("hf://datasets/username/dataset_name", io_config=io_config)
     ```
 

docs/connectors/huggingface.md

@@ -7,9 +7,6 @@ docker
 # Pinned aiohttp due to Ray connection issue in aiohttp==3.12.6
 aiohttp==3.12.4
 
-# Pinned requests due to docker-py issue: https://github.com/docker/docker-py/issues/3256
-requests<2.32.0
-
 # Pinned httpx due to unitycatalog-python issue: https://github.com/unitycatalog/unitycatalog-python/issues/9
 httpx==0.27.2
 
@@ -38,6 +35,7 @@ Pillow==10.4.0
 opencv-python==4.10.0.84
 tiktoken==0.9.0
 duckdb==1.1.2
+datasets==4.0.0
 
 # TQDM
 tqdm

requirements-dev.txt

@@ -2,14 +2,18 @@ use std::fmt::{Display, Formatter};
 
 use serde::{Deserialize, Serialize};
 
-use crate::{unity::UnityConfig, AzureConfig, GCSConfig, HTTPConfig, S3Config};
+use crate::{
+    huggingface::HuggingFaceConfig, unity::UnityConfig, AzureConfig, GCSConfig, HTTPConfig,
+    S3Config,
+};
 #[derive(Clone, Default, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)]
 pub struct IOConfig {
     pub s3: S3Config,
     pub azure: AzureConfig,
     pub gcs: GCSConfig,
     pub http: HTTPConfig,
     pub unity: UnityConfig,
+    pub hf: HuggingFaceConfig,
 }
 
 impl IOConfig {
@@ -36,6 +40,10 @@ impl IOConfig {
             "Unity config = {{ {} }}",
             self.unity.multiline_display().join(", ")
         ));
+        res.push(format!(
+            "Hugging Face config = {{ {} }}",
+            self.hf.multiline_display().join(", ")
+        ));
         res
     }
 }

src/common/io-config/src/config.rs

@@ -0,0 +1,32 @@
+use std::fmt::{Display, Formatter};
+
+use serde::{Deserialize, Serialize};
+
+use crate::ObfuscatedString;
+
+#[derive(Clone, Default, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)]
+pub struct HuggingFaceConfig {
+    pub token: Option<ObfuscatedString>,
+    pub anonymous: bool,
+}
+
+impl HuggingFaceConfig {
+    pub fn multiline_display(&self) -> Vec<String> {
+        let mut res = vec![];
+        if let Some(token) = &self.token {
+            res.push(format!("Token = {token}"));
+        }
+        res.push(format!("Anonymous = {}", self.anonymous));
+        res
+    }
+}
+
+impl Display for HuggingFaceConfig {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::result::Result<(), std::fmt::Error> {
+        write!(
+            f,
+            "HuggingFaceConfig\n{}",
+            self.multiline_display().join("\n")
+        )
+    }
+}

src/common/io-config/src/huggingface.rs

@@ -7,6 +7,7 @@ mod azure;
 mod config;
 mod gcs;
 mod http;
+mod huggingface;
 mod s3;
 mod unity;
 
@@ -23,6 +24,7 @@ pub use crate::{
     config::IOConfig,
     gcs::GCSConfig,
     http::HTTPConfig,
+    huggingface::HuggingFaceConfig,
     s3::{S3Config, S3Credentials},
     unity::UnityConfig,
 };