ctest: Add translation of Rust types.

Author: mbyx

ctest-next/Cargo.toml

@@ -12,4 +12,8 @@ askama = "0.14.0"
 cc = "1.2.25"
 quote = "1.0.40"
 syn = { version = "2.0.101", features = ["full", "visit", "extra-traits"] }
+thiserror = "2.0.12"
+
+[dev-dependencies]
 tempfile = "3.20.0"
+

ctest-next/build.rs

@@ -1,31 +1,55 @@
 use std::env;
 
+// When we call `cargo test` for a cross compiled target, the following is required:
+// - CARGO_TARGET_{}_LINKER: To link the integration tests.
+// - CARGO_TARGET_{}_RUNNER: To run the integration tests.
+//
+// This is already set by the CI for all platforms, so there is no problem up till here.
+//
+// The integration tests (which are run in qemu, but use host rustc and cc) require the following:
+// - TARGET_PLATFORM or target set manually. (We forward TARGET in build.rs for this.)
+// - HOST_PLATFORM or host set manually. (We forward HOST in build.rs for this.)
+// - LINKER: To link the C headers. (We forward CARGO_TARGET_{}_LINKER for this.)
+// - FLAGS: Any flags to pass when compiling the test binary for the cross compiled platform. (Forwarded from CARGO_TARGET_{}_RUSTFLAGS)
+// - RUNNER: To run the test binary with. (Forward the same runner as CARGO_TARGET_{}_RUNNER)
+//
+// The TARGET_PLATFORM and HOST_PLATFORM variables are not an issue, cargo will
+// automatically set TARGET and PLATFORM and we will forward them.
+//
+// Similarly FLAGS and RUNNER are also not an issue, if CARGO_TARGET_{}_RUSTFLAGS are present they're forwarded.
+// And RUSTFLAGS works by default anyway. Similarly the test binary doesn't require any external applications so just
+// the RUNNER is enough to run it.
+//
+// However since rustc and cc are the host versions, they will only work if we specify the correct variables for them.
+// Because we only use them to compile, not run things.
+// For CC we MUST specify CC or CC_target otherwise it will fail. (Other flags like AR etc. work without forwarding because it is run in the host.)
+// For rustc we MUST specify the correct linker. Usually this is the same as CC or CC_target.
+//
+// In the CI, the CARGO_TARGET_{} variables are always set.
+
 fn main() {
     let host = env::var("HOST").unwrap();
     let target = env::var("TARGET").unwrap();
-    let target_key = target.replace('-', "_");
+    let target_key = target.replace('-', "_").to_uppercase();
 
     println!("cargo:rustc-env=HOST_PLATFORM={host}");
     println!("cargo:rustc-env=TARGET_PLATFORM={target}");
 
-    let linker = env::var(format!("CARGO_TARGET_{}_LINKER", target_key.to_uppercase()))
-        .or_else(|_| env::var("CC"))
-        .or_else(|_| env::var(format!("CC_{target_key}")))
-        .unwrap_or_default();
-
-    let runner =
-        env::var(format!("CARGO_TARGET_{}_RUNNER", target_key.to_uppercase())).unwrap_or_default();
-
+    let linker = env::var(format!("CARGO_TARGET_{target_key}_LINKER")).unwrap_or_default();
+    let runner = env::var(format!("CARGO_TARGET_{target_key}_RUNNER")).unwrap_or_default();
     // As we invoke rustc directly this does not get passed to it, although RUSTFLAGS does.
-    let flags = env::var(format!(
-        "CARGO_TARGET_{}_RUSTFLAGS",
-        target_key.to_uppercase()
-    ))
-    .unwrap_or_default();
+    let flags = env::var(format!("CARGO_TARGET_{target_key}_RUSTFLAGS")).unwrap_or_default();
 
+    // Forward build environment variables so they can be used in tests.
     println!("cargo:rustc-env=LINKER={linker}");
     println!("cargo:rustc-env=RUNNER={runner}");
     println!("cargo:rustc-env=FLAGS={flags}");
 
+    // Rerun this build script if any of the environment variables change.
+    println!("cargo:rerun-if-changed-env=HOST");
     println!("cargo:rerun-if-changed-env=TARGET");
+    println!("cargo:rerun-if-changed-env=CARGO_TARGET_{target_key}_LINKER",);
+    println!("cargo:rerun-if-changed-env=CC");
+    println!("cargo:rerun-if-changed-env=CC_{target_key}");
+    println!("cargo:rerun-if-changed-env=CARGO_TARGET_{target_key}_RUNNER",);
 }

ctest-next/src/generator.rs

@@ -16,13 +16,12 @@ use crate::{
 };
 
 /// A builder used to generate a test suite.
-#[non_exhaustive]
 #[derive(Default, Debug, Clone)]
 pub struct TestGenerator {
     headers: Vec<String>,
-    target: Option<String>,
-    host: Option<String>,
-    includes: Vec<PathBuf>,
+    pub(crate) target: Option<String>,
+    pub(crate) host: Option<String>,
+    pub(crate) includes: Vec<PathBuf>,
     out_dir: Option<PathBuf>,
 }
 
@@ -33,12 +32,19 @@ impl TestGenerator {
     }
 
     /// Add a header to be included as part of the generated C file.
+    ///
+    /// The generate C test will be compiled by a C compiler, and this can be
+    /// used to ensure that all the necessary header files are included to test
+    /// all FFI definitions.
     pub fn header(&mut self, header: &str) -> &mut Self {
         self.headers.push(header.to_string());
         self
     }
 
     /// Configures the target to compile C code for.
+    ///
+    /// Note that for Cargo builds this defaults to `$TARGET` and it's not
+    /// necessary to call.
     pub fn target(&mut self, target: &str) -> &mut Self {
         self.target = Some(target.to_string());
         self
@@ -65,71 +71,13 @@ impl TestGenerator {
         self
     }
 
-    /// Generate all tests for the given crate and output the Rust side to a file.
-    pub fn generate<P: AsRef<Path>>(&mut self, crate_path: P, output_file_path: P) -> Result<()> {
-        let output_file_path = self.generate_files(crate_path, output_file_path)?;
-
-        let target = self
-            .target
-            .clone()
-            .unwrap_or(env::var("TARGET_PLATFORM").unwrap());
-        let host = self
-            .host
-            .clone()
-            .unwrap_or(env::var("HOST_PLATFORM").unwrap());
-
-        let mut cfg = cc::Build::new();
-        // FIXME: Cpp not supported.
-        cfg.file(output_file_path.with_extension("c"));
-        cfg.host(&host);
-        if target.contains("msvc") {
-            cfg.flag("/W3")
-                .flag("/Wall")
-                .flag("/WX")
-                // ignored warnings
-                .flag("/wd4820") // warning about adding padding?
-                .flag("/wd4100") // unused parameters
-                .flag("/wd4996") // deprecated functions
-                .flag("/wd4296") // '<' being always false
-                .flag("/wd4255") // converting () to (void)
-                .flag("/wd4668") // using an undefined thing in preprocessor?
-                .flag("/wd4366") // taking ref to packed struct field might be unaligned
-                .flag("/wd4189") // local variable initialized but not referenced
-                .flag("/wd4710") // function not inlined
-                .flag("/wd5045") // compiler will insert Spectre mitigation
-                .flag("/wd4514") // unreferenced inline function removed
-                .flag("/wd4711"); // function selected for automatic inline
-        } else {
-            cfg.flag("-Wall")
-                .flag("-Wextra")
-                .flag("-Werror")
-                .flag("-Wno-unused-parameter")
-                .flag("-Wno-type-limits")
-                // allow taking address of packed struct members:
-                .flag("-Wno-address-of-packed-member")
-                .flag("-Wno-unknown-warning-option")
-                .flag("-Wno-deprecated-declarations"); // allow deprecated items
-        }
-
-        for p in &self.includes {
-            cfg.include(p);
-        }
-
-        let stem: &str = output_file_path.file_stem().unwrap().to_str().unwrap();
-        cfg.target(&target)
-            .out_dir(output_file_path.parent().unwrap())
-            .compile(stem);
-
-        Ok(())
-    }
-
     /// Generate the Rust and C testing files.
     ///
-    /// Returns the path to the generated file.
-    pub(crate) fn generate_files<P: AsRef<Path>>(
+    /// Returns the path to t generated file.
+    pub fn generate_files(
         &mut self,
-        crate_path: P,
-        output_file_path: P,
+        crate_path: impl AsRef<Path>,
+        output_file_path: impl AsRef<Path>,
     ) -> Result<PathBuf> {
         let expanded = expand(crate_path)?;
         let ast = syn::parse_file(&expanded)?;
@@ -140,7 +88,7 @@ impl TestGenerator {
         let output_directory = self
             .out_dir
             .clone()
-            .unwrap_or_else(|| PathBuf::from(env::var_os("OUT_DIR").unwrap()));
+            .unwrap_or_else(|| env::var("OUT_DIR").unwrap().into());
         let output_file_path = output_directory.join(output_file_path);
 
         // Generate the Rust side of the tests.

ctest-next/src/lib.rs

@@ -16,15 +16,14 @@ mod ffi_items;
 mod generator;
 mod macro_expansion;
 mod runner;
-mod rustc_version;
 mod template;
 mod translator;
 
 pub use ast::{Abi, Const, Field, Fn, Parameter, Static, Struct, Type, Union};
 pub use generator::TestGenerator;
 pub use macro_expansion::expand;
-pub use runner::{compile_test, run_test};
-pub use rustc_version::{rustc_version, RustcVersion};
+pub use runner::{__compile_test, __run_test, generate_test};
+pub use translator::TranslationError;
 
 /// A possible error that can be encountered in our library.
 pub type Error = Box<dyn std::error::Error>;

ctest-next/src/macro_expansion.rs

@@ -9,6 +9,8 @@ pub fn expand<P: AsRef<Path>>(crate_path: P) -> Result<String> {
     let output = Command::new(rustc)
         .env("RUSTC_BOOTSTRAP", "1")
         .arg("-Zunpretty=expanded")
+        .arg("--edition")
+        .arg("2024") // By default, -Zunpretty=expanded uses 2015 edition.
         .arg(canonicalize(crate_path)?)
         .output()?;
 

ctest-next/src/runner.rs

@@ -1,50 +1,124 @@
-use crate::Result;
+use crate::{Result, TestGenerator};
 use std::env;
 use std::fs::{canonicalize, File};
 use std::io::Write;
 use std::path::{Path, PathBuf};
 use std::process::Command;
 
+/// Generate all tests for the given crate and output the Rust side to a file.
+pub fn generate_test(
+    generator: &mut TestGenerator,
+    crate_path: impl AsRef<Path>,
+    output_file_path: impl AsRef<Path>,
+) -> Result<PathBuf> {
+    let output_file_path = generator.generate_files(crate_path, output_file_path)?;
+
+    // Search for the target and host to build for if specified manually (generator.target, generator.host),
+    // via build script (TARGET, HOST), or for internal testing (TARGET_PLATFORM, HOST_PLATFORM).
+    let target = generator.target.clone().unwrap_or_else(|| {
+        env::var("TARGET").unwrap_or_else(|_| env::var("TARGET_PLATFORM").unwrap())
+    });
+    let host = generator
+        .host
+        .clone()
+        .unwrap_or_else(|| env::var("HOST").unwrap_or_else(|_| env::var("HOST_PLATFORM").unwrap()));
+
+    let mut cfg = cc::Build::new();
+    // FIXME: Cpp not supported.
+    cfg.file(output_file_path.with_extension("c"));
+    cfg.host(&host);
+
+    if target.contains("msvc") {
+        cfg.flag("/W3")
+            .flag("/Wall")
+            .flag("/WX")
+            // ignored warnings
+            .flag("/wd4820") // warning about adding padding?
+            .flag("/wd4100") // unused parameters
+            .flag("/wd4996") // deprecated functions
+            .flag("/wd4296") // '<' being always false
+            .flag("/wd4255") // converting () to (void)
+            .flag("/wd4668") // using an undefined thing in preprocessor?
+            .flag("/wd4366") // taking ref to packed struct field might be unaligned
+            .flag("/wd4189") // local variable initialized but not referenced
+            .flag("/wd4710") // function not inlined
+            .flag("/wd5045") // compiler will insert Spectre mitigation
+            .flag("/wd4514") // unreferenced inline function removed
+            .flag("/wd4711"); // function selected for automatic inline
+    } else {
+        cfg.flag("-Wall")
+            .flag("-Wextra")
+            .flag("-Werror")
+            .flag("-Wno-unused-parameter")
+            .flag("-Wno-type-limits")
+            // allow taking address of packed struct members:
+            .flag("-Wno-address-of-packed-member")
+            .flag("-Wno-unknown-warning-option")
+            .flag("-Wno-deprecated-declarations"); // allow deprecated items
+    }
+
+    for p in &generator.includes {
+        cfg.include(p);
+    }
+
+    let stem: &str = output_file_path.file_stem().unwrap().to_str().unwrap();
+    cfg.target(&target)
+        .out_dir(output_file_path.parent().unwrap())
+        .compile(stem);
+
+    Ok(output_file_path)
+}
+
 /// Compiles a Rust source file and links it against a static library.
 ///
 /// Returns the path to the generated binary.
-pub fn compile_test<P: AsRef<Path>>(
-    output_dir: P,
-    crate_path: P,
-    library_file: P,
+#[doc(hidden)]
+pub fn __compile_test(
+    output_dir: impl AsRef<Path>,
+    crate_path: impl AsRef<Path>,
+    library_file: impl AsRef<Path>,
 ) -> Result<PathBuf> {
     let rustc = env::var("RUSTC").unwrap_or_else(|_| "rustc".into());
     let output_dir = output_dir.as_ref();
     let crate_path = crate_path.as_ref();
-    let library_file = library_file.as_ref();
+    let library_file = library_file.as_ref().file_stem().unwrap();
 
     let rust_file = output_dir
         .join(crate_path.file_stem().unwrap())
         .with_extension("rs");
     let binary_path = output_dir.join(rust_file.file_stem().unwrap());
 
+    // Create a file that contains the Rust 'bindings' as well as the generated test code.
     File::create(&rust_file)?.write_all(
         format!(
             "include!(r#\"{}\"#);\ninclude!(r#\"{}.rs\"#);",
             canonicalize(crate_path)?.display(),
-            library_file.display()
+            library_file.to_str().unwrap()
         )
         .as_bytes(),
     )?;
 
+    // Compile the test file with the compiled C library file found in `output_dir`
+    // into a binary file, ignoring all warnings about unused items. (not all items
+    // are currently tested)
+
     let mut cmd = Command::new(rustc);
     cmd.arg(&rust_file)
         .arg(format!("-Lnative={}", output_dir.display()))
-        .arg(format!(
-            "-lstatic={}",
-            library_file.file_stem().unwrap().to_str().unwrap()
-        ))
-        .arg("--target")
-        .arg(env::var("TARGET_PLATFORM").unwrap())
+        .arg(format!("-lstatic={}", library_file.to_str().unwrap()))
+        .arg("--edition")
+        .arg("2021") // Defaults to 2015.
         .arg("-o")
         .arg(&binary_path)
         .arg("-Aunused");
 
+    // Pass in a different target, linker or flags if set, useful for cross compilation.
+
+    let target = env::var("TARGET_PLATFORM").unwrap_or_default();
+    if !target.is_empty() {
+        cmd.arg("--target").arg(target);
+    }
+
     let linker = env::var("LINKER").unwrap_or_default();
     if !linker.is_empty() {
         cmd.arg(format!("-Clinker={linker}"));
@@ -64,14 +138,17 @@ pub fn compile_test<P: AsRef<Path>>(
 }
 
 /// Executes the compiled test binary and returns its output.
-pub fn run_test(test_binary: &str) -> Result<String> {
+///
+/// If a RUNNER environment variable is present, it will use that to run the binary.
+#[doc(hidden)]
+pub fn __run_test<P: AsRef<Path>>(test_binary: P) -> Result<String> {
     let runner = env::var("RUNNER").unwrap_or_default();
     let output = if runner.is_empty() {
-        Command::new(test_binary).output()?
+        Command::new(test_binary.as_ref()).output()?
     } else {
         let mut args = runner.split_whitespace();
         let mut cmd = Command::new(args.next().unwrap());
-        cmd.args(args).arg(test_binary).output()?
+        cmd.args(args).arg(test_binary.as_ref()).output()?
     };
 
     if !output.status.success() {