printf: accept non-UTF-8 input in FORMAT and ARGUMENT arguments

Other implementations of `printf` permit arbitrary data to be passed
to `printf`. The only restriction is that a null byte terminates
FORMAT and ARGUMENT argument strings (since they are C strings).

The current implementation only accepts FORMAT and ARGUMENT
arguments that are valid UTF-8 (this is being enforced by clap).

This commit removes the UTF-8 validation by switching to OsStr
and OsString.

This allows users to use `printf` to transmit or reformat null-safe
but not UTF-8-safe data, such as text encoded in an 8-bit text
encoding. See the `non_utf_8_input` test for an example (ISO-8859-1
text).

[drinkcat: also squashed in this commit to ease rebase]
Author: Justin Tracey <[email protected]>

uucore, printf: improve non-UTF-8 format arguments

This fixes handling of format arguments, in part by eliminating duplicate
implementations. Utilities with format arguments other than printf will no
longer accept things like "'a" as numbers, etc.

Author: andrewliebenow

src/uu/echo/src/echo.rs

@@ -6,11 +6,11 @@
 use clap::builder::ValueParser;
 use clap::{Arg, ArgAction, Command};
 use std::env;
-use std::ffi::{OsStr, OsString};
+use std::ffi::OsString;
 use std::io::{self, StdoutLock, Write};
-use uucore::error::{UResult, USimpleError};
+use uucore::error::UResult;
 use uucore::format::{FormatChar, OctalParsing, parse_escape_only};
-use uucore::format_usage;
+use uucore::{format_usage, os_str_as_bytes};
 
 use uucore::locale::get_message;
 
@@ -176,10 +176,9 @@ fn execute(
     escaped: bool,
 ) -> UResult<()> {
     for (i, input) in arguments_after_options.into_iter().enumerate() {
-        let Some(bytes) = bytes_from_os_string(input.as_os_str()) else {
-            return Err(USimpleError::new(1, get_message("echo-error-non-utf8")));
-        };
+        let bytes = os_str_as_bytes(&input)?;
 
+        // Don't print a space before the first argument
         if i > 0 {
             stdout_lock.write_all(b" ")?;
         }
@@ -201,19 +200,3 @@ fn execute(
 
     Ok(())
 }
-
-fn bytes_from_os_string(input: &OsStr) -> Option<&[u8]> {
-    #[cfg(target_family = "unix")]
-    {
-        use std::os::unix::ffi::OsStrExt;
-
-        Some(input.as_bytes())
-    }
-
-    #[cfg(not(target_family = "unix"))]
-    {
-        // TODO
-        // Verify that this works correctly on these platforms
-        input.to_str().map(|st| st.as_bytes())
-    }
-}

src/uu/printf/src/printf.rs

@@ -4,6 +4,7 @@
 // file that was distributed with this source code.
 use clap::{Arg, ArgAction, Command};
 use std::collections::HashMap;
+use std::ffi::OsString;
 use std::io::stdout;
 use std::ops::ControlFlow;
 use uucore::error::{UResult, UUsageError};
@@ -18,21 +19,19 @@ mod options {
     pub const FORMAT: &str = "FORMAT";
     pub const ARGUMENT: &str = "ARGUMENT";
 }
+
 #[uucore::main]
 pub fn uumain(args: impl uucore::Args) -> UResult<()> {
     let matches = uu_app().get_matches_from(args);
 
     let format = matches
-        .get_one::<std::ffi::OsString>(options::FORMAT)
+        .get_one::<OsString>(options::FORMAT)
         .ok_or_else(|| UUsageError::new(1, get_message("printf-error-missing-operand")))?;
     let format = os_str_as_bytes(format)?;
 
-    let values: Vec<_> = match matches.get_many::<std::ffi::OsString>(options::ARGUMENT) {
-        // FIXME: use os_str_as_bytes once FormatArgument supports Vec<u8>
+    let values: Vec<_> = match matches.get_many::<OsString>(options::ARGUMENT) {
         Some(s) => s
-            .map(|os_string| {
-                FormatArgument::Unparsed(std::ffi::OsStr::to_string_lossy(os_string).to_string())
-            })
+            .map(|os_string| FormatArgument::Unparsed(os_string.to_owned()))
             .collect(),
         None => vec![],
     };
@@ -62,7 +61,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> {
                 "{}",
                 get_message_with_args(
                     "printf-warning-ignoring-excess-arguments",
-                    HashMap::from([("arg".to_string(), arg_str.to_string())])
+                    HashMap::from([("arg".to_string(), arg_str.to_string_lossy().to_string())])
                 )
             );
         }
@@ -103,10 +102,10 @@ pub fn uu_app() -> Command {
                 .help(get_message("printf-help-version"))
                 .action(ArgAction::Version),
         )
-        .arg(Arg::new(options::FORMAT).value_parser(clap::value_parser!(std::ffi::OsString)))
+        .arg(Arg::new(options::FORMAT).value_parser(clap::value_parser!(OsString)))
         .arg(
             Arg::new(options::ARGUMENT)
                 .action(ArgAction::Append)
-                .value_parser(clap::value_parser!(std::ffi::OsString)),
+                .value_parser(clap::value_parser!(OsString)),
         )
 }

src/uucore/src/lib/features/checksum.rs

@@ -968,7 +968,7 @@ fn process_checksum_line(
     cached_line_format: &mut Option<LineFormat>,
     last_algo: &mut Option<String>,
 ) -> Result<(), LineCheckError> {
-    let line_bytes = os_str_as_bytes(line)?;
+    let line_bytes = os_str_as_bytes(line).map_err(|e| LineCheckError::UError(Box::new(e)))?;
 
     // Early return on empty or commented lines.
     if line.is_empty() || line_bytes.starts_with(b"#") {

src/uucore/src/lib/features/extendedbigdecimal.rs

@@ -101,6 +101,18 @@ impl From<f64> for ExtendedBigDecimal {
     }
 }
 
+impl From<u8> for ExtendedBigDecimal {
+    fn from(val: u8) -> Self {
+        Self::BigDecimal(val.into())
+    }
+}
+
+impl From<u32> for ExtendedBigDecimal {
+    fn from(val: u32) -> Self {
+        Self::BigDecimal(val.into())
+    }
+}
+
 impl ExtendedBigDecimal {
     pub fn zero() -> Self {
         Self::BigDecimal(0.into())