From a3396535a9f7d13d6c36d475532aa92a4d03d619 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81rni=20Dagur?= Date: Thu, 13 Sep 2018 23:46:19 +0000 Subject: [PATCH 1/6] Add rudimentary splicing for cat on Linux --- Cargo.lock | 1 + src/cat/Cargo.toml | 3 +++ src/cat/cat.rs | 65 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 6ac2fcaea8e..5dc2e6c2890 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -147,6 +147,7 @@ dependencies = [ name = "cat" version = "0.0.1" dependencies = [ + "nix 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "unix_socket 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", "uucore 0.0.1", diff --git a/src/cat/Cargo.toml b/src/cat/Cargo.toml index 584fbdbedf6..ebdce97e629 100644 --- a/src/cat/Cargo.toml +++ b/src/cat/Cargo.toml @@ -18,6 +18,9 @@ features = ["fs"] [target.'cfg(unix)'.dependencies] unix_socket = "0.5.0" +[target.'cfg(any(target_os = "linux", target_os = "android"))'.dependencies] +nix = "0.9.0" + [[bin]] name = "cat" path = "../../uumain.rs" diff --git a/src/cat/cat.rs b/src/cat/cat.rs index 5885a7c0f87..1e3cc965da0 100644 --- a/src/cat/cat.rs +++ b/src/cat/cat.rs @@ -16,6 +16,8 @@ extern crate quick_error; extern crate unix_socket; #[macro_use] extern crate uucore; +#[cfg(any(target_os = "linux", target_os = "android"))] +extern crate nix; // last synced with: cat (GNU coreutils) 8.13 use quick_error::ResultExt; @@ -31,6 +33,14 @@ use std::os::unix::fs::FileTypeExt; #[cfg(unix)] use unix_socket::UnixStream; +/// Linux splice support +#[cfg(any(target_os = "linux", target_os = "android"))] +use nix::fcntl::{splice, SpliceFFlags}; +#[cfg(any(target_os = "linux", target_os = "android"))] +use nix::unistd::pipe; +#[cfg(any(target_os = "linux", target_os = "android"))] +use std::os::unix::io::{AsRawFd, RawFd}; + static SYNTAX: &str = "[OPTION]... [FILE]..."; static SUMMARY: &str = "Concatenate FILE(s), or standard input, to standard output With no FILE, or when FILE is -, read standard input."; @@ -100,6 +110,8 @@ struct OutputOptions { /// Represents an open file handle, stream, or other device struct InputHandle { + #[cfg(any(target_os = "linux", target_os = "android"))] + file_descriptor: RawFd, reader: Box, is_interactive: bool, } @@ -241,6 +253,8 @@ fn open(path: &str) -> CatResult { if path == "-" { let stdin = stdin(); return Ok(InputHandle { + #[cfg(any(target_os = "linux", target_os = "android"))] + file_descriptor: stdin.as_raw_fd(), reader: Box::new(stdin) as Box, is_interactive: is_stdin_interactive(), }); @@ -253,6 +267,8 @@ fn open(path: &str) -> CatResult { let socket = UnixStream::connect(path).context(path)?; socket.shutdown(Shutdown::Write).context(path)?; Ok(InputHandle { + #[cfg(any(target_os = "linux", target_os = "android"))] + file_descriptor: socket.as_raw_fd(), reader: Box::new(socket) as Box, is_interactive: false, }) @@ -260,6 +276,8 @@ fn open(path: &str) -> CatResult { _ => { let file = File::open(path).context(path)?; Ok(InputHandle { + #[cfg(any(target_os = "linux", target_os = "android"))] + file_descriptor: file.as_raw_fd(), reader: Box::new(file) as Box, is_interactive: false, }) @@ -275,6 +293,7 @@ fn open(path: &str) -> CatResult { /// /// * `files` - There is no short circuit when encountiner an error /// reading a file in this vector +#[cfg(not(any(target_os = "linux", target_os = "android")))] fn write_fast(files: Vec) -> CatResult<()> { let mut writer = stdout(); let mut in_buf = [0; 1024 * 64]; @@ -300,6 +319,52 @@ fn write_fast(files: Vec) -> CatResult<()> { _ => Err(CatError::EncounteredErrors(error_count)), } } +#[cfg(any(target_os = "linux", target_os = "android"))] +fn write_fast(files: Vec) -> CatResult<()> { + const BUF_SIZE: usize = 1024 * 16; + let writer = stdout(); + // let _handle = writer.lock(); + let (pipe_rd, pipe_wr) = pipe().unwrap(); + let mut error_count = 0; + + for file in files { + match open(&file[..]) { + Ok(handle) => loop { + let res = splice( + handle.file_descriptor, + None, + pipe_wr, + None, + BUF_SIZE, + SpliceFFlags::empty(), + ).unwrap(); + if res == 0 { + // We read 0 bytes from the input, + // which means we're done copying. + break; + } + let _res = splice ( + pipe_rd, + None, + writer.as_raw_fd(), + None, + BUF_SIZE, + SpliceFFlags::empty(), + ).unwrap(); + }, + Err(error) => { + writeln!(&mut stderr(), "{}", error)?; + error_count += 1; + } + + } + } + match error_count { + 0 => Ok(()), + _ => Err(CatError::EncounteredErrors(error_count)), + } +} + /// State that persists between output of each file struct OutputState { From bf6386e7382f9f376eb01909a1a4aa4c9b3ec089 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81rni=20Dagur?= Date: Sat, 11 May 2019 19:10:16 +0000 Subject: [PATCH 2/6] Do proper error handling --- Cargo.lock | 2 + src/cat/cat.rs | 153 +++++++++++++++++++++++++++---------------------- 2 files changed, 86 insertions(+), 69 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5dc2e6c2890..6d95aac9924 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,3 +1,5 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. [[package]] name = "advapi32-sys" version = "0.2.0" diff --git a/src/cat/cat.rs b/src/cat/cat.rs index 1e3cc965da0..feed1167120 100644 --- a/src/cat/cat.rs +++ b/src/cat/cat.rs @@ -34,12 +34,17 @@ use std::os::unix::fs::FileTypeExt; use unix_socket::UnixStream; /// Linux splice support +use nix::errno::Errno::EPIPE; #[cfg(any(target_os = "linux", target_os = "android"))] use nix::fcntl::{splice, SpliceFFlags}; #[cfg(any(target_os = "linux", target_os = "android"))] use nix::unistd::pipe; #[cfg(any(target_os = "linux", target_os = "android"))] +use nix::Error::Sys; +#[cfg(any(target_os = "linux", target_os = "android"))] use std::os::unix::io::{AsRawFd, RawFd}; +#[cfg(any(target_os = "linux", target_os = "android"))] +use std::process::exit; static SYNTAX: &str = "[OPTION]... [FILE]..."; static SUMMARY: &str = "Concatenate FILE(s), or standard input, to standard output @@ -180,7 +185,10 @@ pub fn uumain(args: Vec) -> i32 { files.push("-".to_owned()); } - let can_write_fast = !(show_tabs || show_nonprint || show_ends || squeeze_blank + let can_write_fast = !(show_tabs + || show_nonprint + || show_ends + || squeeze_blank || number_mode != NumberingMode::NumberNone); let success = if can_write_fast { @@ -202,7 +210,11 @@ pub fn uumain(args: Vec) -> i32 { write_lines(files, &options).is_ok() }; - if success { 0 } else { 1 } + if success { + 0 + } else { + 1 + } } /// Classifies the `InputType` of file at `path` if possible @@ -217,25 +229,13 @@ fn get_input_type(path: &str) -> CatResult { match metadata(path).context(path)?.file_type() { #[cfg(unix)] - ft if ft.is_block_device() => - { - Ok(InputType::BlockDevice) - } + ft if ft.is_block_device() => Ok(InputType::BlockDevice), #[cfg(unix)] - ft if ft.is_char_device() => - { - Ok(InputType::CharacterDevice) - } + ft if ft.is_char_device() => Ok(InputType::CharacterDevice), #[cfg(unix)] - ft if ft.is_fifo() => - { - Ok(InputType::Fifo) - } + ft if ft.is_fifo() => Ok(InputType::Fifo), #[cfg(unix)] - ft if ft.is_socket() => - { - Ok(InputType::Socket) - } + ft if ft.is_socket() => Ok(InputType::Socket), ft if ft.is_dir() => Ok(InputType::Directory), ft if ft.is_file() => Ok(InputType::File), ft if ft.is_symlink() => Ok(InputType::SymLink), @@ -293,20 +293,38 @@ fn open(path: &str) -> CatResult { /// /// * `files` - There is no short circuit when encountiner an error /// reading a file in this vector -#[cfg(not(any(target_os = "linux", target_os = "android")))] fn write_fast(files: Vec) -> CatResult<()> { - let mut writer = stdout(); let mut in_buf = [0; 1024 * 64]; let mut error_count = 0; for file in files { match open(&file[..]) { - Ok(mut handle) => while let Ok(n) = handle.reader.read(&mut in_buf) { - if n == 0 { - break; + Ok(mut handle) => { + // If we're on Linux or Android, try to use the splice() + // system call for faster writing. + #[cfg(any(target_os = "linux", target_os = "android"))] + { + match write_fast_with_splice(&mut handle) { + Err(Sys(err)) => match err { + EPIPE => { + // Our pipe was interrupted, this happens, for example, when + // the shell command `cat my_file.txt | head` is run. We exit + // gracefully. + exit(0); + } + _ => {} + }, + _ => { + // Writing fast with splice worked! We don't need + // to fall back on slower writing. + continue; + } + } } - writer.write_all(&in_buf[..n]).context(&file[..])?; - }, + // If we're not on Linux or Android, or the splice() call failed, + // fall back on slower writing. + write_fast_without_splice(&mut handle, &mut in_buf)? + } Err(error) => { writeln!(&mut stderr(), "{}", error)?; error_count += 1; @@ -319,52 +337,51 @@ fn write_fast(files: Vec) -> CatResult<()> { _ => Err(CatError::EncounteredErrors(error_count)), } } -#[cfg(any(target_os = "linux", target_os = "android"))] -fn write_fast(files: Vec) -> CatResult<()> { + +#[inline] +fn write_fast_without_splice(handle: &mut InputHandle, in_buf: &mut [u8]) -> Result<(), io::Error> { + let mut writer = stdout(); + while let Ok(n) = handle.reader.read(in_buf) { + if n == 0 { + break; + } + writer.write_all(&in_buf[..n])?; + } + Ok(()) +} + +fn write_fast_with_splice(handle: &mut InputHandle) -> Result<(), nix::Error> { const BUF_SIZE: usize = 1024 * 16; + let writer = stdout(); - // let _handle = writer.lock(); let (pipe_rd, pipe_wr) = pipe().unwrap(); - let mut error_count = 0; - - for file in files { - match open(&file[..]) { - Ok(handle) => loop { - let res = splice( - handle.file_descriptor, - None, - pipe_wr, - None, - BUF_SIZE, - SpliceFFlags::empty(), - ).unwrap(); - if res == 0 { - // We read 0 bytes from the input, - // which means we're done copying. - break; - } - let _res = splice ( - pipe_rd, - None, - writer.as_raw_fd(), - None, - BUF_SIZE, - SpliceFFlags::empty(), - ).unwrap(); - }, - Err(error) => { - writeln!(&mut stderr(), "{}", error)?; - error_count += 1; - } + loop { + let res = splice( + handle.file_descriptor, + None, + pipe_wr, + None, + BUF_SIZE, + SpliceFFlags::empty(), + )?; + if res == 0 { + // We read 0 bytes from the input, + // which means we're done copying. + break; } + let _res = splice( + pipe_rd, + None, + writer.as_raw_fd(), + None, + BUF_SIZE, + SpliceFFlags::empty(), + )?; } - match error_count { - 0 => Ok(()), - _ => Err(CatError::EncounteredErrors(error_count)), - } -} + Ok(()) +} /// State that persists between output of each file struct OutputState { @@ -485,10 +502,7 @@ fn write_to_end(in_buf: &[u8], writer: &mut W) -> usize { fn write_tab_to_end(mut in_buf: &[u8], writer: &mut W) -> usize { loop { - match in_buf - .iter() - .position(|c| *c == b'\n' || *c == b'\t') - { + match in_buf.iter().position(|c| *c == b'\n' || *c == b'\t') { Some(p) => { writer.write_all(&in_buf[..p]).unwrap(); if in_buf[p] == b'\n' { @@ -521,7 +535,8 @@ fn write_nonprint_to_end(in_buf: &[u8], writer: &mut W, tab: &[u8]) -> 128...159 => writer.write_all(&[b'M', b'-', b'^', byte - 64]), 160...254 => writer.write_all(&[b'M', b'-', byte - 128]), _ => writer.write_all(&[b'M', b'-', b'^', 63]), - }.unwrap(); + } + .unwrap(); count += 1; } if count != in_buf.len() { From 9e7b8eb005d125b8dddddcdbe825db1544281761 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81rni=20Dagur?= Date: Sun, 12 May 2019 22:43:37 +0000 Subject: [PATCH 3/6] Compile write_fast_with_splice() only on Linux and Android --- src/cat/cat.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cat/cat.rs b/src/cat/cat.rs index feed1167120..ac317a6eb59 100644 --- a/src/cat/cat.rs +++ b/src/cat/cat.rs @@ -350,6 +350,7 @@ fn write_fast_without_splice(handle: &mut InputHandle, in_buf: &mut [u8]) -> Res Ok(()) } +#[cfg(any(target_os = "linux", target_os = "android"))] fn write_fast_with_splice(handle: &mut InputHandle) -> Result<(), nix::Error> { const BUF_SIZE: usize = 1024 * 16; From 6db67e82c16fd5d837d906c78c9ceb0bb4788cc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81rni=20Dagur?= Date: Sat, 18 May 2019 12:49:10 +0000 Subject: [PATCH 4/6] Address some of Arcterus's concerns --- src/cat/cat.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/cat/cat.rs b/src/cat/cat.rs index ac317a6eb59..c13621a2473 100644 --- a/src/cat/cat.rs +++ b/src/cat/cat.rs @@ -304,7 +304,7 @@ fn write_fast(files: Vec) -> CatResult<()> { // system call for faster writing. #[cfg(any(target_os = "linux", target_os = "android"))] { - match write_fast_with_splice(&mut handle) { + match write_fast_using_splice(&mut handle) { Err(Sys(err)) => match err { EPIPE => { // Our pipe was interrupted, this happens, for example, when @@ -323,7 +323,7 @@ fn write_fast(files: Vec) -> CatResult<()> { } // If we're not on Linux or Android, or the splice() call failed, // fall back on slower writing. - write_fast_without_splice(&mut handle, &mut in_buf)? + write_fast_using_read_and_write(&mut handle, &mut in_buf)? } Err(error) => { writeln!(&mut stderr(), "{}", error)?; @@ -339,7 +339,10 @@ fn write_fast(files: Vec) -> CatResult<()> { } #[inline] -fn write_fast_without_splice(handle: &mut InputHandle, in_buf: &mut [u8]) -> Result<(), io::Error> { +fn write_fast_using_read_and_write( + handle: &mut InputHandle, + in_buf: &mut [u8], +) -> Result<(), io::Error> { let mut writer = stdout(); while let Ok(n) = handle.reader.read(in_buf) { if n == 0 { @@ -351,11 +354,11 @@ fn write_fast_without_splice(handle: &mut InputHandle, in_buf: &mut [u8]) -> Res } #[cfg(any(target_os = "linux", target_os = "android"))] -fn write_fast_with_splice(handle: &mut InputHandle) -> Result<(), nix::Error> { +fn write_fast_using_splice(handle: &mut InputHandle) -> Result<(), nix::Error> { const BUF_SIZE: usize = 1024 * 16; let writer = stdout(); - let (pipe_rd, pipe_wr) = pipe().unwrap(); + let (pipe_rd, pipe_wr) = pipe()?; loop { let res = splice( From f1595468df074b5764980c77be0419d7470bc134 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81rni=20Dagur?= Date: Sun, 19 May 2019 15:07:42 +0000 Subject: [PATCH 5/6] Simplify code a whole bunch * Use `io::copy` as `write_fast` fallback. * Document `write_fast_using_splice` with Rustdoc comment. * Inline `write_fast_using_splice`. * Ignore all `splice` related errors. --- src/cat/cat.rs | 59 ++++++++++++++------------------------------------ 1 file changed, 16 insertions(+), 43 deletions(-) diff --git a/src/cat/cat.rs b/src/cat/cat.rs index c13621a2473..e4ef2c0a0f7 100644 --- a/src/cat/cat.rs +++ b/src/cat/cat.rs @@ -34,17 +34,12 @@ use std::os::unix::fs::FileTypeExt; use unix_socket::UnixStream; /// Linux splice support -use nix::errno::Errno::EPIPE; #[cfg(any(target_os = "linux", target_os = "android"))] use nix::fcntl::{splice, SpliceFFlags}; #[cfg(any(target_os = "linux", target_os = "android"))] use nix::unistd::pipe; #[cfg(any(target_os = "linux", target_os = "android"))] -use nix::Error::Sys; -#[cfg(any(target_os = "linux", target_os = "android"))] use std::os::unix::io::{AsRawFd, RawFd}; -#[cfg(any(target_os = "linux", target_os = "android"))] -use std::process::exit; static SYNTAX: &str = "[OPTION]... [FILE]..."; static SUMMARY: &str = "Concatenate FILE(s), or standard input, to standard output @@ -294,8 +289,9 @@ fn open(path: &str) -> CatResult { /// * `files` - There is no short circuit when encountiner an error /// reading a file in this vector fn write_fast(files: Vec) -> CatResult<()> { - let mut in_buf = [0; 1024 * 64]; let mut error_count = 0; + let writer = stdout(); + let mut writer_handle = writer.lock(); for file in files { match open(&file[..]) { @@ -304,26 +300,21 @@ fn write_fast(files: Vec) -> CatResult<()> { // system call for faster writing. #[cfg(any(target_os = "linux", target_os = "android"))] { - match write_fast_using_splice(&mut handle) { - Err(Sys(err)) => match err { - EPIPE => { - // Our pipe was interrupted, this happens, for example, when - // the shell command `cat my_file.txt | head` is run. We exit - // gracefully. - exit(0); - } - _ => {} - }, - _ => { + match write_fast_using_splice(&mut handle, writer.as_raw_fd()) { + Ok(_) => { // Writing fast with splice worked! We don't need // to fall back on slower writing. continue; } + Err(_) => { + // Ignore any error and fall back to slower + // writing below. + } } } // If we're not on Linux or Android, or the splice() call failed, // fall back on slower writing. - write_fast_using_read_and_write(&mut handle, &mut in_buf)? + io::copy(&mut handle.reader, &mut writer_handle).unwrap(); } Err(error) => { writeln!(&mut stderr(), "{}", error)?; @@ -338,26 +329,15 @@ fn write_fast(files: Vec) -> CatResult<()> { } } -#[inline] -fn write_fast_using_read_and_write( - handle: &mut InputHandle, - in_buf: &mut [u8], -) -> Result<(), io::Error> { - let mut writer = stdout(); - while let Ok(n) = handle.reader.read(in_buf) { - if n == 0 { - break; - } - writer.write_all(&in_buf[..n])?; - } - Ok(()) -} - +/// This function is called from `write_fast()` on Linux and Android. The +/// function `splice()` is used to move data between two file descriptors +/// without copying between kernel- and userspace. This results in a large +/// speedup. #[cfg(any(target_os = "linux", target_os = "android"))] -fn write_fast_using_splice(handle: &mut InputHandle) -> Result<(), nix::Error> { +#[inline] +fn write_fast_using_splice(handle: &mut InputHandle, writer: RawFd) -> Result<(), nix::Error> { const BUF_SIZE: usize = 1024 * 16; - let writer = stdout(); let (pipe_rd, pipe_wr) = pipe()?; loop { @@ -374,14 +354,7 @@ fn write_fast_using_splice(handle: &mut InputHandle) -> Result<(), nix::Error> { // which means we're done copying. break; } - let _res = splice( - pipe_rd, - None, - writer.as_raw_fd(), - None, - BUF_SIZE, - SpliceFFlags::empty(), - )?; + let _ = splice(pipe_rd, None, writer, None, BUF_SIZE, SpliceFFlags::empty())?; } Ok(()) From bec661514119fa22f083282d2cd6c45ca4e41195 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81rni=20Dagur?= Date: Sat, 15 Jun 2019 20:28:24 +0000 Subject: [PATCH 6/6] Do not panic if io::copy fails --- src/cat/cat.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/cat/cat.rs b/src/cat/cat.rs index e4ef2c0a0f7..5aa07e201b1 100644 --- a/src/cat/cat.rs +++ b/src/cat/cat.rs @@ -306,7 +306,7 @@ fn write_fast(files: Vec) -> CatResult<()> { // to fall back on slower writing. continue; } - Err(_) => { + _ => { // Ignore any error and fall back to slower // writing below. } @@ -314,10 +314,16 @@ fn write_fast(files: Vec) -> CatResult<()> { } // If we're not on Linux or Android, or the splice() call failed, // fall back on slower writing. - io::copy(&mut handle.reader, &mut writer_handle).unwrap(); + match io::copy(&mut handle.reader, &mut writer_handle) { + Err(error) => { + eprintln!("error when copying file: {}", error); + error_count += 1; + } + _ => {} + } } Err(error) => { - writeln!(&mut stderr(), "{}", error)?; + eprintln!("error when opening file: {}", error); error_count += 1; } }