From e794b40d4d2d0eff574d6410bfb6540ad9a9743d Mon Sep 17 00:00:00 2001 From: Matthias Endler Date: Thu, 7 Nov 2024 16:32:32 +0100 Subject: [PATCH] Support excluded paths in `--dump-inputs` (#1556) --- lychee-bin/src/commands/dump.rs | 121 +++++++++++++++++++++++++++++--- lychee-bin/src/main.rs | 8 ++- lychee-bin/tests/cli.rs | 20 +++++- 3 files changed, 138 insertions(+), 11 deletions(-) diff --git a/lychee-bin/src/commands/dump.rs b/lychee-bin/src/commands/dump.rs index 580cdb9..572c592 100644 --- a/lychee-bin/src/commands/dump.rs +++ b/lychee-bin/src/commands/dump.rs @@ -49,17 +49,17 @@ where // Apply URI remappings (if any) params.client.remap(&mut request.uri)?; - // Avoid panic on broken pipe. - // See https://github.com/rust-lang/rust/issues/46016 - // This can occur when piping the output of lychee - // to another program like `grep`. - let excluded = params.client.is_excluded(&request.uri); if excluded && params.cfg.verbose.log_level() < log::Level::Info { continue; } + if let Err(e) = write(&mut writer, &request, ¶ms.cfg.verbose, excluded) { + // Avoid panic on broken pipe. + // See https://github.com/rust-lang/rust/issues/46016 + // This can occur when piping the output of lychee + // to another program like `grep`. if e.kind() != io::ErrorKind::BrokenPipe { error!("{e}"); return Ok(ExitCode::UnexpectedFailure); @@ -72,22 +72,31 @@ where /// Dump all input sources to stdout without extracting any links and checking /// them. -pub(crate) async fn dump_inputs(sources: S, output: Option<&PathBuf>) -> Result +pub(crate) async fn dump_inputs( + sources: S, + output: Option<&PathBuf>, + excluded_paths: &[PathBuf], +) -> Result where S: futures::Stream>, { - let sources = sources; - tokio::pin!(sources); - if let Some(out_file) = output { fs::File::create(out_file)?; } let mut writer = create_writer(output.cloned())?; + tokio::pin!(sources); while let Some(source) = sources.next().await { let source = source?; + let excluded = excluded_paths + .iter() + .any(|path| source.starts_with(path.to_string_lossy().as_ref())); + if excluded { + continue; + } + writeln!(writer, "{source}")?; } @@ -127,3 +136,97 @@ fn write( fn write_out(writer: &mut Box, out_str: &str) -> io::Result<()> { writeln!(writer, "{out_str}") } + +#[cfg(test)] +mod tests { + use super::*; + use futures::stream; + use tempfile::NamedTempFile; + + #[tokio::test] + async fn test_dump_inputs_basic() -> Result<()> { + // Create temp file for output + let temp_file = NamedTempFile::new()?; + let output_path = temp_file.path().to_path_buf(); + + // Create test input stream + let inputs = vec![ + Ok(String::from("test/path1")), + Ok(String::from("test/path2")), + Ok(String::from("test/path3")), + ]; + let stream = stream::iter(inputs); + + // Run dump_inputs + let result = dump_inputs(stream, Some(&output_path), &[]).await?; + assert_eq!(result, ExitCode::Success); + + // Verify output + let contents = fs::read_to_string(&output_path)?; + assert_eq!(contents, "test/path1\ntest/path2\ntest/path3\n"); + Ok(()) + } + + #[tokio::test] + async fn test_dump_inputs_with_excluded_paths() -> Result<()> { + let temp_file = NamedTempFile::new()?; + let output_path = temp_file.path().to_path_buf(); + + let inputs = vec![ + Ok(String::from("test/path1")), + Ok(String::from("excluded/path")), + Ok(String::from("test/path2")), + ]; + let stream = stream::iter(inputs); + + let excluded = vec![PathBuf::from("excluded")]; + let result = dump_inputs(stream, Some(&output_path), &excluded).await?; + assert_eq!(result, ExitCode::Success); + + let contents = fs::read_to_string(&output_path)?; + assert_eq!(contents, "test/path1\ntest/path2\n"); + Ok(()) + } + + #[tokio::test] + async fn test_dump_inputs_empty_stream() -> Result<()> { + let temp_file = NamedTempFile::new()?; + let output_path = temp_file.path().to_path_buf(); + + let stream = stream::iter::>>(vec![]); + let result = dump_inputs(stream, Some(&output_path), &[]).await?; + assert_eq!(result, ExitCode::Success); + + let contents = fs::read_to_string(&output_path)?; + assert_eq!(contents, ""); + Ok(()) + } + + #[tokio::test] + async fn test_dump_inputs_error_in_stream() -> Result<()> { + let temp_file = NamedTempFile::new()?; + let output_path = temp_file.path().to_path_buf(); + + let inputs: Vec> = vec![ + Ok(String::from("test/path1")), + Err(io::Error::new(io::ErrorKind::Other, "test error").into()), + Ok(String::from("test/path2")), + ]; + let stream = stream::iter(inputs); + + let result = dump_inputs(stream, Some(&output_path), &[]).await; + assert!(result.is_err()); + Ok(()) + } + + #[tokio::test] + async fn test_dump_inputs_to_stdout() -> Result<()> { + // When output path is None, should write to stdout + let inputs = vec![Ok(String::from("test/path1"))]; + let stream = stream::iter(inputs); + + let result = dump_inputs(stream, None, &[]).await?; + assert_eq!(result, ExitCode::Success); + Ok(()) + } +} diff --git a/lychee-bin/src/main.rs b/lychee-bin/src/main.rs index 4fac9ca..cd96c27 100644 --- a/lychee-bin/src/main.rs +++ b/lychee-bin/src/main.rs @@ -99,6 +99,7 @@ use crate::{ }; /// A C-like enum that can be cast to `i32` and used as process exit code. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] enum ExitCode { Success = 0, // NOTE: exit code 1 is used for any `Result::Err` bubbled up to `main()` @@ -297,7 +298,12 @@ async fn run(opts: &LycheeOptions) -> Result { if opts.config.dump_inputs { let sources = collector.collect_sources(inputs); - let exit_code = commands::dump_inputs(sources, opts.config.output.as_ref()).await?; + let exit_code = commands::dump_inputs( + sources, + opts.config.output.as_ref(), + &opts.config.exclude_path, + ) + .await?; return Ok(exit_code as i32); } diff --git a/lychee-bin/tests/cli.rs b/lychee-bin/tests/cli.rs index 0cc5596..ad5cd0d 100644 --- a/lychee-bin/tests/cli.rs +++ b/lychee-bin/tests/cli.rs @@ -14,7 +14,7 @@ mod cli { use http::StatusCode; use lychee_lib::{InputSource, ResponseBody}; use predicates::{ - prelude::predicate, + prelude::{predicate, PredicateBooleanExt}, str::{contains, is_empty}, }; use pretty_assertions::assert_eq; @@ -1653,6 +1653,24 @@ mod cli { Ok(()) } + #[test] + fn test_dump_inputs_glob_exclude_path() -> Result<()> { + let pattern = fixtures_path().join("**/*"); + + let mut cmd = main_command(); + cmd.arg("--dump-inputs") + .arg(pattern) + .arg("--exclude-path") + .arg(fixtures_path().join("dump_inputs/subfolder")) + .assert() + .success() + .stdout(contains("fixtures/dump_inputs/subfolder/test.html").not()) + .stdout(contains("fixtures/dump_inputs/subfolder/file2.md").not()) + .stdout(contains("fixtures/dump_inputs/subfolder").not()); + + Ok(()) + } + #[test] fn test_dump_inputs_url() -> Result<()> { let mut cmd = main_command();