Support excluded paths in --dump-inputs (#1556)

lychee-bin/src/commands/dump.rs

@@ -49,17 +49,17 @@ where
         // Apply URI remappings (if any)
         params.client.remap(&mut request.uri)?;
 
-        // Avoid panic on broken pipe.
-        // See https://github.com/rust-lang/rust/issues/46016
-        // This can occur when piping the output of lychee
-        // to another program like `grep`.
-
         let excluded = params.client.is_excluded(&request.uri);
 
         if excluded && params.cfg.verbose.log_level() < log::Level::Info {
             continue;
         }
+
         if let Err(e) = write(&mut writer, &request, &params.cfg.verbose, excluded) {
+            // Avoid panic on broken pipe.
+            // See https://github.com/rust-lang/rust/issues/46016
+            // This can occur when piping the output of lychee
+            // to another program like `grep`.
             if e.kind() != io::ErrorKind::BrokenPipe {
                 error!("{e}");
                 return Ok(ExitCode::UnexpectedFailure);
@@ -72,22 +72,31 @@ where
 
 /// Dump all input sources to stdout without extracting any links and checking
 /// them.
-pub(crate) async fn dump_inputs<S>(sources: S, output: Option<&PathBuf>) -> Result<ExitCode>
+pub(crate) async fn dump_inputs<S>(
+    sources: S,
+    output: Option<&PathBuf>,
+    excluded_paths: &[PathBuf],
+) -> Result<ExitCode>
 where
     S: futures::Stream<Item = Result<String>>,
 {
-    let sources = sources;
-    tokio::pin!(sources);
-
     if let Some(out_file) = output {
         fs::File::create(out_file)?;
     }
 
     let mut writer = create_writer(output.cloned())?;
 
+    tokio::pin!(sources);
     while let Some(source) = sources.next().await {
         let source = source?;
 
+        let excluded = excluded_paths
+            .iter()
+            .any(|path| source.starts_with(path.to_string_lossy().as_ref()));
+        if excluded {
+            continue;
+        }
+
         writeln!(writer, "{source}")?;
     }
 
@@ -127,3 +136,97 @@ fn write(
 fn write_out(writer: &mut Box<dyn Write>, out_str: &str) -> io::Result<()> {
     writeln!(writer, "{out_str}")
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use futures::stream;
+    use tempfile::NamedTempFile;
+
+    #[tokio::test]
+    async fn test_dump_inputs_basic() -> Result<()> {
+        // Create temp file for output
+        let temp_file = NamedTempFile::new()?;
+        let output_path = temp_file.path().to_path_buf();
+
+        // Create test input stream
+        let inputs = vec![
+            Ok(String::from("test/path1")),
+            Ok(String::from("test/path2")),
+            Ok(String::from("test/path3")),
+        ];
+        let stream = stream::iter(inputs);
+
+        // Run dump_inputs
+        let result = dump_inputs(stream, Some(&output_path), &[]).await?;
+        assert_eq!(result, ExitCode::Success);
+
+        // Verify output
+        let contents = fs::read_to_string(&output_path)?;
+        assert_eq!(contents, "test/path1\ntest/path2\ntest/path3\n");
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_dump_inputs_with_excluded_paths() -> Result<()> {
+        let temp_file = NamedTempFile::new()?;
+        let output_path = temp_file.path().to_path_buf();
+
+        let inputs = vec![
+            Ok(String::from("test/path1")),
+            Ok(String::from("excluded/path")),
+            Ok(String::from("test/path2")),
+        ];
+        let stream = stream::iter(inputs);
+
+        let excluded = vec![PathBuf::from("excluded")];
+        let result = dump_inputs(stream, Some(&output_path), &excluded).await?;
+        assert_eq!(result, ExitCode::Success);
+
+        let contents = fs::read_to_string(&output_path)?;
+        assert_eq!(contents, "test/path1\ntest/path2\n");
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_dump_inputs_empty_stream() -> Result<()> {
+        let temp_file = NamedTempFile::new()?;
+        let output_path = temp_file.path().to_path_buf();
+
+        let stream = stream::iter::<Vec<Result<String>>>(vec![]);
+        let result = dump_inputs(stream, Some(&output_path), &[]).await?;
+        assert_eq!(result, ExitCode::Success);
+
+        let contents = fs::read_to_string(&output_path)?;
+        assert_eq!(contents, "");
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_dump_inputs_error_in_stream() -> Result<()> {
+        let temp_file = NamedTempFile::new()?;
+        let output_path = temp_file.path().to_path_buf();
+
+        let inputs: Vec<Result<String>> = vec![
+            Ok(String::from("test/path1")),
+            Err(io::Error::new(io::ErrorKind::Other, "test error").into()),
+            Ok(String::from("test/path2")),
+        ];
+        let stream = stream::iter(inputs);
+
+        let result = dump_inputs(stream, Some(&output_path), &[]).await;
+        assert!(result.is_err());
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_dump_inputs_to_stdout() -> Result<()> {
+        // When output path is None, should write to stdout
+        let inputs = vec![Ok(String::from("test/path1"))];
+        let stream = stream::iter(inputs);
+
+        let result = dump_inputs(stream, None, &[]).await?;
+        assert_eq!(result, ExitCode::Success);
+        Ok(())
+    }
+}

lychee-bin/src/main.rs

@@ -99,6 +99,7 @@ use crate::{
 };
 
 /// A C-like enum that can be cast to `i32` and used as process exit code.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
 enum ExitCode {
     Success = 0,
     // NOTE: exit code 1 is used for any `Result::Err` bubbled up to `main()`
@@ -297,7 +298,12 @@ async fn run(opts: &LycheeOptions) -> Result<i32> {
 
     if opts.config.dump_inputs {
         let sources = collector.collect_sources(inputs);
-        let exit_code = commands::dump_inputs(sources, opts.config.output.as_ref()).await?;
+        let exit_code = commands::dump_inputs(
+            sources,
+            opts.config.output.as_ref(),
+            &opts.config.exclude_path,
+        )
+        .await?;
 
         return Ok(exit_code as i32);
     }

lychee-bin/tests/cli.rs

@@ -14,7 +14,7 @@ mod cli {
     use http::StatusCode;
     use lychee_lib::{InputSource, ResponseBody};
     use predicates::{
-        prelude::predicate,
+        prelude::{predicate, PredicateBooleanExt},
         str::{contains, is_empty},
     };
     use pretty_assertions::assert_eq;
@@ -1653,6 +1653,24 @@ mod cli {
         Ok(())
     }
 
+    #[test]
+    fn test_dump_inputs_glob_exclude_path() -> Result<()> {
+        let pattern = fixtures_path().join("**/*");
+
+        let mut cmd = main_command();
+        cmd.arg("--dump-inputs")
+            .arg(pattern)
+            .arg("--exclude-path")
+            .arg(fixtures_path().join("dump_inputs/subfolder"))
+            .assert()
+            .success()
+            .stdout(contains("fixtures/dump_inputs/subfolder/test.html").not())
+            .stdout(contains("fixtures/dump_inputs/subfolder/file2.md").not())
+            .stdout(contains("fixtures/dump_inputs/subfolder").not());
+
+        Ok(())
+    }
+
     #[test]
     fn test_dump_inputs_url() -> Result<()> {
         let mut cmd = main_command();