Added support for --exclude-file.

README.md

@@ -197,6 +197,7 @@ OPTIONS:
         --basic-auth <basic-auth>              Basic authentication support. E.g. `username:password`
     -c, --config <config-file>                 Configuration file to use [default: ./lychee.toml]
         --exclude <exclude>...                 Exclude URLs from checking (supports regex)
+        --exclude-file <exclude-file>...       A file or files that contains URLs to exclude from checking
     -f, --format <format>                      Output file format of status report (json, string) [default: string]
         --github-token <github-token>          GitHub API token to use when checking github.com links, to avoid rate
                                                limiting [env: GITHUB_TOKEN=]

fixtures/TEST_EXCLUDE_1.txt

@@ -0,0 +1,3 @@
+https://en.wikipedia.org/*
+https://ldra.com
+https://url-does-not-exist

fixtures/TEST_EXCLUDE_2.txt

@@ -0,0 +1 @@
+https://i.creativecommons.org/p/zero/1.0/88x31.png

lychee-bin/src/main.rs

@@ -63,6 +63,8 @@ use ring as _;
 
 use std::iter::FromIterator;
 use std::{collections::HashSet, fs, str::FromStr, time::Duration};
+use std::io::{self, BufRead};
+use std::fs::File;
 
 use anyhow::{anyhow, Context, Result};
 use headers::{authorization::Basic, Authorization, HeaderMap, HeaderMapExt, HeaderName};
@@ -113,6 +115,18 @@ fn run_main() -> Result<i32> {
     if let Some(c) = Config::load_from_file(&opts.config_file)? {
         opts.config.merge(c)
     }
+
+    // Load excludes from file
+    for path in &opts.config.exclude_file {
+        let file = File::open(path).expect("No such file");
+        opts.config.exclude.append(
+            &mut io::BufReader::new(file)
+                .lines()
+                .map(|l| l.expect("Could not read line"))
+                .collect()
+        );
+    }
+
     let cfg = &opts.config;
 
     let runtime = match cfg.threads {

lychee-bin/src/options.rs

@@ -165,6 +165,11 @@ pub(crate) struct Config {
     #[serde(default)]
     pub(crate) exclude: Vec<String>,
 
+    /// A file or files that contains URLs to exclude from checking
+    #[structopt(long)]
+    #[serde(default)]
+    pub(crate) exclude_file: Vec<String>,
+    
     /// Exclude all private IPs from checking.
     /// Equivalent to `--exclude-private --exclude-link-local --exclude-loopback`
     #[structopt(short = "E", long, verbatim_doc_comment)]
@@ -285,6 +290,7 @@ impl Config {
             scheme: Vec::<String>::new();
             include: Vec::<String>::new();
             exclude: Vec::<String>::new();
+            exclude_file: Vec::<String>::new();
             exclude_all_private: false;
             exclude_private: false;
             exclude_link_local: false;

lychee-bin/tests/cli.rs

@@ -370,4 +370,71 @@ mod cli {
         fs::remove_file(outfile)?;
         Ok(())
     }
+
+    /// Test excludes
+    #[test]
+    fn test_exclude_wildcard() -> Result<()> {
+        let mut cmd = main_command();
+        let test_path = fixtures_path().join("TEST.md");
+
+        cmd.arg(test_path)
+            .arg("--exclude")
+            .arg(".*")
+            .assert()
+            .success()
+            .stdout(contains("Excluded........10"));
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_exclude_multiple_urls() -> Result<()> {
+        let mut cmd = main_command();
+        let test_path = fixtures_path().join("TEST.md");
+
+        cmd.arg(test_path)
+            .arg("--exclude")
+            .arg("https://en.wikipedia.org/*")
+            .arg("https://ldra.com/")
+            .assert()
+            .success()
+            .stdout(contains("Excluded.........2"));
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_exclude_file() -> Result<()> {
+        let mut cmd = main_command();
+        let test_path = fixtures_path().join("TEST.md");
+        let excludes_path = fixtures_path().join("TEST_EXCLUDE_1.txt");
+
+        cmd.arg(test_path)
+            .arg("--exclude-file")
+            .arg(excludes_path)
+            .assert()
+            .success()
+            .stdout(contains("Excluded.........2"));
+
+        Ok(())
+    }    
+
+    #[test]
+    fn test_multiple_exclude_files() -> Result<()> {
+        let mut cmd = main_command();
+        let test_path = fixtures_path().join("TEST.md");
+        let excludes_path1 = fixtures_path().join("TEST_EXCLUDE_1.txt");
+        let excludes_path2 = fixtures_path().join("TEST_EXCLUDE_2.txt");
+
+        cmd.arg(test_path)
+            .arg("--exclude-file")
+            .arg(excludes_path1)
+            .arg(excludes_path2)
+            .assert()
+            .success()
+            .stdout(contains("Excluded.........3"));
+
+        Ok(())
+    }    
 }
+

lychee.example.toml

@@ -52,6 +52,9 @@ headers = []
 # Exclude URLs from checking (supports regex)
 exclude = []
 
+# Exclude URLs contained in a file from checking
+exclude_file = []
+
 include = []
 
 # Exclude all private IPs from checking