Merge branch 'master' of github.com:lycheeverse/lychee into local-files

Cargo.lock

@@ -2231,9 +2231,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.64"
+version = "1.0.67"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79"
+checksum = "a7f9e390c27c3c0ce8bc5d725f6e4d30a29d26659494aa4b17535f7522c5c950"
 dependencies = [
  "itoa",
  "ryu",

README.md

@@ -180,6 +180,8 @@ USAGE:
     lychee [FLAGS] [OPTIONS] [--] [inputs]...
 
 FLAGS:
+        --dump                   Don't perform any link checking. Instead, dump all the links extracted from inputs that
+                                 would be checked
     -E, --exclude-all-private    Exclude all private IPs from checking.
                                  Equivalent to `--exclude-private --exclude-link-local --exclude-loopback`
         --exclude-link-local     Exclude link-local IP address range from checking

lychee-bin/Cargo.toml

@@ -34,7 +34,7 @@ reqwest = { version = "0.11.3", features = ["gzip"] }
 # https://github.com/Homebrew/homebrew-core/pull/70216
 ring = "0.16.20"
 serde = { version = "1.0.125", features = ["derive"] }
-serde_json = "1.0.64"
+serde_json = "1.0.67"
 structopt = "0.3.21"
 tokio = { version = "1.6.0", features = ["full"] }
 toml = "0.5.8"

lychee-bin/src/main.rs

@@ -209,6 +209,15 @@ async fn run(cfg: &Config, inputs: Vec<Input>) -> Result<i32> {
         .await
         .map_err(|e| anyhow!(e))?;
 
+    if cfg.dump {
+        for link in links {
+            if !client.filtered(&link.uri) {
+                println!("{}", link);
+            }
+        }
+        return Ok(ExitCode::Success as i32);
+    }
+
     let pb = if cfg.no_progress {
         None
     } else {

lychee-bin/src/options.rs

@@ -127,6 +127,12 @@ pub(crate) struct Config {
     #[serde(default)]
     pub(crate) no_progress: bool,
 
+    /// Don't perform any link checking.
+    /// Instead, dump all the links extracted from inputs that would be checked
+    #[structopt(long)]
+    #[serde(default)]
+    pub(crate) dump: bool,
+
     /// Maximum number of allowed redirects
     #[structopt(short, long, default_value = &MAX_REDIRECTS_STR)]
     #[serde(default = "max_redirects")]

lychee-lib/src/client.rs

@@ -118,7 +118,7 @@ impl ClientBuilder {
             exclude_private_ips: self.exclude_all_private || self.exclude_private_ips,
             exclude_link_local_ips: self.exclude_all_private || self.exclude_link_local_ips,
             exclude_loopback_ips: self.exclude_all_private || self.exclude_loopback_ips,
-            exclude_mail: self.exclude_all_private || self.exclude_mail,
+            exclude_mail: self.exclude_mail,
         }
     }
 
@@ -199,6 +199,11 @@ impl Client {
         Ok(Response::new(uri, status, source))
     }
 
+    /// Check if the given URI is filtered by the client
+    pub fn filtered(&self, uri: &Uri) -> bool {
+        self.filter.is_excluded(uri)
+    }
+
     pub async fn check_website(&self, uri: &Uri) -> Status {
         let mut retries: i64 = 3;
         let mut wait: u64 = 1;
@@ -288,6 +293,7 @@ where
 #[cfg(test)]
 mod test {
     use std::{
+        convert::TryInto,
         fs::File,
         time::{Duration, Instant},
     };
@@ -297,7 +303,7 @@ mod test {
     use tempfile::tempdir;
 
     use super::ClientBuilder;
-    use crate::{mock_server, test_utils::get_mock_client_response};
+    use crate::{mock_server, test_utils::get_mock_client_response, Uri};
 
     #[tokio::test]
     async fn test_nonexistent() {
@@ -406,6 +412,29 @@ mod test {
         assert!(res.status().is_success());
     }
 
+    #[tokio::test]
+    async fn test_exclude_mail() {
+        let client = ClientBuilder::builder()
+            .exclude_mail(false)
+            .exclude_all_private(true)
+            .build()
+            .client()
+            .unwrap();
+        assert!(!client.filtered(&Uri {
+            url: "mailto://mail@example.org".try_into().unwrap()
+        }));
+
+        let client = ClientBuilder::builder()
+            .exclude_mail(true)
+            .exclude_all_private(true)
+            .build()
+            .client()
+            .unwrap();
+        assert!(client.filtered(&Uri {
+            url: "mailto://mail@example.org".try_into().unwrap()
+        }));
+    }
+
     #[tokio::test]
     async fn test_require_https() {
         let client = ClientBuilder::builder().build().client().unwrap();