diff --git a/Cargo.lock b/Cargo.lock index a71f591..e8cbd95 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2231,9 +2231,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.64" +version = "1.0.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79" +checksum = "a7f9e390c27c3c0ce8bc5d725f6e4d30a29d26659494aa4b17535f7522c5c950" dependencies = [ "itoa", "ryu", diff --git a/README.md b/README.md index 99afbd5..d36875a 100644 --- a/README.md +++ b/README.md @@ -180,6 +180,8 @@ USAGE: lychee [FLAGS] [OPTIONS] [--] [inputs]... FLAGS: + --dump Don't perform any link checking. Instead, dump all the links extracted from inputs that + would be checked -E, --exclude-all-private Exclude all private IPs from checking. Equivalent to `--exclude-private --exclude-link-local --exclude-loopback` --exclude-link-local Exclude link-local IP address range from checking diff --git a/lychee-bin/Cargo.toml b/lychee-bin/Cargo.toml index 0027bcd..83aa4bc 100644 --- a/lychee-bin/Cargo.toml +++ b/lychee-bin/Cargo.toml @@ -34,7 +34,7 @@ reqwest = { version = "0.11.3", features = ["gzip"] } # https://github.com/Homebrew/homebrew-core/pull/70216 ring = "0.16.20" serde = { version = "1.0.125", features = ["derive"] } -serde_json = "1.0.64" +serde_json = "1.0.67" structopt = "0.3.21" tokio = { version = "1.6.0", features = ["full"] } toml = "0.5.8" diff --git a/lychee-bin/src/main.rs b/lychee-bin/src/main.rs index f8ba1bc..6c115e3 100644 --- a/lychee-bin/src/main.rs +++ b/lychee-bin/src/main.rs @@ -209,6 +209,15 @@ async fn run(cfg: &Config, inputs: Vec) -> Result { .await .map_err(|e| anyhow!(e))?; + if cfg.dump { + for link in links { + if !client.filtered(&link.uri) { + println!("{}", link); + } + } + return Ok(ExitCode::Success as i32); + } + let pb = if cfg.no_progress { None } else { diff --git a/lychee-bin/src/options.rs b/lychee-bin/src/options.rs index c0ef15d..547e2bb 100644 --- a/lychee-bin/src/options.rs +++ b/lychee-bin/src/options.rs @@ -127,6 +127,12 @@ pub(crate) struct Config { #[serde(default)] pub(crate) no_progress: bool, + /// Don't perform any link checking. + /// Instead, dump all the links extracted from inputs that would be checked + #[structopt(long)] + #[serde(default)] + pub(crate) dump: bool, + /// Maximum number of allowed redirects #[structopt(short, long, default_value = &MAX_REDIRECTS_STR)] #[serde(default = "max_redirects")] diff --git a/lychee-lib/src/client.rs b/lychee-lib/src/client.rs index 8fdb0a1..0e43df7 100644 --- a/lychee-lib/src/client.rs +++ b/lychee-lib/src/client.rs @@ -118,7 +118,7 @@ impl ClientBuilder { exclude_private_ips: self.exclude_all_private || self.exclude_private_ips, exclude_link_local_ips: self.exclude_all_private || self.exclude_link_local_ips, exclude_loopback_ips: self.exclude_all_private || self.exclude_loopback_ips, - exclude_mail: self.exclude_all_private || self.exclude_mail, + exclude_mail: self.exclude_mail, } } @@ -199,6 +199,11 @@ impl Client { Ok(Response::new(uri, status, source)) } + /// Check if the given URI is filtered by the client + pub fn filtered(&self, uri: &Uri) -> bool { + self.filter.is_excluded(uri) + } + pub async fn check_website(&self, uri: &Uri) -> Status { let mut retries: i64 = 3; let mut wait: u64 = 1; @@ -288,6 +293,7 @@ where #[cfg(test)] mod test { use std::{ + convert::TryInto, fs::File, time::{Duration, Instant}, }; @@ -297,7 +303,7 @@ mod test { use tempfile::tempdir; use super::ClientBuilder; - use crate::{mock_server, test_utils::get_mock_client_response}; + use crate::{mock_server, test_utils::get_mock_client_response, Uri}; #[tokio::test] async fn test_nonexistent() { @@ -406,6 +412,29 @@ mod test { assert!(res.status().is_success()); } + #[tokio::test] + async fn test_exclude_mail() { + let client = ClientBuilder::builder() + .exclude_mail(false) + .exclude_all_private(true) + .build() + .client() + .unwrap(); + assert!(!client.filtered(&Uri { + url: "mailto://mail@example.org".try_into().unwrap() + })); + + let client = ClientBuilder::builder() + .exclude_mail(true) + .exclude_all_private(true) + .build() + .client() + .unwrap(); + assert!(client.filtered(&Uri { + url: "mailto://mail@example.org".try_into().unwrap() + })); + } + #[tokio::test] async fn test_require_https() { let client = ClientBuilder::builder().build().client().unwrap();