From bcc06ae4396d8eae7142358c16df76701334112f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Sep 2021 16:08:51 +0200 Subject: [PATCH 1/3] Bump serde_json from 1.0.64 to 1.0.67 (#310) Bumps [serde_json](https://github.com/serde-rs/json) from 1.0.64 to 1.0.67. - [Release notes](https://github.com/serde-rs/json/releases) - [Commits](https://github.com/serde-rs/json/compare/v1.0.64...v1.0.67) --- updated-dependencies: - dependency-name: serde_json dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- lychee-bin/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e6e8efe..154f50f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2223,9 +2223,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.64" +version = "1.0.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79" +checksum = "a7f9e390c27c3c0ce8bc5d725f6e4d30a29d26659494aa4b17535f7522c5c950" dependencies = [ "itoa", "ryu", diff --git a/lychee-bin/Cargo.toml b/lychee-bin/Cargo.toml index 0027bcd..83aa4bc 100644 --- a/lychee-bin/Cargo.toml +++ b/lychee-bin/Cargo.toml @@ -34,7 +34,7 @@ reqwest = { version = "0.11.3", features = ["gzip"] } # https://github.com/Homebrew/homebrew-core/pull/70216 ring = "0.16.20" serde = { version = "1.0.125", features = ["derive"] } -serde_json = "1.0.64" +serde_json = "1.0.67" structopt = "0.3.21" tokio = { version = "1.6.0", features = ["full"] } toml = "0.5.8" From 8fd34a73673128af14e1818f66419a8ec7305467 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Romanowski?= Date: Mon, 6 Sep 2021 16:10:48 +0200 Subject: [PATCH 2/3] Add no check (dump links only) flag (#99) --- README.md | 2 ++ lychee-bin/src/main.rs | 9 +++++++++ lychee-bin/src/options.rs | 6 ++++++ lychee-lib/src/client.rs | 5 +++++ 4 files changed, 22 insertions(+) diff --git a/README.md b/README.md index 4d7be79..4eb7970 100644 --- a/README.md +++ b/README.md @@ -176,6 +176,8 @@ USAGE: lychee [FLAGS] [OPTIONS] [--] [inputs]... FLAGS: + --dump Don't perform any link checking. Instead, dump all the links extracted from inputs that + would be checked -E, --exclude-all-private Exclude all private IPs from checking. Equivalent to `--exclude-private --exclude-link-local --exclude-loopback` --exclude-link-local Exclude link-local IP address range from checking diff --git a/lychee-bin/src/main.rs b/lychee-bin/src/main.rs index 3f64f8d..f417538 100644 --- a/lychee-bin/src/main.rs +++ b/lychee-bin/src/main.rs @@ -205,6 +205,15 @@ async fn run(cfg: &Config, inputs: Vec) -> Result { .await .map_err(|e| anyhow!(e))?; + if cfg.dump { + for link in links { + if !client.filtered(&link.uri) { + println!("{}", link); + } + } + return Ok(ExitCode::Success as i32); + } + let pb = if cfg.no_progress { None } else { diff --git a/lychee-bin/src/options.rs b/lychee-bin/src/options.rs index be67e0b..ce8d32d 100644 --- a/lychee-bin/src/options.rs +++ b/lychee-bin/src/options.rs @@ -124,6 +124,12 @@ pub(crate) struct Config { #[serde(default)] pub(crate) no_progress: bool, + /// Don't perform any link checking. + /// Instead, dump all the links extracted from inputs that would be checked + #[structopt(long)] + #[serde(default)] + pub(crate) dump: bool, + /// Maximum number of allowed redirects #[structopt(short, long, default_value = &MAX_REDIRECTS_STR)] #[serde(default = "max_redirects")] diff --git a/lychee-lib/src/client.rs b/lychee-lib/src/client.rs index c1175ba..996583b 100644 --- a/lychee-lib/src/client.rs +++ b/lychee-lib/src/client.rs @@ -198,6 +198,11 @@ impl Client { Ok(Response::new(uri, status, source)) } + /// Check if the given URI is filtered by the client + pub fn filtered(&self, uri: &Uri) -> bool { + self.filter.is_excluded(uri) + } + pub async fn check_website(&self, uri: &Uri) -> Status { let mut retries: i64 = 3; let mut wait: u64 = 1; From 1246fa564c1a93d57d8db0285150096cedb1b5e6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 8 Sep 2021 00:21:00 +0200 Subject: [PATCH 3/3] Don't exlude mail on `exclude-all-private` (#316) --- lychee-lib/src/client.rs | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/lychee-lib/src/client.rs b/lychee-lib/src/client.rs index 996583b..5580854 100644 --- a/lychee-lib/src/client.rs +++ b/lychee-lib/src/client.rs @@ -94,7 +94,7 @@ pub struct ClientBuilder { accepted: Option>, /// Response timeout per request timeout: Option, - /// Treat HTTP links as erros when HTTPS is available + /// Treat HTTP links as errors when HTTPS is available require_https: bool, } @@ -119,7 +119,7 @@ impl ClientBuilder { exclude_private_ips: self.exclude_all_private || self.exclude_private_ips, exclude_link_local_ips: self.exclude_all_private || self.exclude_link_local_ips, exclude_loopback_ips: self.exclude_all_private || self.exclude_loopback_ips, - exclude_mail: self.exclude_all_private || self.exclude_mail, + exclude_mail: self.exclude_mail, } } @@ -282,13 +282,16 @@ where #[cfg(test)] mod test { - use std::time::{Duration, Instant}; + use std::{ + convert::TryInto, + time::{Duration, Instant}, + }; use http::{header::HeaderMap, StatusCode}; use reqwest::header; use super::ClientBuilder; - use crate::{mock_server, test_utils::get_mock_client_response}; + use crate::{mock_server, test_utils::get_mock_client_response, Uri}; #[tokio::test] async fn test_nonexistent() { @@ -386,6 +389,29 @@ mod test { assert!(res.status().is_success()); } + #[tokio::test] + async fn test_exclude_mail() { + let client = ClientBuilder::builder() + .exclude_mail(false) + .exclude_all_private(true) + .build() + .client() + .unwrap(); + assert!(!client.filtered(&Uri { + url: "mailto://mail@example.org".try_into().unwrap() + })); + + let client = ClientBuilder::builder() + .exclude_mail(true) + .exclude_all_private(true) + .build() + .client() + .unwrap(); + assert!(client.filtered(&Uri { + url: "mailto://mail@example.org".try_into().unwrap() + })); + } + #[tokio::test] async fn test_require_https() { let client = ClientBuilder::builder().build().client().unwrap();