diff --git a/Cargo.lock b/Cargo.lock index 48cbbdc..bcda10b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2925,6 +2925,7 @@ dependencies = [ "http", "http-body", "hyper", + "hyper-rustls", "hyper-tls", "ipnet", "js-sys", @@ -2934,11 +2935,15 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", + "rustls", + "rustls-native-certs", + "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", "tokio", "tokio-native-tls", + "tokio-rustls", "tokio-socks", "tokio-util", "tower-service", diff --git a/README.md b/README.md index fa45a77..ae3ce48 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,18 @@ apt install gcc pkg-config libc6-dev libssl-dev cargo install lychee ``` +#### Feature flags + +Lychee supports several feature flags: + +- `native-tls` enables the platform-native TLS crate [native-tls](https://crates.io/crates/native-tls). +- `vendored-openssl` compiles and statically links a copy of OpenSSL. See the corresponding feature of the [openssl](https://crates.io/crates/openssl) crate. +- `rustls-tls` enables the alternative TLS crate [rustls](https://crates.io/crates/rustls). +- `email-check` enables checking email addresses using the [check-if-email-exists](https://crates.io/crates/check-if-email-exists) crate. This feature requires the `native-tls` feature. +- `check_example_domains` allows checking example domains such as `example.com`. This feature is useful for testing. + +By default, `native-tls` and `email-check` are enabled. + ## Features This comparison is made on a best-effort basis. Please create a PR to fix diff --git a/benches/Cargo.toml b/benches/Cargo.toml index 5a20741..f2ec03b 100644 --- a/benches/Cargo.toml +++ b/benches/Cargo.toml @@ -8,12 +8,18 @@ edition = "2021" publish = false [dependencies] -lychee-lib = { path = "../lychee-lib"} +lychee-lib = { path = "../lychee-lib", default-features = false } # TODO: Move back to crates.io version once # https://github.com/bheisler/criterion.rs/pull/602 # got released criterion = { git = "https://github.com/bheisler/criterion.rs"} +[features] +email-check = ["lychee-lib/email-check"] +native-tls = ["lychee-lib/native-tls"] +rustls-tls = ["lychee-lib/rustls-tls"] +default = ["native-tls", "email-check"] + [[bench]] name = "extract" path = "src/extract.rs" diff --git a/examples/builder/Cargo.toml b/examples/builder/Cargo.toml index b208574..4049166 100644 --- a/examples/builder/Cargo.toml +++ b/examples/builder/Cargo.toml @@ -8,8 +8,14 @@ name = "builder" path = "builder.rs" [dependencies] -lychee-lib = { path = "../../lychee-lib", version = "0.13.0" } +lychee-lib = { path = "../../lychee-lib", version = "0.13.0", default-features = false } tokio = { version = "1.28.2", features = ["full"] } regex = "1.8.4" http = "0.2.9" -reqwest = { version = "0.11.18", features = ["gzip"] } +reqwest = { version = "0.11.18", default-features = false, features = ["gzip"] } + +[features] +email-check = ["lychee-lib/email-check"] +native-tls = ["lychee-lib/native-tls", "reqwest/native-tls"] +rustls-tls = ["lychee-lib/rustls-tls", "reqwest/rustls-tls-native-roots"] +default = ["native-tls", "email-check"] diff --git a/examples/client_pool/Cargo.toml b/examples/client_pool/Cargo.toml index d24c690..6a7c077 100644 --- a/examples/client_pool/Cargo.toml +++ b/examples/client_pool/Cargo.toml @@ -10,5 +10,11 @@ path = "client_pool.rs" [dependencies] futures = "0.3.27" tokio-stream = "0.1.14" -lychee-lib = { path = "../../lychee-lib", version = "0.13.0" } +lychee-lib = { path = "../../lychee-lib", version = "0.13.0", default-features = false } tokio = { version = "1.28.2", features = ["full"] } + +[features] +email-check = ["lychee-lib/email-check"] +native-tls = ["lychee-lib/native-tls"] +rustls-tls = ["lychee-lib/rustls-tls"] +default = ["native-tls", "email-check"] diff --git a/examples/collect_links/Cargo.toml b/examples/collect_links/Cargo.toml index 41199a2..f0371a0 100644 --- a/examples/collect_links/Cargo.toml +++ b/examples/collect_links/Cargo.toml @@ -8,9 +8,15 @@ name = "collect_links" path = "collect_links.rs" [dependencies] -lychee-lib = { path = "../../lychee-lib", version = "0.13.0" } +lychee-lib = { path = "../../lychee-lib", version = "0.13.0", default-features = false } tokio = { version = "1.28.2", features = ["full"] } regex = "1.8.4" http = "0.2.9" tokio-stream = "0.1.14" -reqwest = { version = "0.11.18", features = ["gzip"] } +reqwest = { version = "0.11.18", default-features = false, features = ["gzip"] } + +[features] +email-check = ["lychee-lib/email-check"] +native-tls = ["lychee-lib/native-tls", "reqwest/native-tls"] +rustls-tls = ["lychee-lib/rustls-tls", "reqwest/rustls-tls-native-roots"] +default = ["native-tls", "email-check"] diff --git a/examples/extract/Cargo.toml b/examples/extract/Cargo.toml index 5f89fe2..8dcf947 100644 --- a/examples/extract/Cargo.toml +++ b/examples/extract/Cargo.toml @@ -8,5 +8,11 @@ name = "extract" path = "extract.rs" [dependencies] -lychee-lib = { path = "../../lychee-lib", version = "0.13.0" } +lychee-lib = { path = "../../lychee-lib", version = "0.13.0", default-features = false } tokio = { version = "1.28.2", features = ["full"] } + +[features] +email-check = ["lychee-lib/email-check"] +native-tls = ["lychee-lib/native-tls"] +rustls-tls = ["lychee-lib/rustls-tls"] +default = ["native-tls", "email-check"] diff --git a/examples/simple/Cargo.toml b/examples/simple/Cargo.toml index 5413529..fa653bd 100644 --- a/examples/simple/Cargo.toml +++ b/examples/simple/Cargo.toml @@ -8,5 +8,11 @@ name = "simple" path = "simple.rs" [dependencies] -lychee-lib = { path = "../../lychee-lib", version = "0.13.0" } +lychee-lib = { path = "../../lychee-lib", version = "0.13.0", default-features = false } tokio = { version = "1.28.2", features = ["full"] } + +[features] +email-check = ["lychee-lib/email-check"] +native-tls = ["lychee-lib/native-tls"] +rustls-tls = ["lychee-lib/rustls-tls"] +default = ["native-tls", "email-check"] diff --git a/lychee-bin/Cargo.toml b/lychee-bin/Cargo.toml index 27c61ef..98c3fc3 100644 --- a/lychee-bin/Cargo.toml +++ b/lychee-bin/Cargo.toml @@ -24,10 +24,10 @@ const_format = "0.2.31" headers = "0.3.8" http = "0.2.9" indicatif = "0.17.5" -openssl-sys = "0.9.88" +openssl-sys = { version = "0.9.88", optional = true } pad = "0.1.6" regex = "1.8.4" -reqwest = { version = "0.11.18", features = ["gzip"] } +reqwest = { version = "0.11.18", default-features = false, features = ["gzip", "json"] } # Make build work on Apple Silicon. # See https://github.com/briansmith/ring/issues/1163 # This is necessary for the homebrew build @@ -76,9 +76,24 @@ tracing-subscriber = { version = "0.3.17", default-features = false, features = [features] #tokio-console = ["console-subscriber", "tracing-subscriber/registry"] + +# Compile and statically link a copy of OpenSSL. vendored-openssl = ["openssl-sys/vendored"] + +# Allow checking example domains such as example.com. check_example_domains = ["lychee-lib/check_example_domains"] +# Enable checking email addresses. Requires the native-tls feature. +email-check = ["lychee-lib/email-check"] + +# Use platform-native TLS. +native-tls = ["lychee-lib/native-tls", "openssl-sys", "reqwest/native-tls"] + +# Use Rustls TLS. +rustls-tls = ["lychee-lib/rustls-tls", "reqwest/rustls-tls-native-roots"] + +default = ["native-tls", "email-check"] + # Unfortunately, it's not possible to automatically enable features # for cargo test. See rust-lang/cargo#2911. # As a workaround we introduce a new feature to allow example domains diff --git a/lychee-bin/src/main.rs b/lychee-bin/src/main.rs index 45a050c..bc49b4b 100644 --- a/lychee-bin/src/main.rs +++ b/lychee-bin/src/main.rs @@ -69,9 +69,11 @@ use color::YELLOW; use commands::CommandParams; use formatters::response::ResponseFormatter; use log::{error, info, warn}; -use openssl_sys as _; + +#[cfg(feature = "native-tls")] +use openssl_sys as _; // required for vendored-openssl feature + use options::LYCHEE_CONFIG_FILE; -// required for vendored-openssl feature use ring as _; // required for apple silicon use lychee_lib::Collector; diff --git a/lychee-lib/Cargo.toml b/lychee-lib/Cargo.toml index 75147b0..d385655 100644 --- a/lychee-lib/Cargo.toml +++ b/lychee-lib/Cargo.toml @@ -17,17 +17,17 @@ repository = "https://github.com/lycheeverse/lychee" version = "0.13.0" [dependencies] -check-if-email-exists = "0.9.0" +check-if-email-exists = { version = "0.9.0", optional = true } email_address = "0.2.4" glob = "0.3.1" http = "0.2.9" linkify = "0.9.0" -openssl-sys = "0.9.88" +openssl-sys = { version = "0.9.88", optional = true } pulldown-cmark = "0.9.3" regex = "1.8.4" # Use trust-dns to avoid lookup failures on high concurrency # https://github.com/seanmonstar/reqwest/issues/296 -reqwest = { version = "0.11.18", features = ["gzip", "trust-dns"] } +reqwest = { version = "0.11.18", default-features = false, features = ["gzip", "trust-dns"] } # Make build work on Apple Silicon. # See https://github.com/briansmith/ring/issues/1163 # This is necessary for the homebrew build @@ -66,12 +66,24 @@ wiremock = "0.5.19" serde_json = "1.0.96" [features] -# Vendor OpenSSL instead of dynamically linking it at runtime. + +# Enable checking email addresses. Requires the native-tls feature. +email-check = ["check-if-email-exists"] + +# Use platform-native TLS. +native-tls = ["openssl-sys", "reqwest/native-tls"] + +# Use Rustls TLS. +rustls-tls = ["reqwest/rustls-tls-native-roots"] + +# Compile and statically link a copy of OpenSSL. vendored-openssl = ["openssl-sys/vendored"] -# Feature flag to include checking reserved example domains + +# Feature flag to include checking reserved example domains # as per RFC 2606, section 3. # This flag is off by default and only exists to allow example domains in # integration tests, which don't respect `#[cfg(test)]`. # See https://users.rust-lang.org/t/36630 check_example_domains = [] -default = [] + +default = ["native-tls", "email-check"] diff --git a/lychee-lib/src/client.rs b/lychee-lib/src/client.rs index 6d830b0..a54e03a 100644 --- a/lychee-lib/src/client.rs +++ b/lychee-lib/src/client.rs @@ -15,7 +15,9 @@ )] use std::{collections::HashSet, time::Duration}; +#[cfg(all(feature = "email-check", feature = "native-tls"))] use check_if_email_exists::{check_email, CheckEmailInput, Reachable}; + use http::{ header::{HeaderMap, HeaderValue}, StatusCode, @@ -32,10 +34,13 @@ use crate::{ quirks::Quirks, remap::Remaps, retry::RetryExt, - types::{mail, uri::github::GithubUri}, + types::uri::github::GithubUri, ErrorKind, Request, Response, Result, Status, Uri, }; +#[cfg(all(feature = "email-check", feature = "native-tls"))] +use crate::types::mail; + /// Default number of redirects before a request is deemed as failed, 5. pub const DEFAULT_MAX_REDIRECTS: usize = 5; /// Default number of retries before a request is deemed as failed, 3. @@ -437,7 +442,13 @@ impl Client { } else if uri.is_file() { self.check_file(uri) } else if uri.is_mail() { - self.check_mail(uri).await + #[cfg(all(feature = "email-check", feature = "native-tls"))] + { + self.check_mail(uri).await + } + + #[cfg(not(all(feature = "email-check", feature = "native-tls")))] + Status::Excluded } else { match self.check_website(uri).await { Status::Ok(code) if self.require_https && uri.scheme() == "http" => { @@ -599,6 +610,7 @@ impl Client { /// URIs may contain query parameters (e.g. `contact@example.com?subject="Hello"`), /// which are ignored by this check. The are not part of the mail address /// and instead passed to a mail client. + #[cfg(all(feature = "email-check", feature = "native-tls"))] pub async fn check_mail(&self, uri: &Uri) -> Status { let address = uri.url.path().to_string(); let input = CheckEmailInput::new(address); @@ -860,7 +872,7 @@ mod tests { // on slow connections, this might take a bit longer than nominal // backed-off timeout (7 secs) - assert!((350..=450).contains(&end.as_millis())); + assert!((350..=550).contains(&end.as_millis())); } #[tokio::test] diff --git a/lychee-lib/src/lib.rs b/lychee-lib/src/lib.rs index cfaeafc..0596417 100644 --- a/lychee-lib/src/lib.rs +++ b/lychee-lib/src/lib.rs @@ -74,9 +74,11 @@ pub mod test_utils; #[cfg(test)] use doc_comment as _; // required for doctest -use openssl_sys as _; // required for vendored-openssl feature use ring as _; // required for apple silicon +#[cfg(feature = "native-tls")] +use openssl_sys as _; // required for vendored-openssl feature + #[doc(inline)] pub use crate::{ // Constants get exposed so that the CLI can use the same defaults as the library diff --git a/lychee-lib/src/types/mail.rs b/lychee-lib/src/types/mail.rs index 584c700..7ef30bf 100644 --- a/lychee-lib/src/types/mail.rs +++ b/lychee-lib/src/types/mail.rs @@ -1,3 +1,5 @@ +#![cfg(all(feature = "email-check", feature = "native-tls"))] + use check_if_email_exists::{CheckEmailOutput, Reachable}; /// A crude way to extract error details from the mail output.