Add optional Rustls support (#1099)

* Add optional Rustls support

This commit adds a non-default feature flag to use Rustls instead of OpenSSL.

My personal motivation is to use Lychee on OpenBSD -current, where the
`openssl` crate frequently fails to link against the unreleased system
LibreSSL. Using the `vendored-openssl` feature helps with compilation, but
segfaults at runtime.

The commit adds three feature flags to the library, binary, benchmark, and all
examples:

- The `native-tls` feature flag toggles the `openssl` crate.
- The `rustls-tls` feature flag toggles the `rustls` crate.
- The `email-check` feature flag toggles the `check-if-email-exists` crate,
  which is the only existing functionality currently incompatible with Rustls.

By default, `native-tls` and `email-check` are enabled. Thus, Lychee (bin and
lib) can be used as before unless default features are disabled.

To use the Rustls feature, pass `--no-default-features --features rustls` to
cargo check/build/test/..., e.g.,

    $ cargo clippy --workspace --all-targets --no-default-features \ --features
    rustls-tls -- --deny warnings

Checking email addresses requires both, `native-tls` and `email-check`, to be
enabled. Otherwise, email addresses are excluded.

The `email-check` feature flag is technically not necessary. I preferred it
over `not(rustls-tls)` because it's clearer and it addresses the AGPL license
issue #594. As far as I understand, a Lychee binary compiled without the
`email-check` feature could be distributed with file-based copyleft for the
MPL-licensed dependencies only. But that's out of scope here.

The benchmark shows a performance regression varying between 2% and 4.4% when
using Rustls instead of OpenSSL on my machine.

PS: The `ring` crate needs to be patched on OpenBSD 7.3 and later until the new
xonly patches have been upstreamed, see the `rust-ring` port.

* Use platform native certificates with Rustls

By default, reqwest uses the webpki-roots crate with Rustls, effectively
bundling Mozilla's root certificates.

This commit uses the rustls-native-certs crate instead to use locally
installed root certificates, to minimize the difference between the
native-tls and rustls-tls features.

* Document feature flags

Cargo.lock

@@ -2925,6 +2925,7 @@ dependencies = [
  "http",
  "http-body",
  "hyper",
+ "hyper-rustls",
  "hyper-tls",
  "ipnet",
  "js-sys",
@@ -2934,11 +2935,15 @@ dependencies = [
  "once_cell",
  "percent-encoding",
  "pin-project-lite",
+ "rustls",
+ "rustls-native-certs",
+ "rustls-pemfile",
  "serde",
  "serde_json",
  "serde_urlencoded",
  "tokio",
  "tokio-native-tls",
+ "tokio-rustls",
  "tokio-socks",
  "tokio-util",
  "tower-service",

README.md

@@ -83,6 +83,18 @@ apt install gcc pkg-config libc6-dev libssl-dev
 cargo install lychee
 ```
 
+#### Feature flags
+
+Lychee supports several feature flags:
+
+- `native-tls` enables the platform-native TLS crate [native-tls](https://crates.io/crates/native-tls).
+- `vendored-openssl` compiles and statically links a copy of OpenSSL. See the corresponding feature of the [openssl](https://crates.io/crates/openssl) crate.
+- `rustls-tls` enables the alternative TLS crate [rustls](https://crates.io/crates/rustls).
+- `email-check` enables checking email addresses using the [check-if-email-exists](https://crates.io/crates/check-if-email-exists) crate. This feature requires the `native-tls` feature.
+- `check_example_domains` allows checking example domains such as `example.com`. This feature is useful for testing.
+
+By default, `native-tls` and `email-check` are enabled.
+
 ## Features
 
 This comparison is made on a best-effort basis. Please create a PR to fix

benches/Cargo.toml

@@ -8,12 +8,18 @@ edition = "2021"
 publish = false
 
 [dependencies]
-lychee-lib = { path = "../lychee-lib"}
+lychee-lib = { path = "../lychee-lib", default-features = false }
 # TODO: Move back to crates.io version once
 # https://github.com/bheisler/criterion.rs/pull/602
 # got released
 criterion = { git = "https://github.com/bheisler/criterion.rs"}
 
+[features]
+email-check = ["lychee-lib/email-check"]
+native-tls = ["lychee-lib/native-tls"]
+rustls-tls = ["lychee-lib/rustls-tls"]
+default = ["native-tls", "email-check"]
+
 [[bench]]
 name = "extract"
 path = "src/extract.rs"

examples/builder/Cargo.toml

@@ -8,8 +8,14 @@ name = "builder"
 path = "builder.rs"
 
 [dependencies]
-lychee-lib = { path = "../../lychee-lib", version = "0.13.0" }
+lychee-lib = { path = "../../lychee-lib", version = "0.13.0", default-features = false }
 tokio = { version = "1.28.2", features = ["full"] }
 regex = "1.8.4"
 http = "0.2.9"
-reqwest = { version = "0.11.18", features = ["gzip"] }
+reqwest = { version = "0.11.18", default-features = false, features = ["gzip"] }
+
+[features]
+email-check = ["lychee-lib/email-check"]
+native-tls = ["lychee-lib/native-tls", "reqwest/native-tls"]
+rustls-tls = ["lychee-lib/rustls-tls", "reqwest/rustls-tls-native-roots"]
+default = ["native-tls", "email-check"]

examples/client_pool/Cargo.toml

@@ -10,5 +10,11 @@ path = "client_pool.rs"
 [dependencies]
 futures = "0.3.27"
 tokio-stream = "0.1.14"
-lychee-lib = { path = "../../lychee-lib", version = "0.13.0" }
+lychee-lib = { path = "../../lychee-lib", version = "0.13.0", default-features = false }
 tokio = { version = "1.28.2", features = ["full"] }
+
+[features]
+email-check = ["lychee-lib/email-check"]
+native-tls = ["lychee-lib/native-tls"]
+rustls-tls = ["lychee-lib/rustls-tls"]
+default = ["native-tls", "email-check"]

examples/collect_links/Cargo.toml

@@ -8,9 +8,15 @@ name = "collect_links"
 path = "collect_links.rs"
 
 [dependencies]
-lychee-lib = { path = "../../lychee-lib", version = "0.13.0" }
+lychee-lib = { path = "../../lychee-lib", version = "0.13.0", default-features = false }
 tokio = { version = "1.28.2", features = ["full"] }
 regex = "1.8.4"
 http = "0.2.9"
 tokio-stream = "0.1.14"
-reqwest = { version = "0.11.18", features = ["gzip"] }
+reqwest = { version = "0.11.18", default-features = false, features = ["gzip"] }
+
+[features]
+email-check = ["lychee-lib/email-check"]
+native-tls = ["lychee-lib/native-tls", "reqwest/native-tls"]
+rustls-tls = ["lychee-lib/rustls-tls", "reqwest/rustls-tls-native-roots"]
+default = ["native-tls", "email-check"]

examples/extract/Cargo.toml

@@ -8,5 +8,11 @@ name = "extract"
 path = "extract.rs"
 
 [dependencies]
-lychee-lib = { path = "../../lychee-lib", version = "0.13.0" }
+lychee-lib = { path = "../../lychee-lib", version = "0.13.0", default-features = false }
 tokio = { version = "1.28.2", features = ["full"] }
+
+[features]
+email-check = ["lychee-lib/email-check"]
+native-tls = ["lychee-lib/native-tls"]
+rustls-tls = ["lychee-lib/rustls-tls"]
+default = ["native-tls", "email-check"]

examples/simple/Cargo.toml

@@ -8,5 +8,11 @@ name = "simple"
 path = "simple.rs"
 
 [dependencies]
-lychee-lib = { path = "../../lychee-lib", version = "0.13.0" }
+lychee-lib = { path = "../../lychee-lib", version = "0.13.0", default-features = false }
 tokio = { version = "1.28.2", features = ["full"] }
+
+[features]
+email-check = ["lychee-lib/email-check"]
+native-tls = ["lychee-lib/native-tls"]
+rustls-tls = ["lychee-lib/rustls-tls"]
+default = ["native-tls", "email-check"]

lychee-bin/Cargo.toml

@@ -24,10 +24,10 @@ const_format = "0.2.31"
 headers = "0.3.8"
 http = "0.2.9"
 indicatif = "0.17.5"
-openssl-sys = "0.9.88"
+openssl-sys = { version = "0.9.88", optional = true }
 pad = "0.1.6"
 regex = "1.8.4"
-reqwest = { version = "0.11.18", features = ["gzip"] }
+reqwest = { version = "0.11.18", default-features = false, features = ["gzip", "json"] }
 # Make build work on Apple Silicon.
 # See https://github.com/briansmith/ring/issues/1163
 # This is necessary for the homebrew build
@@ -76,9 +76,24 @@ tracing-subscriber = { version = "0.3.17", default-features = false, features =
 
 [features]
 #tokio-console = ["console-subscriber", "tracing-subscriber/registry"]
+
+# Compile and statically link a copy of OpenSSL.
 vendored-openssl = ["openssl-sys/vendored"]
+
+# Allow checking example domains such as example.com.
 check_example_domains = ["lychee-lib/check_example_domains"]
 
+# Enable checking email addresses. Requires the native-tls feature.
+email-check = ["lychee-lib/email-check"]
+
+# Use platform-native TLS.
+native-tls = ["lychee-lib/native-tls", "openssl-sys", "reqwest/native-tls"]
+
+# Use Rustls TLS.
+rustls-tls = ["lychee-lib/rustls-tls", "reqwest/rustls-tls-native-roots"]
+
+default = ["native-tls", "email-check"]
+
 # Unfortunately, it's not possible to automatically enable features
 # for cargo test. See rust-lang/cargo#2911.
 # As a workaround we introduce a new feature to allow example domains

lychee-bin/src/main.rs

@@ -69,9 +69,11 @@ use color::YELLOW;
 use commands::CommandParams;
 use formatters::response::ResponseFormatter;
 use log::{error, info, warn};
-use openssl_sys as _;
+
+#[cfg(feature = "native-tls")]
+use openssl_sys as _; // required for vendored-openssl feature
+
 use options::LYCHEE_CONFIG_FILE;
-// required for vendored-openssl feature
 use ring as _; // required for apple silicon
 
 use lychee_lib::Collector;

lychee-lib/Cargo.toml

@@ -17,17 +17,17 @@ repository = "https://github.com/lycheeverse/lychee"
 version = "0.13.0"
 
 [dependencies]
-check-if-email-exists = "0.9.0"
+check-if-email-exists = { version = "0.9.0", optional = true }
 email_address = "0.2.4"
 glob = "0.3.1"
 http = "0.2.9"
 linkify = "0.9.0"
-openssl-sys = "0.9.88"
+openssl-sys = { version = "0.9.88", optional = true }
 pulldown-cmark = "0.9.3"
 regex = "1.8.4"
 # Use trust-dns to avoid lookup failures on high concurrency
 # https://github.com/seanmonstar/reqwest/issues/296
-reqwest = { version = "0.11.18", features = ["gzip", "trust-dns"] }
+reqwest = { version = "0.11.18", default-features = false, features = ["gzip", "trust-dns"] }
 # Make build work on Apple Silicon.
 # See https://github.com/briansmith/ring/issues/1163
 # This is necessary for the homebrew build
@@ -66,12 +66,24 @@ wiremock = "0.5.19"
 serde_json = "1.0.96"
 
 [features]
-# Vendor OpenSSL instead of dynamically linking it at runtime.
+
+# Enable checking email addresses. Requires the native-tls feature.
+email-check = ["check-if-email-exists"]
+
+# Use platform-native TLS.
+native-tls = ["openssl-sys", "reqwest/native-tls"]
+
+# Use Rustls TLS.
+rustls-tls = ["reqwest/rustls-tls-native-roots"]
+
+# Compile and statically link a copy of OpenSSL.
 vendored-openssl = ["openssl-sys/vendored"]
-# Feature flag to include checking reserved example domains 
+
+# Feature flag to include checking reserved example domains
 # as per RFC 2606, section 3.
 # This flag is off by default and only exists to allow example domains in
 # integration tests, which don't respect `#[cfg(test)]`.
 # See https://users.rust-lang.org/t/36630
 check_example_domains = []
-default = []
+
+default = ["native-tls", "email-check"]

lychee-lib/src/client.rs

@@ -15,7 +15,9 @@
 )]
 use std::{collections::HashSet, time::Duration};
 
+#[cfg(all(feature = "email-check", feature = "native-tls"))]
 use check_if_email_exists::{check_email, CheckEmailInput, Reachable};
+
 use http::{
     header::{HeaderMap, HeaderValue},
     StatusCode,
@@ -32,10 +34,13 @@ use crate::{
     quirks::Quirks,
     remap::Remaps,
     retry::RetryExt,
-    types::{mail, uri::github::GithubUri},
+    types::uri::github::GithubUri,
     ErrorKind, Request, Response, Result, Status, Uri,
 };
 
+#[cfg(all(feature = "email-check", feature = "native-tls"))]
+use crate::types::mail;
+
 /// Default number of redirects before a request is deemed as failed, 5.
 pub const DEFAULT_MAX_REDIRECTS: usize = 5;
 /// Default number of retries before a request is deemed as failed, 3.
@@ -437,7 +442,13 @@ impl Client {
         } else if uri.is_file() {
             self.check_file(uri)
         } else if uri.is_mail() {
-            self.check_mail(uri).await
+            #[cfg(all(feature = "email-check", feature = "native-tls"))]
+            {
+                self.check_mail(uri).await
+            }
+
+            #[cfg(not(all(feature = "email-check", feature = "native-tls")))]
+            Status::Excluded
         } else {
             match self.check_website(uri).await {
                 Status::Ok(code) if self.require_https && uri.scheme() == "http" => {
@@ -599,6 +610,7 @@ impl Client {
     /// URIs may contain query parameters (e.g. `contact@example.com?subject="Hello"`),
     /// which are ignored by this check. The are not part of the mail address
     /// and instead passed to a mail client.
+    #[cfg(all(feature = "email-check", feature = "native-tls"))]
     pub async fn check_mail(&self, uri: &Uri) -> Status {
         let address = uri.url.path().to_string();
         let input = CheckEmailInput::new(address);
@@ -860,7 +872,7 @@ mod tests {
 
         // on slow connections, this might take a bit longer than nominal
         // backed-off timeout (7 secs)
-        assert!((350..=450).contains(&end.as_millis()));
+        assert!((350..=550).contains(&end.as_millis()));
     }
 
     #[tokio::test]

lychee-lib/src/lib.rs

@@ -74,9 +74,11 @@ pub mod test_utils;
 
 #[cfg(test)]
 use doc_comment as _; // required for doctest
-use openssl_sys as _; // required for vendored-openssl feature
 use ring as _; // required for apple silicon
 
+#[cfg(feature = "native-tls")]
+use openssl_sys as _; // required for vendored-openssl feature
+
 #[doc(inline)]
 pub use crate::{
     // Constants get exposed so that the CLI can use the same defaults as the library

lychee-lib/src/types/mail.rs

@@ -1,3 +1,5 @@
+#![cfg(all(feature = "email-check", feature = "native-tls"))]
+
 use check_if_email_exists::{CheckEmailOutput, Reachable};
 
 /// A crude way to extract error details from the mail output.