Extend reqwest client settings (#617)

This sets a HTTP connect timeout (for stability)
and a TCP keepalive (for performance).

The connect timeout should help with flaky servers, which
would block the runtime and therefore other requests.

The keepalive helps when making many requests to the same
host. This is a very common pattern for checking internal documentation,
which is an important use-case of lychee.

The settings are currently not configurable by the user
and set to sane defaults. We might make this configurable in the future
if there is demand to do so.
This commit is contained in:
Matthias 2022-05-13 18:51:11 +02:00 committed by GitHub
parent 3f3df8d179
commit 571b49410c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23

View file

@ -45,6 +45,13 @@ pub const DEFAULT_TIMEOUT_SECS: usize = 20;
/// Default user agent, `lychee-<PKG_VERSION>`.
pub const DEFAULT_USER_AGENT: &str = concat!("lychee/", env!("CARGO_PKG_VERSION"));
// Constants currently not configurable by the user.
/// A timeout for only the connect phase of a Client.
const CONNECT_TIMEOUT: u64 = 10;
/// TCP keepalive
/// See `https://tldp.org/HOWTO/TCP-Keepalive-HOWTO/overview.html` for more info
const TCP_KEEPALIVE: u64 = 60;
/// Builder for [`Client`].
///
/// See crate-level documentation for usage example.
@ -256,6 +263,8 @@ impl ClientBuilder {
.gzip(true)
.default_headers(headers)
.danger_accept_invalid_certs(self.allow_insecure)
.connect_timeout(Duration::from_secs(CONNECT_TIMEOUT))
.tcp_keepalive(Duration::from_secs(TCP_KEEPALIVE))
.redirect(reqwest::redirect::Policy::limited(self.max_redirects));
let reqwest_client = (match self.timeout {