Update dependencies; fix flaky tests (#1219)

examples/builder/Cargo.toml

@@ -9,7 +9,7 @@ path = "builder.rs"
 
 [dependencies]
 lychee-lib = { path = "../../lychee-lib", version = "0.13.0", default-features = false }
-tokio = { version = "1.29.1", features = ["full"] }
+tokio = { version = "1.31.0", features = ["full"] }
 regex = "1.9.3"
 http = "0.2.9"
 reqwest = { version = "0.11.18", default-features = false, features = ["gzip"] }

examples/client_pool/Cargo.toml

@@ -8,10 +8,10 @@ name = "client_pool"
 path = "client_pool.rs"
 
 [dependencies]
-futures = "0.3.27"
+futures = "0.3.28"
 tokio-stream = "0.1.14"
 lychee-lib = { path = "../../lychee-lib", version = "0.13.0", default-features = false }
-tokio = { version = "1.29.1", features = ["full"] }
+tokio = { version = "1.31.0", features = ["full"] }
 
 [features]
 email-check = ["lychee-lib/email-check"]

examples/collect_links/Cargo.toml

@@ -9,7 +9,7 @@ path = "collect_links.rs"
 
 [dependencies]
 lychee-lib = { path = "../../lychee-lib", version = "0.13.0", default-features = false }
-tokio = { version = "1.29.1", features = ["full"] }
+tokio = { version = "1.31.0", features = ["full"] }
 regex = "1.9.3"
 http = "0.2.9"
 tokio-stream = "0.1.14"

examples/extract/Cargo.toml

@@ -9,7 +9,7 @@ path = "extract.rs"
 
 [dependencies]
 lychee-lib = { path = "../../lychee-lib", version = "0.13.0", default-features = false }
-tokio = { version = "1.29.1", features = ["full"] }
+tokio = { version = "1.31.0", features = ["full"] }
 
 [features]
 email-check = ["lychee-lib/email-check"]

examples/simple/Cargo.toml

@@ -9,7 +9,7 @@ path = "simple.rs"
 
 [dependencies]
 lychee-lib = { path = "../../lychee-lib", version = "0.13.0", default-features = false }
-tokio = { version = "1.29.1", features = ["full"] }
+tokio = { version = "1.31.0", features = ["full"] }
 
 [features]
 email-check = ["lychee-lib/email-check"]

lychee-bin/Cargo.toml

@@ -19,21 +19,21 @@ version = "0.13.0"
 [dependencies]
 lychee-lib = { path = "../lychee-lib", version = "0.13.0", default-features = false }
 
-anyhow = "1.0.72"
+anyhow = "1.0.73"
 assert-json-diff = "2.0.2"
-clap = { version = "4.3.19", features = ["env", "derive"] }
+clap = { version = "4.3.21", features = ["env", "derive"] }
 console = "0.15.7"
 const_format = "0.2.31"
 csv = "1.2.2"
 dashmap = { version = "5.5.0", features = ["serde"] }
 env_logger = "0.10.0"
-futures = "0.3.27"
+futures = "0.3.28"
 headers = "0.3.8"
 http = "0.2.9"
 humantime = "2.1.0"
 humantime-serde = "1.1.1"
 indicatif = "0.17.6"
-log = "0.4.19"
+log = "0.4.20"
 once_cell = "1.18.0"
 openssl-sys = { version = "0.9.91", optional = true }
 pad = "0.1.6"
@@ -48,10 +48,10 @@ ring = "0.16.20"
 secrecy = { version = "0.8.0", features = ["serde"] }
 serde = { version = "1.0.183", features = ["derive"] }
 serde_json = "1.0.104"
-strum = {version = "0.25.0" , features = ["derive"] }
+strum = { version = "0.25.0", features = ["derive"] }
 supports-color = "2.0.0"
 tabled = "0.14.0"
-tokio = { version = "1.29.1", features = ["full"] }
+tokio = { version = "1.31.0", features = ["full"] }
 tokio-stream = "0.1.14"
 toml = "0.7.6"
 
@@ -59,7 +59,7 @@ toml = "0.7.6"
 assert_cmd = "2.0.12"
 predicates = "3.0.3"
 pretty_assertions = "1.4.0"
-tempfile = "3.6.0"
+tempfile = "3.7.1"
 tracing-subscriber = { version = "0.3.17", default-features = false, features = ["fmt", "registry", "env-filter"] }
 uuid = { version = "1.4.1", features = ["v4"] }
 wiremock = "0.5.19"

lychee-bin/src/archive/wayback/mod.rs

@@ -56,30 +56,61 @@ where
 #[cfg(test)]
 mod tests {
     use crate::archive::wayback::get_wayback_link;
-    use reqwest::Error;
+    use reqwest::{Error, Url};
     use std::{error::Error as StdError, time::Duration};
     use tokio::time::sleep;
 
     #[tokio::test]
     async fn wayback_suggestion() -> Result<(), Box<dyn StdError>> {
-        let url = "https://example.com".parse()?;
+        let target_url = "https://example.com".parse::<Url>()?;
+
+        // Extract domain from target_url without the scheme and trailing slash
+        let expected_ending = (target_url.host_str().ok_or("Invalid target URL")?).to_string();
 
         // This test can be flaky, because the wayback machine does not always
         // return a suggestion. Retry a few times if needed.
         for _ in 0..3 {
-            if let Some(suggestion) = get_wayback_link(&url).await? {
-                assert_eq!(
-                    suggestion
+            match get_wayback_link(&target_url).await {
+                Ok(Some(suggested_url)) => {
+                    // Ensure the host is correct
+                    let host = suggested_url
                         .host_str()
-                        .expect("Suggestion doesn't have a host"),
-                    "web.archive.org"
-                );
-                assert!(suggestion.path().ends_with(url.as_str()));
-                return Ok(());
+                        .ok_or("Suggestion doesn't have a host")?;
+                    assert_eq!(host, "web.archive.org");
+
+                    // Extract the actual archived URL from the Wayback URL
+                    let archived_url = suggested_url
+                        .path()
+                        .trim_start_matches("/web/")
+                        .split_once('/')
+                        .map(|x| x.1)
+                        .ok_or("Failed to extract archived URL from Wayback suggestion")?;
+
+                    // Check the ending of the suggested URL without considering trailing slash
+                    if !archived_url
+                        .trim_end_matches('/')
+                        .ends_with(&expected_ending)
+                    {
+                        return Err(format!(
+                            "Expected suggestion '{archived_url}' to end with '{expected_ending}'"
+                        )
+                        .into());
+                    }
+
+                    return Ok(());
+                }
+                Ok(None) => {
+                    // No suggestion was returned, wait and retry
+                    sleep(Duration::from_secs(1)).await;
+                }
+                Err(e) => {
+                    // Propagate other errors
+                    return Err(format!("Error retrieving Wayback link: {e}").into());
+                }
             }
-            sleep(Duration::from_secs(1)).await; // add delay between retries
         }
-        Err("Did not get a valid Wayback Machine suggestion.".into())
+
+        Err("Did not get a valid Wayback Machine suggestion after multiple attempts.".into())
     }
 
     #[tokio::test]

lychee-bin/tests/cli.rs

@@ -547,6 +547,7 @@ mod cli {
         cmd.arg(test_path)
             .arg("--exclude")
             .arg("https://en.wikipedia.org/*")
+            .arg("--exclude")
             .arg("https://ldra.com/")
             .assert()
             .success()

lychee-lib/Cargo.toml

@@ -21,7 +21,7 @@ async-stream = "0.3.5"
 cached = "0.44.0"
 check-if-email-exists = { version = "0.9.0", optional = true }
 email_address = "0.2.4"
-futures = "0.3.27"
+futures = "0.3.28"
 glob = "0.3.1"
 headers = "0.3.8"
 html5ever = "0.26.0"
@@ -31,8 +31,8 @@ hyper = "0.14.27"
 ip_network = "0.4.1"
 jwalk = "0.8.1"
 linkify = "0.10.0"
-log = "0.4.19"
-octocrab = "0.29.1"
+log = "0.4.20"
+octocrab = "0.29.3"
 once_cell = "1.18.0"
 openssl-sys = { version = "0.9.91", optional = true }
 path-clean = "1.0.1"
@@ -52,8 +52,8 @@ secrecy = "0.8.0"
 serde = { version = "1.0.183", features = ["derive"] }
 serde_with = "3.2.0"
 shellexpand = "3.1.0"
-thiserror = "1.0.44"
-tokio = { version = "1.29.1", features = ["full"] }
+thiserror = "1.0.45"
+tokio = { version = "1.31.0", features = ["full"] }
 typed-builder = "0.15.2"
 url = { version = "2.4.0", features = ["serde"] }
 
@@ -63,7 +63,7 @@ features = ["runtime-tokio"]
 
 [dev-dependencies]
 doc-comment = "0.3.3"
-tempfile = "3.6.0"
+tempfile = "3.7.1"
 wiremock = "0.5.19"
 serde_json = "1.0.104"