mirror of
https://github.com/Hopiu/lychee.git
synced 2026-03-16 20:50:25 +00:00
Upgrade to 2024 edition (#1711)
* Upgrade to 2024 edition * Revert expr_2021 -> expr * resolve merge conflicts * make lint happy
This commit is contained in:
parent
208fa80aa6
commit
63cdb70e6d
51 changed files with 577 additions and 501 deletions
719
Cargo.lock
generated
719
Cargo.lock
generated
File diff suppressed because it is too large
Load diff
|
|
@ -4,7 +4,7 @@ version = "0.0.0"
|
|||
authors = ["Matthias Endler <matthias@endler.dev>"]
|
||||
license = "Apache-2.0/MIT"
|
||||
description = "Criterion benchmarks of the lychee crates"
|
||||
edition = "2021"
|
||||
edition = "2024"
|
||||
publish = false
|
||||
|
||||
[dependencies]
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
use criterion::{black_box, criterion_group, criterion_main, Criterion};
|
||||
use lychee_lib::extract::Extractor;
|
||||
use criterion::{Criterion, black_box, criterion_group, criterion_main};
|
||||
use lychee_lib::InputContent;
|
||||
use lychee_lib::extract::Extractor;
|
||||
use std::path::PathBuf;
|
||||
|
||||
fn extract(paths: &[PathBuf]) {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
[package]
|
||||
name = "builder"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
edition = "2024"
|
||||
|
||||
[[example]]
|
||||
name = "builder"
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
use http::header::{self, HeaderMap};
|
||||
use http::StatusCode;
|
||||
use http::header::{self, HeaderMap};
|
||||
use lychee_lib::{ClientBuilder, Result};
|
||||
use regex::RegexSet;
|
||||
use reqwest::Method;
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
[package]
|
||||
name = "chain"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
edition = "2024"
|
||||
|
||||
[[example]]
|
||||
name = "chain"
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
use async_trait::async_trait;
|
||||
use lychee_lib::{chain::RequestChain, ChainResult, ClientBuilder, Handler, Result, Status};
|
||||
use lychee_lib::{ChainResult, ClientBuilder, Handler, Result, Status, chain::RequestChain};
|
||||
use reqwest::{Method, Request};
|
||||
|
||||
#[derive(Debug)]
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
[package]
|
||||
name = "client_pool"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
edition = "2024"
|
||||
|
||||
[[example]]
|
||||
name = "client_pool"
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
[package]
|
||||
name = "collect_links"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
edition = "2024"
|
||||
|
||||
[[example]]
|
||||
name = "collect_links"
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
[package]
|
||||
name = "extract"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
edition = "2024"
|
||||
|
||||
[[example]]
|
||||
name = "extract"
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
use lychee_lib::extract::Extractor;
|
||||
use lychee_lib::Result;
|
||||
use lychee_lib::extract::Extractor;
|
||||
use lychee_lib::{FileType, InputContent};
|
||||
use std::fs;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
[package]
|
||||
name = "simple"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
edition = "2024"
|
||||
|
||||
[[example]]
|
||||
name = "simple"
|
||||
|
|
|
|||
|
|
@ -4,13 +4,13 @@ authors = ["Matthias Endler <matthias@endler.dev>"]
|
|||
description = "A fast, async link checker"
|
||||
documentation = "https://docs.rs/lychee"
|
||||
homepage = "https://github.com/lycheeverse/lychee"
|
||||
edition = "2021"
|
||||
edition = "2024"
|
||||
keywords = ["link", "checker", "cli", "link-checker", "validator"]
|
||||
license = "Apache-2.0 OR MIT"
|
||||
repository = "https://github.com/lycheeverse/lychee"
|
||||
readme = "../README.md"
|
||||
version.workspace = true
|
||||
rust-version = "1.83.0"
|
||||
rust-version = "1.85.0"
|
||||
|
||||
[dependencies]
|
||||
# NOTE: We need to specify the version of lychee-lib here because crates.io
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize};
|
|||
use std::{fmt::Display, time::Duration};
|
||||
use strum::{Display, EnumIter, EnumString, VariantNames};
|
||||
|
||||
use crate::color::{color, GREEN, PINK};
|
||||
use crate::color::{GREEN, PINK, color};
|
||||
|
||||
mod wayback;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
use crate::time::{self, timestamp, Timestamp};
|
||||
use crate::time::{self, Timestamp, timestamp};
|
||||
use anyhow::Result;
|
||||
use dashmap::DashMap;
|
||||
use lychee_lib::{CacheStatus, Status, Uri};
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ use crate::formatters::response::ResponseFormatter;
|
|||
use crate::options::OutputMode;
|
||||
use crate::parse::parse_duration_secs;
|
||||
use crate::verbosity::Verbosity;
|
||||
use crate::{cache::Cache, stats::ResponseStats, ExitCode};
|
||||
use crate::{ExitCode, cache::Cache, stats::ResponseStats};
|
||||
|
||||
use super::CommandParams;
|
||||
|
||||
|
|
|
|||
|
|
@ -6,8 +6,8 @@ use std::io::{self, Write};
|
|||
use std::path::PathBuf;
|
||||
use tokio_stream::StreamExt;
|
||||
|
||||
use crate::verbosity::Verbosity;
|
||||
use crate::ExitCode;
|
||||
use crate::verbosity::Verbosity;
|
||||
|
||||
use super::CommandParams;
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ use lychee_lib::{CacheStatus, ResponseBody, Status};
|
|||
|
||||
use crate::formatters::color::{DIM, GREEN, NORMAL, PINK, YELLOW};
|
||||
|
||||
use super::{ResponseFormatter, MAX_RESPONSE_OUTPUT_WIDTH};
|
||||
use super::{MAX_RESPONSE_OUTPUT_WIDTH, ResponseFormatter};
|
||||
|
||||
/// A colorized formatter for the response body
|
||||
///
|
||||
|
|
|
|||
|
|
@ -115,8 +115,10 @@ mod emoji_tests {
|
|||
);
|
||||
|
||||
// Just assert the output contains the string
|
||||
assert!(formatter
|
||||
.format_detailed_response(&body)
|
||||
.ends_with("| URL is missing a host"));
|
||||
assert!(
|
||||
formatter
|
||||
.format_detailed_response(&body)
|
||||
.ends_with("| URL is missing a host")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ use std::{
|
|||
time::Duration,
|
||||
};
|
||||
|
||||
use crate::formatters::color::{color, BOLD_GREEN, BOLD_PINK, BOLD_YELLOW, DIM, NORMAL};
|
||||
use crate::formatters::color::{BOLD_GREEN, BOLD_PINK, BOLD_YELLOW, DIM, NORMAL, color};
|
||||
use crate::{formatters::get_response_formatter, options, stats::ResponseStats};
|
||||
|
||||
use super::StatsFormatter;
|
||||
|
|
@ -185,8 +185,10 @@ mod tests {
|
|||
assert!(result.contains("🚫 2 Errors"));
|
||||
|
||||
assert!(result.contains("[https://example.com/]:"));
|
||||
assert!(result
|
||||
.contains("https://github.com/mre/idiomatic-rust-doesnt-exist-man | 404 Not Found"));
|
||||
assert!(
|
||||
result
|
||||
.contains("https://github.com/mre/idiomatic-rust-doesnt-exist-man | 404 Not Found")
|
||||
);
|
||||
assert!(result.contains("https://github.com/mre/boom | 500 Internal Server Error"));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -157,8 +157,10 @@ mod tests {
|
|||
assert!(result.contains("❓ Unknown..........0"));
|
||||
assert!(result.contains("🚫 Errors...........2"));
|
||||
assert!(result.contains("Errors in https://example.com/"));
|
||||
assert!(result
|
||||
.contains("https://github.com/mre/idiomatic-rust-doesnt-exist-man | 404 Not Found"));
|
||||
assert!(
|
||||
result
|
||||
.contains("https://github.com/mre/idiomatic-rust-doesnt-exist-man | 404 Not Found")
|
||||
);
|
||||
assert!(result.contains("https://github.com/mre/boom | 500 Internal Server Error"));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,8 +9,8 @@ use http::StatusCode;
|
|||
use lychee_lib::{InputSource, ResponseBody, Status};
|
||||
use std::fmt::Write;
|
||||
use tabled::{
|
||||
settings::{object::Segment, Alignment, Modify, Style},
|
||||
Table, Tabled,
|
||||
settings::{Alignment, Modify, Style, object::Segment},
|
||||
};
|
||||
|
||||
use crate::stats::ResponseStats;
|
||||
|
|
|
|||
|
|
@ -63,7 +63,7 @@ use std::io::{self, BufRead, BufReader, ErrorKind, Write};
|
|||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::{bail, Context, Error, Result};
|
||||
use anyhow::{Context, Error, Result, bail};
|
||||
use clap::Parser;
|
||||
use commands::CommandParams;
|
||||
use formatters::{get_stats_formatter, log::init_logging};
|
||||
|
|
@ -95,7 +95,7 @@ use crate::formatters::duration::Duration;
|
|||
use crate::{
|
||||
cache::{Cache, StoreExt},
|
||||
formatters::stats::StatsFormatter,
|
||||
options::{Config, LycheeOptions, LYCHEE_CACHE_FILE, LYCHEE_IGNORE_FILE},
|
||||
options::{Config, LYCHEE_CACHE_FILE, LYCHEE_IGNORE_FILE, LycheeOptions},
|
||||
};
|
||||
|
||||
/// A C-like enum that can be cast to `i32` and used as process exit code.
|
||||
|
|
@ -169,7 +169,9 @@ fn load_config() -> Result<LycheeOptions> {
|
|||
|
||||
// TODO: Remove this warning and the parameter with 1.0
|
||||
if !&opts.config.exclude_file.is_empty() {
|
||||
warn!("WARNING: `--exclude-file` is deprecated and will soon be removed; use the `{LYCHEE_IGNORE_FILE}` file to ignore URL patterns instead. To exclude paths of files and directories, use `--exclude-path`.");
|
||||
warn!(
|
||||
"WARNING: `--exclude-file` is deprecated and will soon be removed; use the `{LYCHEE_IGNORE_FILE}` file to ignore URL patterns instead. To exclude paths of files and directories, use `--exclude-path`."
|
||||
);
|
||||
}
|
||||
|
||||
// TODO: Remove this warning and the parameter with 1.0
|
||||
|
|
@ -296,7 +298,9 @@ async fn run(opts: &LycheeOptions) -> Result<i32> {
|
|||
(Some(base), None) => Some(base),
|
||||
(None, Some(base_url)) => Some(base_url),
|
||||
(Some(_base), Some(base_url)) => {
|
||||
warn!("WARNING: Both, `--base` and `--base-url` are set. Using `base-url` and ignoring `--base` (as it's deprecated).");
|
||||
warn!(
|
||||
"WARNING: Both, `--base` and `--base-url` are set. Using `base-url` and ignoring `--base` (as it's deprecated)."
|
||||
);
|
||||
Some(base_url)
|
||||
}
|
||||
};
|
||||
|
|
@ -383,7 +387,9 @@ async fn run(opts: &LycheeOptions) -> Result<i32> {
|
|||
}
|
||||
|
||||
if github_issues && opts.config.github_token.is_none() {
|
||||
warn!("There were issues with GitHub URLs. You could try setting a GitHub token and running lychee again.",);
|
||||
warn!(
|
||||
"There were issues with GitHub URLs. You could try setting a GitHub token and running lychee again.",
|
||||
);
|
||||
}
|
||||
|
||||
if opts.config.cache {
|
||||
|
|
|
|||
|
|
@ -1,18 +1,18 @@
|
|||
use crate::archive::Archive;
|
||||
use crate::parse::parse_base;
|
||||
use crate::verbosity::Verbosity;
|
||||
use anyhow::{anyhow, Context, Error, Result};
|
||||
use anyhow::{Context, Error, Result, anyhow};
|
||||
use clap::builder::PossibleValuesParser;
|
||||
use clap::{arg, builder::TypedValueParser, Parser};
|
||||
use clap::{Parser, arg, builder::TypedValueParser};
|
||||
use const_format::{concatcp, formatcp};
|
||||
use http::{
|
||||
header::{HeaderName, HeaderValue},
|
||||
HeaderMap,
|
||||
header::{HeaderName, HeaderValue},
|
||||
};
|
||||
use lychee_lib::{
|
||||
Base, BasicAuthSelector, FileExtensions, FileType, Input, StatusCodeExcluder,
|
||||
StatusCodeSelector, DEFAULT_MAX_REDIRECTS, DEFAULT_MAX_RETRIES, DEFAULT_RETRY_WAIT_TIME_SECS,
|
||||
DEFAULT_TIMEOUT_SECS, DEFAULT_USER_AGENT,
|
||||
Base, BasicAuthSelector, DEFAULT_MAX_REDIRECTS, DEFAULT_MAX_RETRIES,
|
||||
DEFAULT_RETRY_WAIT_TIME_SECS, DEFAULT_TIMEOUT_SECS, DEFAULT_USER_AGENT, FileExtensions,
|
||||
FileType, Input, StatusCodeExcluder, StatusCodeSelector,
|
||||
};
|
||||
use reqwest::tls;
|
||||
use secrecy::{ExposeSecret, SecretString};
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
use anyhow::{Context, Result};
|
||||
use lychee_lib::{remap::Remaps, Base};
|
||||
use lychee_lib::{Base, remap::Remaps};
|
||||
use std::time::Duration;
|
||||
|
||||
/// Parse seconds into a `Duration`
|
||||
|
|
|
|||
|
|
@ -108,7 +108,7 @@ impl<'de> Deserialize<'de> for Verbosity {
|
|||
level => {
|
||||
return Err(serde::de::Error::custom(format!(
|
||||
"invalid log level `{level}`"
|
||||
)))
|
||||
)));
|
||||
}
|
||||
};
|
||||
Ok(Verbosity {
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ mod cli {
|
|||
use http::{Method, StatusCode};
|
||||
use lychee_lib::{InputSource, ResponseBody};
|
||||
use predicates::{
|
||||
prelude::{predicate, PredicateBooleanExt},
|
||||
prelude::{PredicateBooleanExt, predicate},
|
||||
str::{contains, is_empty},
|
||||
};
|
||||
use pretty_assertions::assert_eq;
|
||||
|
|
@ -24,7 +24,7 @@ mod cli {
|
|||
use serde_json::Value;
|
||||
use tempfile::NamedTempFile;
|
||||
use uuid::Uuid;
|
||||
use wiremock::{matchers::basic_auth, Mock, ResponseTemplate};
|
||||
use wiremock::{Mock, ResponseTemplate, matchers::basic_auth};
|
||||
|
||||
type Result<T> = std::result::Result<T, Box<dyn Error>>;
|
||||
|
||||
|
|
|
|||
|
|
@ -3,14 +3,14 @@ name = "lychee-lib"
|
|||
authors = ["Matthias Endler <matthias@endler.dev>"]
|
||||
description = "A fast, async link checker"
|
||||
documentation = "https://docs.rs/lychee_lib"
|
||||
edition = "2021"
|
||||
edition = "2024"
|
||||
homepage = "https://github.com/lycheeverse/lychee"
|
||||
keywords = ["link", "checker", "cli", "link-checker", "validator"]
|
||||
license = "Apache-2.0 OR MIT"
|
||||
repository = "https://github.com/lycheeverse/lychee"
|
||||
readme = "../README.md"
|
||||
version.workspace = true
|
||||
rust-version = "1.83.0"
|
||||
rust-version = "1.85.0"
|
||||
|
||||
[dependencies]
|
||||
async-stream = "0.3.6"
|
||||
|
|
|
|||
|
|
@ -3,8 +3,8 @@ use log::warn;
|
|||
use std::path::{Path, PathBuf};
|
||||
|
||||
use crate::{
|
||||
utils::fragment_checker::{FragmentChecker, FragmentInput},
|
||||
Base, ErrorKind, Status, Uri,
|
||||
utils::fragment_checker::{FragmentChecker, FragmentInput},
|
||||
};
|
||||
|
||||
/// A utility for checking the existence and validity of file-based URIs.
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ use crate::ErrorKind;
|
|||
use crate::{Status, Uri};
|
||||
|
||||
#[cfg(all(feature = "email-check", feature = "native-tls"))]
|
||||
use check_if_email_exists::{check_email, CheckEmailInput, Reachable};
|
||||
use check_if_email_exists::{CheckEmailInput, Reachable, check_email};
|
||||
|
||||
#[cfg(all(feature = "email-check", feature = "native-tls"))]
|
||||
use crate::types::mail;
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
use crate::{
|
||||
BasicAuthCredentials, ErrorKind, Status, Uri,
|
||||
chain::{Chain, ChainResult, ClientRequestChains, Handler, RequestChain},
|
||||
quirks::Quirks,
|
||||
retry::RetryExt,
|
||||
types::uri::github::GithubUri,
|
||||
utils::fragment_checker::{FragmentChecker, FragmentInput},
|
||||
BasicAuthCredentials, ErrorKind, Status, Uri,
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
use http::{Method, StatusCode};
|
||||
|
|
|
|||
|
|
@ -16,8 +16,8 @@
|
|||
use std::{collections::HashSet, sync::Arc, time::Duration};
|
||||
|
||||
use http::{
|
||||
header::{HeaderMap, HeaderValue},
|
||||
StatusCode,
|
||||
header::{HeaderMap, HeaderValue},
|
||||
};
|
||||
use log::debug;
|
||||
use octocrab::Octocrab;
|
||||
|
|
@ -28,11 +28,11 @@ use secrecy::{ExposeSecret, SecretString};
|
|||
use typed_builder::TypedBuilder;
|
||||
|
||||
use crate::{
|
||||
Base, BasicAuthCredentials, ErrorKind, Request, Response, Result, Status, Uri,
|
||||
chain::RequestChain,
|
||||
checker::{file::FileChecker, mail::MailChecker, website::WebsiteChecker},
|
||||
filter::{Excludes, Filter, Includes},
|
||||
remap::Remaps,
|
||||
Base, BasicAuthCredentials, ErrorKind, Request, Response, Result, Status, Uri,
|
||||
};
|
||||
|
||||
/// Default number of redirects before a request is deemed as failed, 5.
|
||||
|
|
@ -561,17 +561,17 @@ mod tests {
|
|||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use http::{header::HeaderMap, StatusCode};
|
||||
use http::{StatusCode, header::HeaderMap};
|
||||
use reqwest::header;
|
||||
use tempfile::tempdir;
|
||||
use wiremock::matchers::path;
|
||||
|
||||
use super::ClientBuilder;
|
||||
use crate::{
|
||||
ErrorKind, Request, Status, Uri,
|
||||
chain::{ChainResult, Handler, RequestChain},
|
||||
mock_server,
|
||||
test_utils::get_mock_client_response,
|
||||
ErrorKind, Request, Status, Uri,
|
||||
};
|
||||
|
||||
#[tokio::test]
|
||||
|
|
|
|||
|
|
@ -1,13 +1,13 @@
|
|||
use crate::ErrorKind;
|
||||
use crate::InputSource;
|
||||
use crate::{
|
||||
basic_auth::BasicAuthExtractor, extract::Extractor, types::uri::raw::RawUri,
|
||||
types::FileExtensions, utils::request, Base, Input, Request, Result,
|
||||
Base, Input, Request, Result, basic_auth::BasicAuthExtractor, extract::Extractor,
|
||||
types::FileExtensions, types::uri::raw::RawUri, utils::request,
|
||||
};
|
||||
use futures::TryStreamExt;
|
||||
use futures::{
|
||||
stream::{self, Stream},
|
||||
StreamExt,
|
||||
stream::{self, Stream},
|
||||
};
|
||||
use par_stream::ParStreamExt;
|
||||
use std::path::PathBuf;
|
||||
|
|
@ -186,10 +186,9 @@ mod tests {
|
|||
|
||||
use super::*;
|
||||
use crate::{
|
||||
mock_server,
|
||||
Result, Uri, mock_server,
|
||||
test_utils::{load_fixture, mail, path, website},
|
||||
types::{FileType, Input, InputSource},
|
||||
Result, Uri,
|
||||
};
|
||||
|
||||
// Helper function to run the collector on the given inputs
|
||||
|
|
|
|||
|
|
@ -169,7 +169,7 @@ impl LinkExtractor {
|
|||
attr_name: &str,
|
||||
elem_name: &str,
|
||||
attr_value: &'a str,
|
||||
) -> Option<impl Iterator<Item = &'a str>> {
|
||||
) -> Option<impl Iterator<Item = &'a str> + use<'a>> {
|
||||
// For a comprehensive list of elements that might contain URLs/URIs
|
||||
// see https://www.w3.org/TR/REC-html40/index/attributes.html
|
||||
// and https://html.spec.whatwg.org/multipage/indices.html#attributes-1
|
||||
|
|
|
|||
|
|
@ -279,13 +279,11 @@ impl Emitter for &mut LinkExtractor {
|
|||
fn emit_current_tag(&mut self) -> Option<State> {
|
||||
self.flush_links();
|
||||
|
||||
let next_state = if self.current_element.is_closing {
|
||||
if self.current_element.is_closing {
|
||||
None
|
||||
} else {
|
||||
html5gum::naive_next_state(self.current_element.name.as_bytes())
|
||||
};
|
||||
|
||||
next_state
|
||||
}
|
||||
}
|
||||
|
||||
fn emit_current_doctype(&mut self) {}
|
||||
|
|
|
|||
|
|
@ -208,7 +208,9 @@ mod tests {
|
|||
#[test]
|
||||
fn test_parse_srcset_with_commas() {
|
||||
assert_eq!(
|
||||
parse("/cdn-cgi/image/format=webp,width=640/https://img.youtube.com/vi/hVBl8_pgQf0/maxresdefault.jpg 640w, /cdn-cgi/image/format=webp,width=750/https://img.youtube.com/vi/hVBl8_pgQf0/maxresdefault.jpg 750w"),
|
||||
parse(
|
||||
"/cdn-cgi/image/format=webp,width=640/https://img.youtube.com/vi/hVBl8_pgQf0/maxresdefault.jpg 640w, /cdn-cgi/image/format=webp,width=750/https://img.youtube.com/vi/hVBl8_pgQf0/maxresdefault.jpg 750w"
|
||||
),
|
||||
vec![
|
||||
"/cdn-cgi/image/format=webp,width=640/https://img.youtube.com/vi/hVBl8_pgQf0/maxresdefault.jpg",
|
||||
"/cdn-cgi/image/format=webp,width=750/https://img.youtube.com/vi/hVBl8_pgQf0/maxresdefault.jpg"
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
use crate::types::{uri::raw::RawUri, FileType, InputContent};
|
||||
use crate::types::{FileType, InputContent, uri::raw::RawUri};
|
||||
|
||||
pub mod html;
|
||||
pub mod markdown;
|
||||
|
|
@ -63,10 +63,10 @@ mod tests {
|
|||
|
||||
use super::*;
|
||||
use crate::{
|
||||
Uri,
|
||||
test_utils::{load_fixture, mail, website},
|
||||
types::{FileType, InputContent, InputSource},
|
||||
utils::url::find_links,
|
||||
Uri,
|
||||
};
|
||||
|
||||
fn extract_uris(input: &str, file_type: FileType) -> HashSet<Uri> {
|
||||
|
|
|
|||
|
|
@ -259,8 +259,8 @@ mod tests {
|
|||
|
||||
use super::{Excludes, Filter, Includes};
|
||||
use crate::{
|
||||
test_utils::{mail, website},
|
||||
Uri,
|
||||
test_utils::{mail, website},
|
||||
};
|
||||
|
||||
// Note: the standard library, as of Rust stable 1.47.0, does not expose
|
||||
|
|
|
|||
|
|
@ -90,15 +90,15 @@ pub use crate::{
|
|||
chain::{ChainResult, Handler},
|
||||
// Constants get exposed so that the CLI can use the same defaults as the library
|
||||
client::{
|
||||
check, Client, ClientBuilder, DEFAULT_MAX_REDIRECTS, DEFAULT_MAX_RETRIES,
|
||||
DEFAULT_RETRY_WAIT_TIME_SECS, DEFAULT_TIMEOUT_SECS, DEFAULT_USER_AGENT,
|
||||
Client, ClientBuilder, DEFAULT_MAX_REDIRECTS, DEFAULT_MAX_RETRIES,
|
||||
DEFAULT_RETRY_WAIT_TIME_SECS, DEFAULT_TIMEOUT_SECS, DEFAULT_USER_AGENT, check,
|
||||
},
|
||||
collector::Collector,
|
||||
filter::{Excludes, Filter, Includes},
|
||||
types::{
|
||||
uri::valid::Uri, AcceptRange, AcceptRangeError, Base, BasicAuthCredentials,
|
||||
BasicAuthSelector, CacheStatus, CookieJar, ErrorKind, FileExtensions, FileType, Input,
|
||||
InputContent, InputSource, Request, Response, ResponseBody, Result, Status,
|
||||
StatusCodeExcluder, StatusCodeSelector,
|
||||
AcceptRange, AcceptRangeError, Base, BasicAuthCredentials, BasicAuthSelector, CacheStatus,
|
||||
CookieJar, ErrorKind, FileExtensions, FileType, Input, InputContent, InputSource, Request,
|
||||
Response, ResponseBody, Result, Status, StatusCodeExcluder, StatusCodeSelector,
|
||||
uri::valid::Uri,
|
||||
},
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
use crate::{
|
||||
chain::{ChainResult, Handler},
|
||||
Status,
|
||||
chain::{ChainResult, Handler},
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
use header::HeaderValue;
|
||||
|
|
@ -108,7 +108,7 @@ impl Handler<Request, Status> for Quirks {
|
|||
#[cfg(test)]
|
||||
mod tests {
|
||||
use header::HeaderValue;
|
||||
use http::{header, Method};
|
||||
use http::{Method, header};
|
||||
use reqwest::{Request, Url};
|
||||
|
||||
use super::Quirks;
|
||||
|
|
|
|||
|
|
@ -114,9 +114,9 @@ impl TryFrom<&[String]> for Remaps {
|
|||
for remap in remaps {
|
||||
let params: Vec<_> = remap.split_whitespace().collect();
|
||||
if params.len() != 2 {
|
||||
return Err(ErrorKind::InvalidUrlRemap(
|
||||
format!("Cannot parse into URI remapping, must be a Regex pattern and a URL separated by whitespaces: {remap}"
|
||||
)));
|
||||
return Err(ErrorKind::InvalidUrlRemap(format!(
|
||||
"Cannot parse into URI remapping, must be a Regex pattern and a URL separated by whitespaces: {remap}"
|
||||
)));
|
||||
}
|
||||
|
||||
let pattern = Regex::new(params[0])?;
|
||||
|
|
|
|||
|
|
@ -2,14 +2,14 @@ use async_trait::async_trait;
|
|||
use std::str::FromStr;
|
||||
|
||||
use headers::authorization::Credentials;
|
||||
use headers::{authorization::Basic, Authorization};
|
||||
use headers::{Authorization, authorization::Basic};
|
||||
use http::header::AUTHORIZATION;
|
||||
use reqwest::Request;
|
||||
use serde::Deserialize;
|
||||
use thiserror::Error;
|
||||
|
||||
use crate::chain::{ChainResult, Handler};
|
||||
use crate::Status;
|
||||
use crate::chain::{ChainResult, Handler};
|
||||
|
||||
#[derive(Copy, Clone, Debug, Error, PartialEq)]
|
||||
pub enum BasicAuthCredentialsParseError {
|
||||
|
|
@ -22,7 +22,9 @@ pub enum BasicAuthCredentialsParseError {
|
|||
#[error("Missing basic auth username")]
|
||||
MissingUsername,
|
||||
|
||||
#[error("Too many values separated by colon. Expected 2, got {0}. Valid form is '<username>:<password>'")]
|
||||
#[error(
|
||||
"Too many values separated by colon. Expected 2, got {0}. Valid form is '<username>:<password>'"
|
||||
)]
|
||||
TooManyParts(usize),
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ use std::str::FromStr;
|
|||
use serde_with::DeserializeFromStr;
|
||||
use thiserror::Error;
|
||||
|
||||
use crate::{types::basic_auth::BasicAuthCredentialsParseError, BasicAuthCredentials};
|
||||
use crate::{BasicAuthCredentials, types::basic_auth::BasicAuthCredentialsParseError};
|
||||
|
||||
#[derive(Clone, Debug, Error, PartialEq)]
|
||||
pub enum BasicAuthSelectorParseError {
|
||||
|
|
@ -13,7 +13,9 @@ pub enum BasicAuthSelectorParseError {
|
|||
#[error("Missing basic auth credentials or URI. Valid form is '<uri> <username>:<password>'")]
|
||||
InvalidSyntax,
|
||||
|
||||
#[error("Too many space separated values. Expected 2, got {0}. Valid form is '<uri> <username>:<password>'")]
|
||||
#[error(
|
||||
"Too many space separated values. Expected 2, got {0}. Valid form is '<uri> <username>:<password>'"
|
||||
)]
|
||||
TooManyParts(usize),
|
||||
|
||||
#[error("Basic auth credentials error")]
|
||||
|
|
|
|||
|
|
@ -90,8 +90,8 @@ impl From<&Status> for CacheStatus {
|
|||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use serde::de::value::{BorrowedStrDeserializer, Error as DeserializerError};
|
||||
use serde::Deserialize;
|
||||
use serde::de::value::{BorrowedStrDeserializer, Error as DeserializerError};
|
||||
|
||||
use crate::CacheStatus;
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ use tokio::task::JoinError;
|
|||
|
||||
use super::InputContent;
|
||||
use crate::types::StatusCodeSelectorError;
|
||||
use crate::{basic_auth::BasicAuthExtractorError, utils, Uri};
|
||||
use crate::{Uri, basic_auth::BasicAuthExtractorError, utils};
|
||||
|
||||
/// Kinds of status errors
|
||||
/// Note: The error messages can change over time, so don't match on the output
|
||||
|
|
@ -119,7 +119,9 @@ pub enum ErrorKind {
|
|||
InvalidGlobPattern(#[from] glob::PatternError),
|
||||
|
||||
/// The GitHub API could not be called because of a missing GitHub token.
|
||||
#[error("GitHub token not specified. To check GitHub links reliably, use `--github-token` flag / `GITHUB_TOKEN` env var.")]
|
||||
#[error(
|
||||
"GitHub token not specified. To check GitHub links reliably, use `--github-token` flag / `GITHUB_TOKEN` env var."
|
||||
)]
|
||||
MissingGitHubToken,
|
||||
|
||||
/// Used an insecure URI where a secure variant was reachable
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
use crate::types::FileType;
|
||||
use crate::{utils, ErrorKind, Result};
|
||||
use crate::{ErrorKind, Result, utils};
|
||||
use async_stream::try_stream;
|
||||
use futures::stream::Stream;
|
||||
use glob::glob_with;
|
||||
|
|
@ -11,7 +11,7 @@ use shellexpand::tilde;
|
|||
use std::fmt::Display;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use tokio::io::{stdin, AsyncReadExt};
|
||||
use tokio::io::{AsyncReadExt, stdin};
|
||||
|
||||
use super::file::FileExtensions;
|
||||
|
||||
|
|
@ -357,7 +357,7 @@ impl Input {
|
|||
&self,
|
||||
pattern: &str,
|
||||
ignore_case: bool,
|
||||
) -> impl Stream<Item = Result<InputContent>> + '_ {
|
||||
) -> impl Stream<Item = Result<InputContent>> + '_ + use<'_> {
|
||||
let glob_expanded = tilde(&pattern).to_string();
|
||||
let mut match_opts = glob::MatchOptions::new();
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
use std::{collections::HashSet, fmt::Display, str::FromStr};
|
||||
|
||||
use serde::{de::Visitor, Deserialize};
|
||||
use serde::{Deserialize, de::Visitor};
|
||||
|
||||
use crate::{
|
||||
types::accept::AcceptRange, types::status_code::StatusCodeSelectorError, AcceptRangeError,
|
||||
AcceptRangeError, types::accept::AcceptRange, types::status_code::StatusCodeSelectorError,
|
||||
};
|
||||
|
||||
/// A [`StatusCodeExcluder`] holds ranges of HTTP status codes, and determines
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
use std::{collections::HashSet, fmt::Display, str::FromStr};
|
||||
|
||||
use serde::{de::Visitor, Deserialize};
|
||||
use serde::{Deserialize, de::Visitor};
|
||||
use thiserror::Error;
|
||||
|
||||
use crate::{types::accept::AcceptRange, AcceptRangeError};
|
||||
use crate::{AcceptRangeError, types::accept::AcceptRange};
|
||||
|
||||
#[derive(Debug, Error, PartialEq)]
|
||||
pub enum StatusCodeSelectorError {
|
||||
|
|
|
|||
|
|
@ -187,16 +187,20 @@ mod tests {
|
|||
.is_err()
|
||||
);
|
||||
|
||||
assert!(GithubUri::try_from(website(
|
||||
"https://github.com/marketplace/actions/lychee-broken-link-checker"
|
||||
))
|
||||
.is_err());
|
||||
assert!(
|
||||
GithubUri::try_from(website(
|
||||
"https://github.com/marketplace/actions/lychee-broken-link-checker"
|
||||
))
|
||||
.is_err()
|
||||
);
|
||||
|
||||
assert!(GithubUri::try_from(website("https://github.com/features/actions")).is_err());
|
||||
|
||||
assert!(GithubUri::try_from(website(
|
||||
"https://pkg.go.dev/github.com/Debian/pkg-go-tools/cmd/pgt-gopath"
|
||||
))
|
||||
.is_err());
|
||||
assert!(
|
||||
GithubUri::try_from(website(
|
||||
"https://pkg.go.dev/github.com/Debian/pkg-go-tools/cmd/pgt-gopath"
|
||||
))
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,13 +1,13 @@
|
|||
use std::{
|
||||
collections::{hash_map::Entry, HashMap, HashSet},
|
||||
collections::{HashMap, HashSet, hash_map::Entry},
|
||||
path::Path,
|
||||
sync::Arc,
|
||||
};
|
||||
|
||||
use crate::{
|
||||
Result,
|
||||
extract::{html::html5gum::extract_html_fragments, markdown::extract_markdown_fragments},
|
||||
types::{ErrorKind, FileType},
|
||||
Result,
|
||||
};
|
||||
use percent_encoding::percent_decode_str;
|
||||
use tokio::{fs, sync::Mutex};
|
||||
|
|
|
|||
|
|
@ -7,10 +7,10 @@ use std::{
|
|||
};
|
||||
|
||||
use crate::{
|
||||
basic_auth::BasicAuthExtractor,
|
||||
types::{uri::raw::RawUri, InputSource},
|
||||
utils::{path, url},
|
||||
Base, BasicAuthCredentials, ErrorKind, Request, Result, Uri,
|
||||
basic_auth::BasicAuthExtractor,
|
||||
types::{InputSource, uri::raw::RawUri},
|
||||
utils::{path, url},
|
||||
};
|
||||
|
||||
/// Extract basic auth credentials for a given URL.
|
||||
|
|
@ -230,9 +230,11 @@ mod tests {
|
|||
let requests = create(uris, &source, None, Some(&base), None);
|
||||
|
||||
assert_eq!(requests.len(), 1);
|
||||
assert!(requests
|
||||
.iter()
|
||||
.any(|r| r.uri.url.as_str() == "https://example.com/path/relative.html"));
|
||||
assert!(
|
||||
requests
|
||||
.iter()
|
||||
.any(|r| r.uri.url.as_str() == "https://example.com/path/relative.html")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -244,9 +246,11 @@ mod tests {
|
|||
let requests = create(uris, &source, None, Some(&base), None);
|
||||
|
||||
assert_eq!(requests.len(), 1);
|
||||
assert!(requests
|
||||
.iter()
|
||||
.any(|r| r.uri.url.as_str() == "https://another.com/page"));
|
||||
assert!(
|
||||
requests
|
||||
.iter()
|
||||
.any(|r| r.uri.url.as_str() == "https://another.com/page")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -258,9 +262,11 @@ mod tests {
|
|||
let requests = create(uris, &source, None, Some(&base), None);
|
||||
|
||||
assert_eq!(requests.len(), 1);
|
||||
assert!(requests
|
||||
.iter()
|
||||
.any(|r| r.uri.url.as_str() == "https://example.com/root-relative"));
|
||||
assert!(
|
||||
requests
|
||||
.iter()
|
||||
.any(|r| r.uri.url.as_str() == "https://example.com/root-relative")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -272,9 +278,11 @@ mod tests {
|
|||
let requests = create(uris, &source, None, Some(&base), None);
|
||||
|
||||
assert_eq!(requests.len(), 1);
|
||||
assert!(requests
|
||||
.iter()
|
||||
.any(|r| r.uri.url.as_str() == "https://example.com/parent"));
|
||||
assert!(
|
||||
requests
|
||||
.iter()
|
||||
.any(|r| r.uri.url.as_str() == "https://example.com/parent")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -286,9 +294,11 @@ mod tests {
|
|||
let requests = create(uris, &source, None, Some(&base), None);
|
||||
|
||||
assert_eq!(requests.len(), 1);
|
||||
assert!(requests
|
||||
.iter()
|
||||
.any(|r| r.uri.url.as_str() == "https://example.com/path/page.html#fragment"));
|
||||
assert!(
|
||||
requests
|
||||
.iter()
|
||||
.any(|r| r.uri.url.as_str() == "https://example.com/path/page.html#fragment")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -300,9 +310,11 @@ mod tests {
|
|||
let requests = create(uris, &source, Some(&root_dir), None, None);
|
||||
|
||||
assert_eq!(requests.len(), 1);
|
||||
assert!(requests
|
||||
.iter()
|
||||
.any(|r| r.uri.url.as_str() == "file:///some/relative.html"));
|
||||
assert!(
|
||||
requests
|
||||
.iter()
|
||||
.any(|r| r.uri.url.as_str() == "file:///some/relative.html")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -314,9 +326,11 @@ mod tests {
|
|||
let requests = create(uris, &source, Some(&root_dir), None, None);
|
||||
|
||||
assert_eq!(requests.len(), 1);
|
||||
assert!(requests
|
||||
.iter()
|
||||
.any(|r| r.uri.url.as_str() == "https://another.com/page"));
|
||||
assert!(
|
||||
requests
|
||||
.iter()
|
||||
.any(|r| r.uri.url.as_str() == "https://another.com/page")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -328,9 +342,11 @@ mod tests {
|
|||
let requests = create(uris, &source, Some(&root_dir), None, None);
|
||||
|
||||
assert_eq!(requests.len(), 1);
|
||||
assert!(requests
|
||||
.iter()
|
||||
.any(|r| r.uri.url.as_str() == "file:///tmp/lychee/root-relative"));
|
||||
assert!(
|
||||
requests
|
||||
.iter()
|
||||
.any(|r| r.uri.url.as_str() == "file:///tmp/lychee/root-relative")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -342,9 +358,11 @@ mod tests {
|
|||
let requests = create(uris, &source, Some(&root_dir), None, None);
|
||||
|
||||
assert_eq!(requests.len(), 1);
|
||||
assert!(requests
|
||||
.iter()
|
||||
.any(|r| r.uri.url.as_str() == "file:///parent"));
|
||||
assert!(
|
||||
requests
|
||||
.iter()
|
||||
.any(|r| r.uri.url.as_str() == "file:///parent")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -356,9 +374,11 @@ mod tests {
|
|||
let requests = create(uris, &source, Some(&root_dir), None, None);
|
||||
|
||||
assert_eq!(requests.len(), 1);
|
||||
assert!(requests
|
||||
.iter()
|
||||
.any(|r| r.uri.url.as_str() == "file:///some/page.html#fragment"));
|
||||
assert!(
|
||||
requests
|
||||
.iter()
|
||||
.any(|r| r.uri.url.as_str() == "file:///some/page.html#fragment")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -371,9 +391,11 @@ mod tests {
|
|||
let requests = create(uris, &source, Some(&root_dir), Some(&base), None);
|
||||
|
||||
assert_eq!(requests.len(), 1);
|
||||
assert!(requests
|
||||
.iter()
|
||||
.any(|r| r.uri.url.as_str() == "https://example.com/path/relative.html"));
|
||||
assert!(
|
||||
requests
|
||||
.iter()
|
||||
.any(|r| r.uri.url.as_str() == "https://example.com/path/relative.html")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -386,9 +408,11 @@ mod tests {
|
|||
let requests = create(uris, &source, Some(&root_dir), Some(&base), None);
|
||||
|
||||
assert_eq!(requests.len(), 1);
|
||||
assert!(requests
|
||||
.iter()
|
||||
.any(|r| r.uri.url.as_str() == "https://another.com/page"));
|
||||
assert!(
|
||||
requests
|
||||
.iter()
|
||||
.any(|r| r.uri.url.as_str() == "https://another.com/page")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -401,9 +425,11 @@ mod tests {
|
|||
let requests = create(uris, &source, Some(&root_dir), Some(&base), None);
|
||||
|
||||
assert_eq!(requests.len(), 1);
|
||||
assert!(requests
|
||||
.iter()
|
||||
.any(|r| r.uri.url.as_str() == "https://example.com/tmp/lychee/root-relative"));
|
||||
assert!(
|
||||
requests
|
||||
.iter()
|
||||
.any(|r| r.uri.url.as_str() == "https://example.com/tmp/lychee/root-relative")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -416,9 +442,11 @@ mod tests {
|
|||
let requests = create(uris, &source, Some(&root_dir), Some(&base), None);
|
||||
|
||||
assert_eq!(requests.len(), 1);
|
||||
assert!(requests
|
||||
.iter()
|
||||
.any(|r| r.uri.url.as_str() == "https://example.com/parent"));
|
||||
assert!(
|
||||
requests
|
||||
.iter()
|
||||
.any(|r| r.uri.url.as_str() == "https://example.com/parent")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -431,9 +459,11 @@ mod tests {
|
|||
let requests = create(uris, &source, Some(&root_dir), Some(&base), None);
|
||||
|
||||
assert_eq!(requests.len(), 1);
|
||||
assert!(requests
|
||||
.iter()
|
||||
.any(|r| r.uri.url.as_str() == "https://example.com/path/page.html#fragment"));
|
||||
assert!(
|
||||
requests
|
||||
.iter()
|
||||
.any(|r| r.uri.url.as_str() == "https://example.com/path/page.html#fragment")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -444,9 +474,11 @@ mod tests {
|
|||
let requests = create(uris, &source, None, None, None);
|
||||
|
||||
assert_eq!(requests.len(), 1);
|
||||
assert!(requests
|
||||
.iter()
|
||||
.any(|r| r.uri.url.as_str() == "https://example.com/page"));
|
||||
assert!(
|
||||
requests
|
||||
.iter()
|
||||
.any(|r| r.uri.url.as_str() == "https://example.com/page")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
|
|||
Loading…
Reference in a new issue