fix: only check the fragment when it's a file (#1713)

* fix: only check the fragment when it's a file
* add dir fragment test
* Clean up unused fragment_check in Client

---------

Signed-off-by: Keming <kemingy94@gmail.com>
Co-authored-by: Matthias <matthias@endler.dev>

fixtures/fragments/file1.md

@@ -65,3 +65,5 @@ without related HTML element. Browser will scroll to the top of the page.
 ##### Lets wear a hat: être
 
 A link to the non-existing fragment: [try](https://github.com/lycheeverse/lychee#non-existent-anchor).
+
+Skip the fragment check for directories like: [empty](empty_dir/).

lychee-bin/tests/cli.rs

@@ -1843,8 +1843,8 @@ mod cli {
             .stderr(contains(
                 "https://github.com/lycheeverse/lychee#non-existent-anchor",
             ))
-            .stdout(contains("27 Total"))
-            .stdout(contains("22 OK"))
+            .stdout(contains("28 Total"))
+            .stdout(contains("23 OK"))
             // 4 failures because of missing fragments
             .stdout(contains("5 Errors"));
     }

lychee-lib/src/checker/file.rs

@@ -123,7 +123,8 @@ impl FileChecker {
     ///
     /// Returns a `Status` indicating the result of the check.
     async fn check_existing_path(&self, path: &Path, uri: &Uri) -> Status {
-        if self.include_fragments {
+        // only files can contain content with fragments
+        if self.include_fragments && path.is_file() {
             self.check_fragment(path, uri).await
         } else {
             Status::Ok(StatusCode::OK)
@@ -175,12 +176,12 @@ impl FileChecker {
                 Ok(true) => Status::Ok(StatusCode::OK),
                 Ok(false) => ErrorKind::InvalidFragment(uri.clone()).into(),
                 Err(err) => {
-                    warn!("Skipping fragment check due to the following error: {err}");
+                    warn!("Skipping fragment check for {uri} due to the following error: {err}");
                     Status::Ok(StatusCode::OK)
                 }
             },
             Err(err) => {
-                warn!("Skipping fragment check due to the following error: {err}");
+                warn!("Skipping fragment check for {uri} due to the following error: {err}");
                 Status::Ok(StatusCode::OK)
             }
         }

lychee-lib/src/client.rs

@@ -13,13 +13,13 @@
     clippy::default_trait_access,
     clippy::used_underscore_binding
 )]
-use std::{collections::HashSet, path::Path, sync::Arc, time::Duration};
+use std::{collections::HashSet, sync::Arc, time::Duration};
 
 use http::{
     header::{HeaderMap, HeaderValue},
     StatusCode,
 };
-use log::{debug, warn};
+use log::debug;
 use octocrab::Octocrab;
 use regex::RegexSet;
 use reqwest::{header, redirect, tls};
@@ -32,7 +32,6 @@ use crate::{
     checker::{file::FileChecker, mail::MailChecker, website::WebsiteChecker},
     filter::{Excludes, Filter, Includes},
     remap::Remaps,
-    utils::fragment_checker::{FragmentChecker, FragmentInput},
     Base, BasicAuthCredentials, ErrorKind, Request, Response, Result, Status, Uri,
 };
 
@@ -410,7 +409,6 @@ impl ClientBuilder {
                 self.fallback_extensions,
                 self.include_fragments,
             ),
-            fragment_checker: FragmentChecker::new(),
         })
     }
 }
@@ -435,9 +433,6 @@ pub struct Client {
 
     /// A checker for email URLs.
     email_checker: MailChecker,
-
-    /// Caches Fragments
-    fragment_checker: FragmentChecker,
 }
 
 impl Client {
@@ -535,24 +530,6 @@ impl Client {
     pub async fn check_mail(&self, uri: &Uri) -> Status {
         self.email_checker.check_mail(uri).await
     }
-
-    /// Checks a `file` URI's fragment.
-    pub async fn check_fragment(&self, path: &Path, uri: &Uri) -> Status {
-        match FragmentInput::from_path(path).await {
-            Ok(input) => match self.fragment_checker.check(input, &uri.url).await {
-                Ok(true) => Status::Ok(StatusCode::OK),
-                Ok(false) => ErrorKind::InvalidFragment(uri.clone()).into(),
-                Err(err) => {
-                    warn!("Skipping fragment check due to the following error: {err}");
-                    Status::Ok(StatusCode::OK)
-                }
-            },
-            Err(err) => {
-                warn!("Skipping fragment check due to the following error: {err}");
-                Status::Ok(StatusCode::OK)
-            }
-        }
-    }
 }
 
 /// A shorthand function to check a single URI.

lychee-lib/src/utils/fragment_checker.rs

@@ -6,7 +6,7 @@ use std::{
 
 use crate::{
     extract::{html::html5gum::extract_html_fragments, markdown::extract_markdown_fragments},
-    types::FileType,
+    types::{ErrorKind, FileType},
     Result,
 };
 use percent_encoding::percent_decode_str;
@@ -21,7 +21,9 @@ pub(crate) struct FragmentInput {
 
 impl FragmentInput {
     pub(crate) async fn from_path(path: &Path) -> Result<Self> {
-        let content = fs::read_to_string(path).await?;
+        let content = fs::read_to_string(path)
+            .await
+            .map_err(|err| ErrorKind::ReadFileInput(err, path.to_path_buf()))?;
         let file_type = FileType::from(path);
         Ok(Self { content, file_type })
     }