Add progress bar

Cargo.lock

@@ -473,6 +473,23 @@ dependencies = [
  "cache-padded",
 ]
 
+[[package]]
+name = "console"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c0b1aacfaffdbff75be81c15a399b4bedf78aaefe840e8af1d299ac2ade885d2"
+dependencies = [
+ "encode_unicode",
+ "lazy_static",
+ "libc",
+ "regex",
+ "terminal_size",
+ "termios",
+ "unicode-width",
+ "winapi 0.3.9",
+ "winapi-util",
+]
+
 [[package]]
 name = "cookie"
 version = "0.12.0"
@@ -572,6 +589,12 @@ version = "1.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cd56b59865bce947ac5958779cfa508f6c3b9497cc762b7e24a12d11ccde2c4f"
 
+[[package]]
+name = "encode_unicode"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
+
 [[package]]
 name = "encoding_rs"
 version = "0.8.23"
@@ -1093,6 +1116,18 @@ dependencies = [
  "hashbrown",
 ]
 
+[[package]]
+name = "indicatif"
+version = "0.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7baab56125e25686df467fe470785512329883aab42696d661247aca2a2896e4"
+dependencies = [
+ "console",
+ "lazy_static",
+ "number_prefix",
+ "regex",
+]
+
 [[package]]
 name = "infer"
 version = "0.1.7"
@@ -1288,6 +1323,7 @@ dependencies = [
  "gumdrop",
  "http",
  "hubcaps",
+ "indicatif",
  "linkify",
  "log",
  "pretty_env_logger",
@@ -1546,6 +1582,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "number_prefix"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a"
+
 [[package]]
 name = "object"
 version = "0.20.0"
@@ -2116,6 +2158,25 @@ dependencies = [
  "winapi-util",
 ]
 
+[[package]]
+name = "terminal_size"
+version = "0.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a14cd9f8c72704232f0bfc8455c0e861f0ad4eb60cc9ec8a170e231414c1e13"
+dependencies = [
+ "libc",
+ "winapi 0.3.9",
+]
+
+[[package]]
+name = "termios"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6f0fcee7b24a25675de40d5bb4de6e41b0df07bc9856295e7e2b3a3600c400c2"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "thiserror"
 version = "1.0.20"
@@ -2391,6 +2452,12 @@ version = "1.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0"
 
+[[package]]
+name = "unicode-width"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3"
+
 [[package]]
 name = "unicode-xid"
 version = "0.2.1"

Cargo.toml

@@ -22,6 +22,7 @@ pretty_env_logger = "0.4"
 regex = "1.3.9"
 url = "2.1.1"
 check-if-email-exists = "0.8.13"
+indicatif = "0.15.0"
 
 [dependencies.reqwest]
 features = ["gzip"]

src/checker.rs

@@ -3,6 +3,7 @@ use anyhow::anyhow;
 use anyhow::{Context, Result};
 use check_if_email_exists::{check_email, CheckEmailInput};
 use hubcaps::{Credentials, Github};
+use indicatif::ProgressBar;
 use regex::{Regex, RegexSet};
 use reqwest::header::{self, HeaderMap, HeaderValue};
 use std::{collections::HashSet, convert::TryFrom, time::Duration};
@@ -69,7 +70,7 @@ impl From<reqwest::Error> for Status {
 
 /// A link checker using an API token for Github links
 /// otherwise a normal HTTP client.
-pub(crate) struct Checker {
+pub(crate) struct Checker<'a> {
     reqwest_client: reqwest::Client,
     github: Github,
     excludes: Option<RegexSet>,
@@ -77,9 +78,10 @@ pub(crate) struct Checker {
     method: RequestMethod,
     accepted: Option<HashSet<reqwest::StatusCode>>,
     verbose: bool,
+    progress_bar: Option<&'a ProgressBar>,
 }
 
-impl Checker {
+impl<'a> Checker<'a> {
     /// Creates a new link checker
     pub fn try_new(
         token: String,
@@ -93,6 +95,7 @@ impl Checker {
         accepted: Option<HashSet<http::StatusCode>>,
         timeout: Option<Duration>,
         verbose: bool,
+        progress_bar: Option<&'a ProgressBar>,
     ) -> Result<Self> {
         let mut headers = header::HeaderMap::new();
         // Faking the user agent is necessary for some websites, unfortunately.
@@ -127,6 +130,7 @@ impl Checker {
             method,
             accepted,
             verbose,
+            progress_bar,
         })
     }
 
@@ -210,11 +214,44 @@ impl Checker {
         uri.scheme() != self.scheme
     }
 
+    fn status_message(&self, status: &Status, uri: &Uri) -> Option<String> {
+        match status {
+            Status::Ok(code) => {
+                if self.verbose {
+                    Some(format!("✅{} [{}]", uri, code))
+                } else {
+                    None
+                }
+            }
+            Status::Failed(code) => Some(format!("🚫{} [{}]", uri, code)),
+            Status::Redirected => {
+                if self.verbose {
+                    Some(format!("🔀️{}", uri))
+                } else {
+                    None
+                }
+            }
+            Status::Excluded => {
+                if self.verbose {
+                    Some(format!("👻{}", uri))
+                } else {
+                    None
+                }
+            }
+            Status::Error(e) => Some(format!("⚡ {} ({})", uri, e)),
+            Status::Timeout => Some(format!("⌛{}", uri)),
+        }
+    }
+
     pub async fn check(&self, uri: &extract::Uri) -> Status {
         if self.excluded(&uri) {
             return Status::Excluded;
         }
 
+        if let Some(pb) = self.progress_bar {
+            pb.set_message(&uri.to_string());
+        }
+
         let ret = match uri {
             Uri::Website(url) => self.check_real(url).await,
             Uri::Mail(address) => {
@@ -228,32 +265,18 @@ impl Checker {
             }
         };
 
-        match &ret {
-            Status::Ok(code) => {
-                if self.verbose {
-                    println!("✅{} [{}]", uri, code);
-                }
+        if let Some(pb) = self.progress_bar {
+            pb.inc(1);
+            // regular println! inteferes with progress bar
+            if let Some(message) = self.status_message(&ret, uri) {
+                pb.println(message);
             }
-            Status::Failed(code) => {
-                println!("🚫{} [{}]", uri, code);
+        } else {
+            if let Some(message) = self.status_message(&ret, uri) {
+                println!("{}", message);
             }
-            Status::Redirected => {
-                if self.verbose {
-                    println!("🔀️{}", uri);
-                }
-            }
-            Status::Excluded => {
-                if self.verbose {
-                    println!("👻{}", uri);
-                }
-            }
-            Status::Error(e) => {
-                println!("⚡ {} ({})", uri, e);
-            }
-            Status::Timeout => {
-                println!("⌛{}", uri);
-            }
-        };
+        }
+
         ret
     }
 }
@@ -267,7 +290,7 @@ mod test {
     use wiremock::matchers::method;
     use wiremock::{Mock, MockServer, ResponseTemplate};
 
-    fn get_checker(allow_insecure: bool, custom_headers: HeaderMap) -> Checker {
+    fn get_checker(allow_insecure: bool, custom_headers: HeaderMap) -> Checker<'static> {
         let checker = Checker::try_new(
             "DUMMY_GITHUB_TOKEN".to_string(),
             None,
@@ -280,6 +303,7 @@ mod test {
             None,
             None,
             false,
+            None,
         )
         .unwrap();
         checker
@@ -410,6 +434,7 @@ mod test {
             None,
             None,
             false,
+            None,
         )
         .unwrap();
         assert_eq!(

src/main.rs

@@ -1,3 +1,5 @@
+#![type_length_limit = "7912782"]
+
 #[macro_use]
 extern crate log;
 
@@ -5,6 +7,7 @@ use anyhow::anyhow;
 use anyhow::Result;
 use futures::future::join_all;
 use gumdrop::Options;
+use indicatif::{ProgressBar, ProgressStyle};
 use regex::RegexSet;
 use reqwest::header::{HeaderMap, HeaderName};
 use std::{collections::HashSet, convert::TryInto, env, time::Duration};
@@ -66,7 +69,19 @@ async fn run(opts: LycheeOptions) -> Result<i32> {
         None => None,
     };
     let timeout = parse_timeout(opts.timeout)?;
-
+    let links = collector::collect_links(opts.inputs).await?;
+    let progress_bar = if opts.progress {
+        Some(
+            ProgressBar::new(links.len() as u64)
+            .with_style(
+                ProgressStyle::default_bar()
+                .template("{spinner:.green} [{elapsed_precise}] [{bar:40.cyan/blue}] {pos}/{len} ({eta}) {wide_msg}")
+                .progress_chars("#>-")
+            )
+        )
+    } else {
+        None
+    };
     let checker = Checker::try_new(
         env::var("GITHUB_TOKEN")?,
         Some(excludes),
@@ -79,15 +94,21 @@ async fn run(opts: LycheeOptions) -> Result<i32> {
         accepted,
         Some(timeout),
         opts.verbose,
+        progress_bar.as_ref(),
     )?;
 
-    let links = collector::collect_links(opts.inputs).await?;
     let futures: Vec<_> = links.iter().map(|l| checker.check(l)).collect();
     let results = join_all(futures).await;
 
+    // note that prints may interfere progress bar so this must go before summary
+    if let Some(progress_bar) = progress_bar {
+        progress_bar.finish_and_clear();
+    }
+
     if opts.verbose {
         print_summary(&links, &results);
     }
+
     Ok(results.iter().all(|r| r.is_success()) as i32)
 }
 

src/options.rs

@@ -11,6 +11,9 @@ pub(crate) struct LycheeOptions {
     #[options(help = "Verbose program output")]
     pub verbose: bool,
 
+    #[options(help = "Show progress", default = "true")]
+    pub progress: bool,
+
     #[options(help = "Maximum number of allowed redirects", default = "10")]
     pub max_redirects: usize,