diff --git a/Cargo.lock b/Cargo.lock index 3121068..c204266 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -473,6 +473,23 @@ dependencies = [ "cache-padded", ] +[[package]] +name = "console" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b1aacfaffdbff75be81c15a399b4bedf78aaefe840e8af1d299ac2ade885d2" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "regex", + "terminal_size", + "termios", + "unicode-width", + "winapi 0.3.9", + "winapi-util", +] + [[package]] name = "cookie" version = "0.12.0" @@ -572,6 +589,12 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd56b59865bce947ac5958779cfa508f6c3b9497cc762b7e24a12d11ccde2c4f" +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "encoding_rs" version = "0.8.23" @@ -1093,6 +1116,18 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "indicatif" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7baab56125e25686df467fe470785512329883aab42696d661247aca2a2896e4" +dependencies = [ + "console", + "lazy_static", + "number_prefix", + "regex", +] + [[package]] name = "infer" version = "0.1.7" @@ -1288,6 +1323,7 @@ dependencies = [ "gumdrop", "http", "hubcaps", + "indicatif", "linkify", "log", "pretty_env_logger", @@ -1546,6 +1582,12 @@ dependencies = [ "libc", ] +[[package]] +name = "number_prefix" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a" + [[package]] name = "object" version = "0.20.0" @@ -2116,6 +2158,25 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "terminal_size" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a14cd9f8c72704232f0bfc8455c0e861f0ad4eb60cc9ec8a170e231414c1e13" +dependencies = [ + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "termios" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f0fcee7b24a25675de40d5bb4de6e41b0df07bc9856295e7e2b3a3600c400c2" +dependencies = [ + "libc", +] + [[package]] name = "thiserror" version = "1.0.20" @@ -2391,6 +2452,12 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0" +[[package]] +name = "unicode-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" + [[package]] name = "unicode-xid" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index a1bf703..abb1f86 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ pretty_env_logger = "0.4" regex = "1.3.9" url = "2.1.1" check-if-email-exists = "0.8.13" +indicatif = "0.15.0" [dependencies.reqwest] features = ["gzip"] diff --git a/src/checker.rs b/src/checker.rs index 71650f1..c2f4ec5 100644 --- a/src/checker.rs +++ b/src/checker.rs @@ -3,6 +3,7 @@ use anyhow::anyhow; use anyhow::{Context, Result}; use check_if_email_exists::{check_email, CheckEmailInput}; use hubcaps::{Credentials, Github}; +use indicatif::ProgressBar; use regex::{Regex, RegexSet}; use reqwest::header::{self, HeaderMap, HeaderValue}; use std::{collections::HashSet, convert::TryFrom, time::Duration}; @@ -69,7 +70,7 @@ impl From for Status { /// A link checker using an API token for Github links /// otherwise a normal HTTP client. -pub(crate) struct Checker { +pub(crate) struct Checker<'a> { reqwest_client: reqwest::Client, github: Github, excludes: Option, @@ -77,9 +78,10 @@ pub(crate) struct Checker { method: RequestMethod, accepted: Option>, verbose: bool, + progress_bar: Option<&'a ProgressBar>, } -impl Checker { +impl<'a> Checker<'a> { /// Creates a new link checker pub fn try_new( token: String, @@ -93,6 +95,7 @@ impl Checker { accepted: Option>, timeout: Option, verbose: bool, + progress_bar: Option<&'a ProgressBar>, ) -> Result { let mut headers = header::HeaderMap::new(); // Faking the user agent is necessary for some websites, unfortunately. @@ -127,6 +130,7 @@ impl Checker { method, accepted, verbose, + progress_bar, }) } @@ -210,11 +214,44 @@ impl Checker { uri.scheme() != self.scheme } + fn status_message(&self, status: &Status, uri: &Uri) -> Option { + match status { + Status::Ok(code) => { + if self.verbose { + Some(format!("✅{} [{}]", uri, code)) + } else { + None + } + } + Status::Failed(code) => Some(format!("🚫{} [{}]", uri, code)), + Status::Redirected => { + if self.verbose { + Some(format!("🔀️{}", uri)) + } else { + None + } + } + Status::Excluded => { + if self.verbose { + Some(format!("👻{}", uri)) + } else { + None + } + } + Status::Error(e) => Some(format!("⚡ {} ({})", uri, e)), + Status::Timeout => Some(format!("⌛{}", uri)), + } + } + pub async fn check(&self, uri: &extract::Uri) -> Status { if self.excluded(&uri) { return Status::Excluded; } + if let Some(pb) = self.progress_bar { + pb.set_message(&uri.to_string()); + } + let ret = match uri { Uri::Website(url) => self.check_real(url).await, Uri::Mail(address) => { @@ -228,32 +265,18 @@ impl Checker { } }; - match &ret { - Status::Ok(code) => { - if self.verbose { - println!("✅{} [{}]", uri, code); - } + if let Some(pb) = self.progress_bar { + pb.inc(1); + // regular println! inteferes with progress bar + if let Some(message) = self.status_message(&ret, uri) { + pb.println(message); } - Status::Failed(code) => { - println!("🚫{} [{}]", uri, code); + } else { + if let Some(message) = self.status_message(&ret, uri) { + println!("{}", message); } - Status::Redirected => { - if self.verbose { - println!("🔀️{}", uri); - } - } - Status::Excluded => { - if self.verbose { - println!("👻{}", uri); - } - } - Status::Error(e) => { - println!("⚡ {} ({})", uri, e); - } - Status::Timeout => { - println!("⌛{}", uri); - } - }; + } + ret } } @@ -267,7 +290,7 @@ mod test { use wiremock::matchers::method; use wiremock::{Mock, MockServer, ResponseTemplate}; - fn get_checker(allow_insecure: bool, custom_headers: HeaderMap) -> Checker { + fn get_checker(allow_insecure: bool, custom_headers: HeaderMap) -> Checker<'static> { let checker = Checker::try_new( "DUMMY_GITHUB_TOKEN".to_string(), None, @@ -280,6 +303,7 @@ mod test { None, None, false, + None, ) .unwrap(); checker @@ -410,6 +434,7 @@ mod test { None, None, false, + None, ) .unwrap(); assert_eq!( diff --git a/src/main.rs b/src/main.rs index 6aa51d8..781f6c5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,3 +1,5 @@ +#![type_length_limit = "7912782"] + #[macro_use] extern crate log; @@ -5,6 +7,7 @@ use anyhow::anyhow; use anyhow::Result; use futures::future::join_all; use gumdrop::Options; +use indicatif::{ProgressBar, ProgressStyle}; use regex::RegexSet; use reqwest::header::{HeaderMap, HeaderName}; use std::{collections::HashSet, convert::TryInto, env, time::Duration}; @@ -66,7 +69,19 @@ async fn run(opts: LycheeOptions) -> Result { None => None, }; let timeout = parse_timeout(opts.timeout)?; - + let links = collector::collect_links(opts.inputs).await?; + let progress_bar = if opts.progress { + Some( + ProgressBar::new(links.len() as u64) + .with_style( + ProgressStyle::default_bar() + .template("{spinner:.green} [{elapsed_precise}] [{bar:40.cyan/blue}] {pos}/{len} ({eta}) {wide_msg}") + .progress_chars("#>-") + ) + ) + } else { + None + }; let checker = Checker::try_new( env::var("GITHUB_TOKEN")?, Some(excludes), @@ -79,15 +94,21 @@ async fn run(opts: LycheeOptions) -> Result { accepted, Some(timeout), opts.verbose, + progress_bar.as_ref(), )?; - let links = collector::collect_links(opts.inputs).await?; let futures: Vec<_> = links.iter().map(|l| checker.check(l)).collect(); let results = join_all(futures).await; + // note that prints may interfere progress bar so this must go before summary + if let Some(progress_bar) = progress_bar { + progress_bar.finish_and_clear(); + } + if opts.verbose { print_summary(&links, &results); } + Ok(results.iter().all(|r| r.is_success()) as i32) } diff --git a/src/options.rs b/src/options.rs index 9f6db52..200ed42 100644 --- a/src/options.rs +++ b/src/options.rs @@ -11,6 +11,9 @@ pub(crate) struct LycheeOptions { #[options(help = "Verbose program output")] pub verbose: bool, + #[options(help = "Show progress", default = "true")] + pub progress: bool, + #[options(help = "Maximum number of allowed redirects", default = "10")] pub max_redirects: usize,