From 4499e1363a5b65b959cb556a2b9df858eaaa0d44 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 26 Aug 2020 15:31:29 +0200 Subject: [PATCH 1/8] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 7505e17..7ff2b28 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,7 @@ SOON: - exclude private domains (https://github.com/appscodelabs/liche/blob/a5102b0bf90203b467a4f3b4597d22cd83d94f99/url_checker.go) - recursion - extended statistics: request latency +- use colored output (https://crates.io/crates/colored) ## Users From df54ce1eefa52baa9d1d962c676ecfdfe4f25fb0 Mon Sep 17 00:00:00 2001 From: Xiaochuan Yu Date: Sat, 10 Oct 2020 00:31:28 -0400 Subject: [PATCH 2/8] Add progress bar --- Cargo.lock | 67 ++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 1 + src/checker.rs | 79 +++++++++++++++++++++++++++++++++----------------- src/main.rs | 25 ++++++++++++++-- src/options.rs | 3 ++ 5 files changed, 146 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3121068..c204266 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -473,6 +473,23 @@ dependencies = [ "cache-padded", ] +[[package]] +name = "console" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b1aacfaffdbff75be81c15a399b4bedf78aaefe840e8af1d299ac2ade885d2" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "regex", + "terminal_size", + "termios", + "unicode-width", + "winapi 0.3.9", + "winapi-util", +] + [[package]] name = "cookie" version = "0.12.0" @@ -572,6 +589,12 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd56b59865bce947ac5958779cfa508f6c3b9497cc762b7e24a12d11ccde2c4f" +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "encoding_rs" version = "0.8.23" @@ -1093,6 +1116,18 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "indicatif" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7baab56125e25686df467fe470785512329883aab42696d661247aca2a2896e4" +dependencies = [ + "console", + "lazy_static", + "number_prefix", + "regex", +] + [[package]] name = "infer" version = "0.1.7" @@ -1288,6 +1323,7 @@ dependencies = [ "gumdrop", "http", "hubcaps", + "indicatif", "linkify", "log", "pretty_env_logger", @@ -1546,6 +1582,12 @@ dependencies = [ "libc", ] +[[package]] +name = "number_prefix" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a" + [[package]] name = "object" version = "0.20.0" @@ -2116,6 +2158,25 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "terminal_size" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a14cd9f8c72704232f0bfc8455c0e861f0ad4eb60cc9ec8a170e231414c1e13" +dependencies = [ + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "termios" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f0fcee7b24a25675de40d5bb4de6e41b0df07bc9856295e7e2b3a3600c400c2" +dependencies = [ + "libc", +] + [[package]] name = "thiserror" version = "1.0.20" @@ -2391,6 +2452,12 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0" +[[package]] +name = "unicode-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" + [[package]] name = "unicode-xid" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index a1bf703..abb1f86 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ pretty_env_logger = "0.4" regex = "1.3.9" url = "2.1.1" check-if-email-exists = "0.8.13" +indicatif = "0.15.0" [dependencies.reqwest] features = ["gzip"] diff --git a/src/checker.rs b/src/checker.rs index 71650f1..c2f4ec5 100644 --- a/src/checker.rs +++ b/src/checker.rs @@ -3,6 +3,7 @@ use anyhow::anyhow; use anyhow::{Context, Result}; use check_if_email_exists::{check_email, CheckEmailInput}; use hubcaps::{Credentials, Github}; +use indicatif::ProgressBar; use regex::{Regex, RegexSet}; use reqwest::header::{self, HeaderMap, HeaderValue}; use std::{collections::HashSet, convert::TryFrom, time::Duration}; @@ -69,7 +70,7 @@ impl From for Status { /// A link checker using an API token for Github links /// otherwise a normal HTTP client. -pub(crate) struct Checker { +pub(crate) struct Checker<'a> { reqwest_client: reqwest::Client, github: Github, excludes: Option, @@ -77,9 +78,10 @@ pub(crate) struct Checker { method: RequestMethod, accepted: Option>, verbose: bool, + progress_bar: Option<&'a ProgressBar>, } -impl Checker { +impl<'a> Checker<'a> { /// Creates a new link checker pub fn try_new( token: String, @@ -93,6 +95,7 @@ impl Checker { accepted: Option>, timeout: Option, verbose: bool, + progress_bar: Option<&'a ProgressBar>, ) -> Result { let mut headers = header::HeaderMap::new(); // Faking the user agent is necessary for some websites, unfortunately. @@ -127,6 +130,7 @@ impl Checker { method, accepted, verbose, + progress_bar, }) } @@ -210,11 +214,44 @@ impl Checker { uri.scheme() != self.scheme } + fn status_message(&self, status: &Status, uri: &Uri) -> Option { + match status { + Status::Ok(code) => { + if self.verbose { + Some(format!("✅{} [{}]", uri, code)) + } else { + None + } + } + Status::Failed(code) => Some(format!("🚫{} [{}]", uri, code)), + Status::Redirected => { + if self.verbose { + Some(format!("🔀️{}", uri)) + } else { + None + } + } + Status::Excluded => { + if self.verbose { + Some(format!("👻{}", uri)) + } else { + None + } + } + Status::Error(e) => Some(format!("⚡ {} ({})", uri, e)), + Status::Timeout => Some(format!("⌛{}", uri)), + } + } + pub async fn check(&self, uri: &extract::Uri) -> Status { if self.excluded(&uri) { return Status::Excluded; } + if let Some(pb) = self.progress_bar { + pb.set_message(&uri.to_string()); + } + let ret = match uri { Uri::Website(url) => self.check_real(url).await, Uri::Mail(address) => { @@ -228,32 +265,18 @@ impl Checker { } }; - match &ret { - Status::Ok(code) => { - if self.verbose { - println!("✅{} [{}]", uri, code); - } + if let Some(pb) = self.progress_bar { + pb.inc(1); + // regular println! inteferes with progress bar + if let Some(message) = self.status_message(&ret, uri) { + pb.println(message); } - Status::Failed(code) => { - println!("🚫{} [{}]", uri, code); + } else { + if let Some(message) = self.status_message(&ret, uri) { + println!("{}", message); } - Status::Redirected => { - if self.verbose { - println!("🔀️{}", uri); - } - } - Status::Excluded => { - if self.verbose { - println!("👻{}", uri); - } - } - Status::Error(e) => { - println!("⚡ {} ({})", uri, e); - } - Status::Timeout => { - println!("⌛{}", uri); - } - }; + } + ret } } @@ -267,7 +290,7 @@ mod test { use wiremock::matchers::method; use wiremock::{Mock, MockServer, ResponseTemplate}; - fn get_checker(allow_insecure: bool, custom_headers: HeaderMap) -> Checker { + fn get_checker(allow_insecure: bool, custom_headers: HeaderMap) -> Checker<'static> { let checker = Checker::try_new( "DUMMY_GITHUB_TOKEN".to_string(), None, @@ -280,6 +303,7 @@ mod test { None, None, false, + None, ) .unwrap(); checker @@ -410,6 +434,7 @@ mod test { None, None, false, + None, ) .unwrap(); assert_eq!( diff --git a/src/main.rs b/src/main.rs index 6aa51d8..781f6c5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,3 +1,5 @@ +#![type_length_limit = "7912782"] + #[macro_use] extern crate log; @@ -5,6 +7,7 @@ use anyhow::anyhow; use anyhow::Result; use futures::future::join_all; use gumdrop::Options; +use indicatif::{ProgressBar, ProgressStyle}; use regex::RegexSet; use reqwest::header::{HeaderMap, HeaderName}; use std::{collections::HashSet, convert::TryInto, env, time::Duration}; @@ -66,7 +69,19 @@ async fn run(opts: LycheeOptions) -> Result { None => None, }; let timeout = parse_timeout(opts.timeout)?; - + let links = collector::collect_links(opts.inputs).await?; + let progress_bar = if opts.progress { + Some( + ProgressBar::new(links.len() as u64) + .with_style( + ProgressStyle::default_bar() + .template("{spinner:.green} [{elapsed_precise}] [{bar:40.cyan/blue}] {pos}/{len} ({eta}) {wide_msg}") + .progress_chars("#>-") + ) + ) + } else { + None + }; let checker = Checker::try_new( env::var("GITHUB_TOKEN")?, Some(excludes), @@ -79,15 +94,21 @@ async fn run(opts: LycheeOptions) -> Result { accepted, Some(timeout), opts.verbose, + progress_bar.as_ref(), )?; - let links = collector::collect_links(opts.inputs).await?; let futures: Vec<_> = links.iter().map(|l| checker.check(l)).collect(); let results = join_all(futures).await; + // note that prints may interfere progress bar so this must go before summary + if let Some(progress_bar) = progress_bar { + progress_bar.finish_and_clear(); + } + if opts.verbose { print_summary(&links, &results); } + Ok(results.iter().all(|r| r.is_success()) as i32) } diff --git a/src/options.rs b/src/options.rs index 9f6db52..200ed42 100644 --- a/src/options.rs +++ b/src/options.rs @@ -11,6 +11,9 @@ pub(crate) struct LycheeOptions { #[options(help = "Verbose program output")] pub verbose: bool, + #[options(help = "Show progress", default = "true")] + pub progress: bool, + #[options(help = "Maximum number of allowed redirects", default = "10")] pub max_redirects: usize, From e80cb70a9862a17197ca2a58bf222525175f0bee Mon Sep 17 00:00:00 2001 From: Xiaochuan Yu Date: Sat, 10 Oct 2020 11:24:56 -0400 Subject: [PATCH 3/8] default true flags don't work? --- src/options.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/options.rs b/src/options.rs index 200ed42..ee0fb85 100644 --- a/src/options.rs +++ b/src/options.rs @@ -11,7 +11,7 @@ pub(crate) struct LycheeOptions { #[options(help = "Verbose program output")] pub verbose: bool, - #[options(help = "Show progress", default = "true")] + #[options(help = "Show progress")] pub progress: bool, #[options(help = "Maximum number of allowed redirects", default = "10")] From 096b8756cb9749210be82a4205fdc9c2adcb48b5 Mon Sep 17 00:00:00 2001 From: Xiaochuan Yu Date: Mon, 12 Oct 2020 23:40:26 -0400 Subject: [PATCH 4/8] use action-rs Actions instead --- .github/workflows/rust.yml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 3c13d1b..7234d1f 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -10,13 +10,15 @@ env: CARGO_TERM_COLOR: always jobs: - build: - + build_and_test: + name: Rust project runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Build - run: cargo build --verbose - - name: Run tests - run: cargo test --verbose + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + toolchain: stable + - uses: actions-rs/cargo@v1 + with: + command: build + args: --release --all-features From 292064ce901e1318d5c2763b7c7cfccf74414ba2 Mon Sep 17 00:00:00 2001 From: Xiaochuan Yu Date: Wed, 14 Oct 2020 19:13:40 -0400 Subject: [PATCH 5/8] Update rust.yml --- .github/workflows/rust.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 7234d1f..b57d366 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -1,5 +1,3 @@ -name: Rust - on: push: branches: [ master ] @@ -11,7 +9,7 @@ env: jobs: build_and_test: - name: Rust project + name: Build and test with Cargo runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 From 65e8a409737895a4058009033beb9f52034320f6 Mon Sep 17 00:00:00 2001 From: Xiaochuan Yu Date: Wed, 14 Oct 2020 19:19:14 -0400 Subject: [PATCH 6/8] remove names --- .github/workflows/rust.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index b57d366..517d7d8 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -9,7 +9,6 @@ env: jobs: build_and_test: - name: Build and test with Cargo runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 From 92b9ed11ca514e56d8b867664c8e9859fd0efe43 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 15 Oct 2020 01:58:35 +0200 Subject: [PATCH 7/8] Remove workaround for type recursion --- src/main.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main.rs b/src/main.rs index 781f6c5..d306026 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,3 @@ -#![type_length_limit = "7912782"] - #[macro_use] extern crate log; From 812c5972a3aa6004273a9fe512b95c7f9a3da91a Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 15 Oct 2020 02:13:39 +0200 Subject: [PATCH 8/8] Mention progress bar support by @xiaochuanyu --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7ff2b28..7a756cc 100644 --- a/README.md +++ b/README.md @@ -40,12 +40,13 @@ lychee can... (e.g. `--accept 200,204`) - accept a request timeout (`--timeout`) in seconds. Default is 20s. Set to 0 for no timeout. - check e-mail links using [check-if-mail-exists](https://github.com/amaurymartiny/check-if-email-exists) +- show the progress interactively with progress bar and in-flight requests (`--progress`) by @xiaochuanyu + SOON: - automatically retry and backoff - check relative (`base-url` to set project root) -- show the progress interactively with progress bar and in-flight requests (`--progress`) - usable as a library (https://github.com/raviqqe/liche/issues/13) - exclude private domains (https://github.com/appscodelabs/liche/blob/a5102b0bf90203b467a4f3b4597d22cd83d94f99/url_checker.go) - recursion