diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 3c13d1b..517d7d8 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -1,5 +1,3 @@ -name: Rust - on: push: branches: [ master ] @@ -10,13 +8,14 @@ env: CARGO_TERM_COLOR: always jobs: - build: - + build_and_test: runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Build - run: cargo build --verbose - - name: Run tests - run: cargo test --verbose + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + toolchain: stable + - uses: actions-rs/cargo@v1 + with: + command: build + args: --release --all-features diff --git a/Cargo.lock b/Cargo.lock index 3121068..c204266 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -473,6 +473,23 @@ dependencies = [ "cache-padded", ] +[[package]] +name = "console" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b1aacfaffdbff75be81c15a399b4bedf78aaefe840e8af1d299ac2ade885d2" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "regex", + "terminal_size", + "termios", + "unicode-width", + "winapi 0.3.9", + "winapi-util", +] + [[package]] name = "cookie" version = "0.12.0" @@ -572,6 +589,12 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd56b59865bce947ac5958779cfa508f6c3b9497cc762b7e24a12d11ccde2c4f" +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "encoding_rs" version = "0.8.23" @@ -1093,6 +1116,18 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "indicatif" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7baab56125e25686df467fe470785512329883aab42696d661247aca2a2896e4" +dependencies = [ + "console", + "lazy_static", + "number_prefix", + "regex", +] + [[package]] name = "infer" version = "0.1.7" @@ -1288,6 +1323,7 @@ dependencies = [ "gumdrop", "http", "hubcaps", + "indicatif", "linkify", "log", "pretty_env_logger", @@ -1546,6 +1582,12 @@ dependencies = [ "libc", ] +[[package]] +name = "number_prefix" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a" + [[package]] name = "object" version = "0.20.0" @@ -2116,6 +2158,25 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "terminal_size" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a14cd9f8c72704232f0bfc8455c0e861f0ad4eb60cc9ec8a170e231414c1e13" +dependencies = [ + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "termios" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f0fcee7b24a25675de40d5bb4de6e41b0df07bc9856295e7e2b3a3600c400c2" +dependencies = [ + "libc", +] + [[package]] name = "thiserror" version = "1.0.20" @@ -2391,6 +2452,12 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0" +[[package]] +name = "unicode-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" + [[package]] name = "unicode-xid" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index a1bf703..abb1f86 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ pretty_env_logger = "0.4" regex = "1.3.9" url = "2.1.1" check-if-email-exists = "0.8.13" +indicatif = "0.15.0" [dependencies.reqwest] features = ["gzip"] diff --git a/README.md b/README.md index 303e0a4..310ccfb 100644 --- a/README.md +++ b/README.md @@ -40,18 +40,20 @@ lychee can... (e.g. `--accept 200,204`) - accept a request timeout (`--timeout`) in seconds. Default is 20s. Set to 0 for no timeout. - check e-mail links using [check-if-mail-exists](https://github.com/amaurymartiny/check-if-email-exists) +- show the progress interactively with progress bar and in-flight requests (`--progress`) by @xiaochuanyu + SOON: - report output in HTML, SQL, CSV, XML, JSON, YAML... - automatically retry and backoff - check relative (`base-url` to set project root) -- show the progress interactively with progress bar and in-flight requests (`--progress`) - usable as a library (https://github.com/raviqqe/liche/issues/13) - exclude private domains (https://github.com/appscodelabs/liche/blob/a5102b0bf90203b467a4f3b4597d22cd83d94f99/url_checker.go) - recursion - extended statistics: request latency - lychee.toml +- use colored output (https://crates.io/crates/colored) ## Users diff --git a/src/checker.rs b/src/checker.rs index 71650f1..c2f4ec5 100644 --- a/src/checker.rs +++ b/src/checker.rs @@ -3,6 +3,7 @@ use anyhow::anyhow; use anyhow::{Context, Result}; use check_if_email_exists::{check_email, CheckEmailInput}; use hubcaps::{Credentials, Github}; +use indicatif::ProgressBar; use regex::{Regex, RegexSet}; use reqwest::header::{self, HeaderMap, HeaderValue}; use std::{collections::HashSet, convert::TryFrom, time::Duration}; @@ -69,7 +70,7 @@ impl From for Status { /// A link checker using an API token for Github links /// otherwise a normal HTTP client. -pub(crate) struct Checker { +pub(crate) struct Checker<'a> { reqwest_client: reqwest::Client, github: Github, excludes: Option, @@ -77,9 +78,10 @@ pub(crate) struct Checker { method: RequestMethod, accepted: Option>, verbose: bool, + progress_bar: Option<&'a ProgressBar>, } -impl Checker { +impl<'a> Checker<'a> { /// Creates a new link checker pub fn try_new( token: String, @@ -93,6 +95,7 @@ impl Checker { accepted: Option>, timeout: Option, verbose: bool, + progress_bar: Option<&'a ProgressBar>, ) -> Result { let mut headers = header::HeaderMap::new(); // Faking the user agent is necessary for some websites, unfortunately. @@ -127,6 +130,7 @@ impl Checker { method, accepted, verbose, + progress_bar, }) } @@ -210,11 +214,44 @@ impl Checker { uri.scheme() != self.scheme } + fn status_message(&self, status: &Status, uri: &Uri) -> Option { + match status { + Status::Ok(code) => { + if self.verbose { + Some(format!("✅{} [{}]", uri, code)) + } else { + None + } + } + Status::Failed(code) => Some(format!("🚫{} [{}]", uri, code)), + Status::Redirected => { + if self.verbose { + Some(format!("🔀️{}", uri)) + } else { + None + } + } + Status::Excluded => { + if self.verbose { + Some(format!("👻{}", uri)) + } else { + None + } + } + Status::Error(e) => Some(format!("⚡ {} ({})", uri, e)), + Status::Timeout => Some(format!("⌛{}", uri)), + } + } + pub async fn check(&self, uri: &extract::Uri) -> Status { if self.excluded(&uri) { return Status::Excluded; } + if let Some(pb) = self.progress_bar { + pb.set_message(&uri.to_string()); + } + let ret = match uri { Uri::Website(url) => self.check_real(url).await, Uri::Mail(address) => { @@ -228,32 +265,18 @@ impl Checker { } }; - match &ret { - Status::Ok(code) => { - if self.verbose { - println!("✅{} [{}]", uri, code); - } + if let Some(pb) = self.progress_bar { + pb.inc(1); + // regular println! inteferes with progress bar + if let Some(message) = self.status_message(&ret, uri) { + pb.println(message); } - Status::Failed(code) => { - println!("🚫{} [{}]", uri, code); + } else { + if let Some(message) = self.status_message(&ret, uri) { + println!("{}", message); } - Status::Redirected => { - if self.verbose { - println!("🔀️{}", uri); - } - } - Status::Excluded => { - if self.verbose { - println!("👻{}", uri); - } - } - Status::Error(e) => { - println!("⚡ {} ({})", uri, e); - } - Status::Timeout => { - println!("⌛{}", uri); - } - }; + } + ret } } @@ -267,7 +290,7 @@ mod test { use wiremock::matchers::method; use wiremock::{Mock, MockServer, ResponseTemplate}; - fn get_checker(allow_insecure: bool, custom_headers: HeaderMap) -> Checker { + fn get_checker(allow_insecure: bool, custom_headers: HeaderMap) -> Checker<'static> { let checker = Checker::try_new( "DUMMY_GITHUB_TOKEN".to_string(), None, @@ -280,6 +303,7 @@ mod test { None, None, false, + None, ) .unwrap(); checker @@ -410,6 +434,7 @@ mod test { None, None, false, + None, ) .unwrap(); assert_eq!( diff --git a/src/main.rs b/src/main.rs index 6aa51d8..d306026 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,6 +5,7 @@ use anyhow::anyhow; use anyhow::Result; use futures::future::join_all; use gumdrop::Options; +use indicatif::{ProgressBar, ProgressStyle}; use regex::RegexSet; use reqwest::header::{HeaderMap, HeaderName}; use std::{collections::HashSet, convert::TryInto, env, time::Duration}; @@ -66,7 +67,19 @@ async fn run(opts: LycheeOptions) -> Result { None => None, }; let timeout = parse_timeout(opts.timeout)?; - + let links = collector::collect_links(opts.inputs).await?; + let progress_bar = if opts.progress { + Some( + ProgressBar::new(links.len() as u64) + .with_style( + ProgressStyle::default_bar() + .template("{spinner:.green} [{elapsed_precise}] [{bar:40.cyan/blue}] {pos}/{len} ({eta}) {wide_msg}") + .progress_chars("#>-") + ) + ) + } else { + None + }; let checker = Checker::try_new( env::var("GITHUB_TOKEN")?, Some(excludes), @@ -79,15 +92,21 @@ async fn run(opts: LycheeOptions) -> Result { accepted, Some(timeout), opts.verbose, + progress_bar.as_ref(), )?; - let links = collector::collect_links(opts.inputs).await?; let futures: Vec<_> = links.iter().map(|l| checker.check(l)).collect(); let results = join_all(futures).await; + // note that prints may interfere progress bar so this must go before summary + if let Some(progress_bar) = progress_bar { + progress_bar.finish_and_clear(); + } + if opts.verbose { print_summary(&links, &results); } + Ok(results.iter().all(|r| r.is_success()) as i32) } diff --git a/src/options.rs b/src/options.rs index 9f6db52..ee0fb85 100644 --- a/src/options.rs +++ b/src/options.rs @@ -11,6 +11,9 @@ pub(crate) struct LycheeOptions { #[options(help = "Verbose program output")] pub verbose: bool, + #[options(help = "Show progress")] + pub progress: bool, + #[options(help = "Maximum number of allowed redirects", default = "10")] pub max_redirects: usize,