From 2bf62e7709d7c0d3e2b716289ed69b7bced2c35c Mon Sep 17 00:00:00 2001 From: Milton Fabian Bastidas Guerra Date: Mon, 26 Oct 2020 03:23:45 -0500 Subject: [PATCH] Add support for basic auth #18 (#20) [Issue #18](https://github.com/hello-rust/lychee/issues/18) * Add headers crate to type headers and create auth header * Add cmd param basic-auth to set property to the main * Add simple test to test if with auth headres is no broken Signed-off-by: FabianBG --- Cargo.lock | 109 ++++++++++++++++++++++++++++++++++++++++++++----- Cargo.toml | 1 + src/checker.rs | 6 +-- src/main.rs | 36 +++++++++++++++- src/options.rs | 5 +++ 5 files changed, 142 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 63ad262..bcdabbe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -21,7 +21,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fc95d1bdb8e6666b2b217308eeeb09f2d6728d104be3e31916cc74d15420331" dependencies = [ - "generic-array", + "generic-array 0.14.4", ] [[package]] @@ -455,13 +455,25 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +[[package]] +name = "block-buffer" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" +dependencies = [ + "block-padding", + "byte-tools", + "byteorder", + "generic-array 0.12.3", +] + [[package]] name = "block-buffer" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" dependencies = [ - "generic-array", + "generic-array 0.14.4", ] [[package]] @@ -470,7 +482,16 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa136449e765dc7faa244561ccae839c394048667929af599b5d931ebe7b7f10" dependencies = [ - "generic-array", + "generic-array 0.14.4", +] + +[[package]] +name = "block-padding" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" +dependencies = [ + "byte-tools", ] [[package]] @@ -508,6 +529,12 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "byte-tools" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" + [[package]] name = "byteorder" version = "1.3.4" @@ -736,7 +763,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab" dependencies = [ - "generic-array", + "generic-array 0.14.4", "subtle", ] @@ -752,13 +779,22 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198" +[[package]] +name = "digest" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" +dependencies = [ + "generic-array 0.12.3", +] + [[package]] name = "digest" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" dependencies = [ - "generic-array", + "generic-array 0.14.4", ] [[package]] @@ -831,6 +867,12 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1cd41440ae7e4734bbd42302f63eaba892afc93a3912dad84006247f0dedb0e" +[[package]] +name = "fake-simd" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" + [[package]] name = "fast-socks5" version = "0.3.1" @@ -1056,6 +1098,15 @@ dependencies = [ "byteorder", ] +[[package]] +name = "generic-array" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec" +dependencies = [ + "typenum", +] + [[package]] name = "generic-array" version = "0.14.4" @@ -1148,6 +1199,31 @@ dependencies = [ "autocfg", ] +[[package]] +name = "headers" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed18eb2459bf1a09ad2d6b1547840c3e5e62882fa09b9a6a20b1de8e3228848f" +dependencies = [ + "base64 0.12.3", + "bitflags", + "bytes 0.5.6", + "headers-core", + "http", + "mime", + "sha-1", + "time 0.1.43", +] + +[[package]] +name = "headers-core" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429" +dependencies = [ + "http", +] + [[package]] name = "heck" version = "0.3.1" @@ -1172,7 +1248,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe1149865383e4526a43aee8495f9a325f0b806c63ce6427d06336a590abbbc9" dependencies = [ - "digest", + "digest 0.9.0", "hmac", ] @@ -1183,7 +1259,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "126888268dcc288495a26bf004b38c5fdbb31682f992c84ceb046a1f0fe38840" dependencies = [ "crypto-mac", - "digest", + "digest 0.9.0", ] [[package]] @@ -1566,6 +1642,7 @@ dependencies = [ "check-if-email-exists", "futures", "glob", + "headers", "http", "hubcaps", "indicatif", @@ -2444,6 +2521,18 @@ dependencies = [ "serde", ] +[[package]] +name = "sha-1" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d94d0bede923b3cea61f3f1ff57ff8cdfd77b400fb8f9998949e0cf04163df" +dependencies = [ + "block-buffer 0.7.3", + "digest 0.8.1", + "fake-simd", + "opaque-debug 0.2.3", +] + [[package]] name = "sha1" version = "0.6.0" @@ -2456,10 +2545,10 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2933378ddfeda7ea26f48c555bdad8bb446bf8a3d17832dc83e380d444cfb8c1" dependencies = [ - "block-buffer", + "block-buffer 0.9.0", "cfg-if", "cpuid-bool", - "digest", + "digest 0.9.0", "opaque-debug 0.3.0", ] @@ -3043,7 +3132,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8326b2c654932e3e4f9196e69d08fdf7cfd718e1dc6f66b347e6024a0c961402" dependencies = [ - "generic-array", + "generic-array 0.14.4", "subtle", ] diff --git a/Cargo.toml b/Cargo.toml index 4167e76..cd6d220 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,7 @@ toml = "0.5.7" serde = { version = "1.0", features = ["derive"] } pulldown-cmark = "0.8.0" quick-xml = "0.20.0" +headers = "0.3.2" [dependencies.reqwest] features = ["gzip"] diff --git a/src/checker.rs b/src/checker.rs index 487819c..78d414c 100644 --- a/src/checker.rs +++ b/src/checker.rs @@ -5,10 +5,11 @@ use crate::{ use anyhow::anyhow; use anyhow::{Context, Result}; use check_if_email_exists::{check_email, CheckEmailInput}; +use headers::{HeaderMap, HeaderValue}; use hubcaps::{Credentials, Github}; use indicatif::ProgressBar; use regex::{Regex, RegexSet}; -use reqwest::header::{self, HeaderMap, HeaderValue}; +use reqwest::header; use std::net::IpAddr; use std::{collections::HashSet, convert::TryFrom, time::Duration}; use tokio::time::delay_for; @@ -139,12 +140,11 @@ impl<'a> Checker<'a> { verbose: bool, progress_bar: Option<&'a ProgressBar>, ) -> Result { - let mut headers = header::HeaderMap::new(); + let mut headers = HeaderMap::new(); // Faking the user agent is necessary for some websites, unfortunately. // Otherwise we get a 403 from the firewall (e.g. Sucuri/Cloudproxy on ldra.com). headers.insert(header::USER_AGENT, HeaderValue::from_str(&user_agent)?); headers.insert(header::TRANSFER_ENCODING, HeaderValue::from_str("chunked")?); - headers.extend(custom_headers); let builder = reqwest::ClientBuilder::new() diff --git a/src/main.rs b/src/main.rs index 3b06ec5..fce9aa0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,9 +4,10 @@ extern crate log; use anyhow::anyhow; use anyhow::Result; use futures::future::join_all; +use headers::authorization::Basic; +use headers::{Authorization, HeaderMap, HeaderMapExt, HeaderName}; use indicatif::{ProgressBar, ProgressStyle}; use regex::RegexSet; -use reqwest::header::{HeaderMap, HeaderName}; use std::{collections::HashSet, convert::TryInto, env, time::Duration}; use structopt::StructOpt; @@ -69,7 +70,13 @@ fn main() -> Result<()> { async fn run(cfg: Config, inputs: Vec) -> Result { let includes = RegexSet::new(&cfg.include).ok(); let excludes = Excludes::from_options(&cfg); - let headers = parse_headers(cfg.headers)?; + let mut headers = parse_headers(cfg.headers)?; + + if let Some(auth) = cfg.basic_auth { + let auth_header = parse_basic_auth(&auth)?; + headers.typed_insert(auth_header); + } + let accepted = match cfg.accept { Some(accept) => parse_statuscodes(accept)?, None => None, @@ -155,6 +162,17 @@ fn parse_statuscodes(accept: String) -> Result> Ok(Some(statuscodes)) } +fn parse_basic_auth(auth: &str) -> Result> { + let params: Vec<_> = auth.split(':').collect(); + if params.len() != 2 { + return Err(anyhow!( + "Basic auth value should be of the form username:password, got {}", + auth + )); + } + Ok(Authorization::basic(params[0], params[1])) +} + #[cfg(test)] mod test { use super::*; @@ -186,4 +204,18 @@ mod test { ); assert_eq!(actual, expected); } + + #[test] + fn test_parse_basic_auth() { + let mut expected = HeaderMap::new(); + expected.insert( + header::AUTHORIZATION, + "Basic YWxhZGluOmFicmV0ZXNlc2Ftbw==".parse().unwrap(), + ); + + let mut actual = HeaderMap::new(); + let auth_header = parse_basic_auth("aladin:abretesesamo").unwrap(); + actual.typed_insert(auth_header); + assert_eq!(expected, actual); + } } diff --git a/src/options.rs b/src/options.rs index 6a21271..a39aa5a 100644 --- a/src/options.rs +++ b/src/options.rs @@ -138,6 +138,10 @@ pub(crate) struct Config { #[structopt(short, long, help = "Base URL to check relative URls")] #[serde(default)] pub base_url: Option, + + #[structopt(long, help = "Basic autentication support. Ex 'username:password'")] + #[serde(default)] + pub basic_auth: Option, } impl Config { @@ -188,6 +192,7 @@ impl Config { timeout: TIMEOUT; method: METHOD; base_url: None; + basic_auth: None; } self