Hopiu/lychee
1
0
Fork 0
mirror of https://github.com/Hopiu/lychee.git synced 2026-04-05 22:10:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add support for basic auth #18 (#20)

Browse source 
[Issue #18](https://github.com/hello-rust/lychee/issues/18)
* Add headers crate to type headers and create auth header
* Add cmd param basic-auth to set property to the main
* Add simple test to test if with auth headres is no broken

Signed-off-by: FabianBG <f4b4g3@gmail.com>
This commit is contained in:
Milton Fabian Bastidas Guerra 2020-10-26 03:23:45 -05:00 committed by GitHub
parent f0e4c3adc1
commit 2bf62e7709
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 142 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

109
Cargo.lock generated
View file

@ -21,7 +21,7 @@ version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fc95d1bdb8e6666b2b217308eeeb09f2d6728d104be3e31916cc74d15420331"
dependencies = [
"generic-array",
"generic-array 0.14.4",
]
[[package]]
@ -455,13 +455,25 @@ version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
[[package]]
name = "block-buffer"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b"
dependencies = [
"block-padding",
"byte-tools",
"byteorder",
"generic-array 0.12.3",
]
[[package]]
name = "block-buffer"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4"
dependencies = [
"generic-array",
"generic-array 0.14.4",
]
[[package]]
@ -470,7 +482,16 @@ version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa136449e765dc7faa244561ccae839c394048667929af599b5d931ebe7b7f10"
dependencies = [
"generic-array",
"generic-array 0.14.4",
]
[[package]]
name = "block-padding"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5"
dependencies = [
"byte-tools",
]
[[package]]
@ -508,6 +529,12 @@ dependencies = [
"stable_deref_trait",
]
[[package]]
name = "byte-tools"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7"
[[package]]
name = "byteorder"
version = "1.3.4"
@ -736,7 +763,7 @@ version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab"
dependencies = [
"generic-array",
"generic-array 0.14.4",
"subtle",
]
@ -752,13 +779,22 @@ version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198"
[[package]]
name = "digest"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5"
dependencies = [
"generic-array 0.12.3",
]
[[package]]
name = "digest"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066"
dependencies = [
"generic-array",
"generic-array 0.14.4",
]
[[package]]
@ -831,6 +867,12 @@ version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e1cd41440ae7e4734bbd42302f63eaba892afc93a3912dad84006247f0dedb0e"
[[package]]
name = "fake-simd"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed"
[[package]]
name = "fast-socks5"
version = "0.3.1"
@ -1056,6 +1098,15 @@ dependencies = [
"byteorder",
]
[[package]]
name = "generic-array"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec"
dependencies = [
"typenum",
]
[[package]]
name = "generic-array"
version = "0.14.4"
@ -1148,6 +1199,31 @@ dependencies = [
"autocfg",
]
[[package]]
name = "headers"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed18eb2459bf1a09ad2d6b1547840c3e5e62882fa09b9a6a20b1de8e3228848f"
dependencies = [
"base64 0.12.3",
"bitflags",
"bytes 0.5.6",
"headers-core",
"http",
"mime",
"sha-1",
"time 0.1.43",
]
[[package]]
name = "headers-core"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429"
dependencies = [
"http",
]
[[package]]
name = "heck"
version = "0.3.1"
@ -1172,7 +1248,7 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe1149865383e4526a43aee8495f9a325f0b806c63ce6427d06336a590abbbc9"
dependencies = [
"digest",
"digest 0.9.0",
"hmac",
]
@ -1183,7 +1259,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "126888268dcc288495a26bf004b38c5fdbb31682f992c84ceb046a1f0fe38840"
dependencies = [
"crypto-mac",
"digest",
"digest 0.9.0",
]
[[package]]
@ -1566,6 +1642,7 @@ dependencies = [
"check-if-email-exists",
"futures",
"glob",
"headers",
"http",
"hubcaps",
"indicatif",
@ -2444,6 +2521,18 @@ dependencies = [
"serde",
]
[[package]]
name = "sha-1"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7d94d0bede923b3cea61f3f1ff57ff8cdfd77b400fb8f9998949e0cf04163df"
dependencies = [
"block-buffer 0.7.3",
"digest 0.8.1",
"fake-simd",
"opaque-debug 0.2.3",
]
[[package]]
name = "sha1"
version = "0.6.0"
@ -2456,10 +2545,10 @@ version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2933378ddfeda7ea26f48c555bdad8bb446bf8a3d17832dc83e380d444cfb8c1"
dependencies = [
"block-buffer",
"block-buffer 0.9.0",
"cfg-if",
"cpuid-bool",
"digest",
"digest 0.9.0",
"opaque-debug 0.3.0",
]
@ -3043,7 +3132,7 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8326b2c654932e3e4f9196e69d08fdf7cfd718e1dc6f66b347e6024a0c961402"
dependencies = [
"generic-array",
"generic-array 0.14.4",
"subtle",
]

1
Cargo.toml
View file

@ -27,6 +27,7 @@ toml = "0.5.7"
serde = { version = "1.0", features = ["derive"] }
pulldown-cmark = "0.8.0"
quick-xml = "0.20.0"
headers = "0.3.2"
[dependencies.reqwest]
features = ["gzip"]

6
src/checker.rs
View file

@ -5,10 +5,11 @@ use crate::{
use anyhow::anyhow;
use anyhow::{Context, Result};
use check_if_email_exists::{check_email, CheckEmailInput};
use headers::{HeaderMap, HeaderValue};
use hubcaps::{Credentials, Github};
use indicatif::ProgressBar;
use regex::{Regex, RegexSet};
use reqwest::header::{self, HeaderMap, HeaderValue};
use reqwest::header;
use std::net::IpAddr;
use std::{collections::HashSet, convert::TryFrom, time::Duration};
use tokio::time::delay_for;
@ -139,12 +140,11 @@ impl<'a> Checker<'a> {
verbose: bool,
progress_bar: Option<&'a ProgressBar>,
) -> Result<Self> {
let mut headers = header::HeaderMap::new();
let mut headers = HeaderMap::new();
// Faking the user agent is necessary for some websites, unfortunately.
// Otherwise we get a 403 from the firewall (e.g. Sucuri/Cloudproxy on ldra.com).
headers.insert(header::USER_AGENT, HeaderValue::from_str(&user_agent)?);
headers.insert(header::TRANSFER_ENCODING, HeaderValue::from_str("chunked")?);
headers.extend(custom_headers);
let builder = reqwest::ClientBuilder::new()

36
src/main.rs
View file

@ -4,9 +4,10 @@ extern crate log;
use anyhow::anyhow;
use anyhow::Result;
use futures::future::join_all;
use headers::authorization::Basic;
use headers::{Authorization, HeaderMap, HeaderMapExt, HeaderName};
use indicatif::{ProgressBar, ProgressStyle};
use regex::RegexSet;
use reqwest::header::{HeaderMap, HeaderName};
use std::{collections::HashSet, convert::TryInto, env, time::Duration};
use structopt::StructOpt;
@ -69,7 +70,13 @@ fn main() -> Result<()> {
async fn run(cfg: Config, inputs: Vec<String>) -> Result<i32> {
let includes = RegexSet::new(&cfg.include).ok();
let excludes = Excludes::from_options(&cfg);
let headers = parse_headers(cfg.headers)?;
let mut headers = parse_headers(cfg.headers)?;
if let Some(auth) = cfg.basic_auth {
let auth_header = parse_basic_auth(&auth)?;
headers.typed_insert(auth_header);
}
let accepted = match cfg.accept {
Some(accept) => parse_statuscodes(accept)?,
None => None,
@ -155,6 +162,17 @@ fn parse_statuscodes(accept: String) -> Result<Option<HashSet<http::StatusCode>>
Ok(Some(statuscodes))
}
fn parse_basic_auth(auth: &str) -> Result<Authorization<Basic>> {
let params: Vec<_> = auth.split(':').collect();
if params.len() != 2 {
return Err(anyhow!(
"Basic auth value should be of the form username:password, got {}",
auth
));
}
Ok(Authorization::basic(params[0], params[1]))
}
#[cfg(test)]
mod test {
use super::*;
@ -186,4 +204,18 @@ mod test {
);
assert_eq!(actual, expected);
}
#[test]
fn test_parse_basic_auth() {
let mut expected = HeaderMap::new();
expected.insert(
header::AUTHORIZATION,
"Basic YWxhZGluOmFicmV0ZXNlc2Ftbw==".parse().unwrap(),
);
let mut actual = HeaderMap::new();
let auth_header = parse_basic_auth("aladin:abretesesamo").unwrap();
actual.typed_insert(auth_header);
assert_eq!(expected, actual);
}
}

5
src/options.rs
View file

@ -138,6 +138,10 @@ pub(crate) struct Config {
#[structopt(short, long, help = "Base URL to check relative URls")]
#[serde(default)]
pub base_url: Option<String>,
#[structopt(long, help = "Basic autentication support. Ex 'username:password'")]
#[serde(default)]
pub basic_auth: Option<String>,
}
impl Config {
@ -188,6 +192,7 @@ impl Config {
timeout: TIMEOUT;
method: METHOD;
base_url: None;
basic_auth: None;
}
self