From e1755583766087fa0ff3bb3b35533d986422e7a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Romanowski?= Date: Sat, 17 Oct 2020 10:01:06 +0200 Subject: [PATCH] Add --exclude-all-private flag and cli integration test --- Cargo.lock | 86 ++++++++++++++++++++++++++++++++++++ Cargo.toml | 2 + fixtures/TEST_ALL_PRIVATE.md | 12 +++++ src/checker.rs | 10 +++-- src/options.rs | 12 +++-- tests/cli.rs | 30 +++++++++++++ 6 files changed, 146 insertions(+), 6 deletions(-) create mode 100644 fixtures/TEST_ALL_PRIVATE.md create mode 100644 tests/cli.rs diff --git a/Cargo.lock b/Cargo.lock index c204266..24e7e35 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -68,6 +68,19 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71938f30533e4d95a6d17aa530939da3842c2ab6f4f84b9dae68447e4129f74a" +[[package]] +name = "assert_cmd" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c88b9ca26f9c16ec830350d309397e74ee9abdfd8eb1f71cb6ecc71a3fc818da" +dependencies = [ + "doc-comment", + "predicates", + "predicates-core", + "predicates-tree", + "wait-timeout", +] + [[package]] name = "async-channel" version = "1.4.0" @@ -577,6 +590,18 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4d0e2d24e5ee3b23a01de38eefdcd978907890701f08ffffd4cb457ca4ee8d6" +[[package]] +name = "difference" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198" + +[[package]] +name = "doc-comment" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" + [[package]] name = "dtoa" version = "0.4.6" @@ -675,6 +700,15 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "float-cmp" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1267f4ac4f343772758f7b1bdcbe767c218bbab93bb432acbf5162bbf85a6c4" +dependencies = [ + "num-traits", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1317,6 +1351,7 @@ name = "lychee" version = "0.3.0" dependencies = [ "anyhow", + "assert_cmd", "check-if-email-exists", "futures", "glob", @@ -1326,6 +1361,7 @@ dependencies = [ "indicatif", "linkify", "log", + "predicates", "pretty_env_logger", "regex", "reqwest", @@ -1525,6 +1561,12 @@ dependencies = [ "version_check", ] +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + [[package]] name = "nuclei" version = "0.1.1" @@ -1753,6 +1795,35 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c36fa947111f5c62a733b652544dd0016a43ce89619538a8ef92724a6f501a20" +[[package]] +name = "predicates" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96bfead12e90dccead362d62bb2c90a5f6fc4584963645bc7f71a735e0b0735a" +dependencies = [ + "difference", + "float-cmp", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06075c3a3e92559ff8929e7a280684489ea27fe44805174c3ebd9328dcb37178" + +[[package]] +name = "predicates-tree" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e63c4859013b38a76eca2414c64911fba30def9e3202ac461a2d22831220124" +dependencies = [ + "predicates-core", + "treeline", +] + [[package]] name = "pretty_env_logger" version = "0.4.0" @@ -2374,6 +2445,12 @@ dependencies = [ "tracing-serde", ] +[[package]] +name = "treeline" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41" + [[package]] name = "trust-dns-proto" version = "0.19.5" @@ -2508,6 +2585,15 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" +[[package]] +name = "wait-timeout" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" +dependencies = [ + "libc", +] + [[package]] name = "waker-fn" version = "1.0.0" diff --git a/Cargo.toml b/Cargo.toml index abb1f86..e5631a3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,3 +34,5 @@ version = "0.2" [dev-dependencies] wiremock = "0.2.4" +assert_cmd = "1.0" +predicates = "1.0" diff --git a/fixtures/TEST_ALL_PRIVATE.md b/fixtures/TEST_ALL_PRIVATE.md new file mode 100644 index 0000000..30ddc0b --- /dev/null +++ b/fixtures/TEST_ALL_PRIVATE.md @@ -0,0 +1,12 @@ +Test file: "private" URLs (should all be excluded when using `-E` flag). + +- Loopback: http://127.0.0.1 +- Link-local 1: http://169.254.0.1 +- Link-local 2: https://169.254.10.1:8080 +- Private class A: http://10.0.1.1 +- Private class B: http://172.16.42.42 +- Private class C: http://192.168.10.1 + +IPv6: + +- Loopback: http://[::1] diff --git a/src/checker.rs b/src/checker.rs index 993563f..3206891 100644 --- a/src/checker.rs +++ b/src/checker.rs @@ -80,11 +80,15 @@ pub(crate) struct Excludes { impl Excludes { pub fn from_options(options: &LycheeOptions) -> Self { + // exclude_all_private option turns on all "private" excludes, + // including private IPs, link-local IPs and loopback IPs + let enable_exclude = |opt| opt || options.exclude_all_private; + Self { regex: RegexSet::new(&options.exclude).ok(), - private_ips: options.exclude_private, - link_local_ips: options.exclude_link_local, - loopback_ips: options.exclude_loopback, + private_ips: enable_exclude(options.exclude_private), + link_local_ips: enable_exclude(options.exclude_link_local), + loopback_ips: enable_exclude(options.exclude_loopback), } } } diff --git a/src/options.rs b/src/options.rs index fd6b28f..b1ec428 100644 --- a/src/options.rs +++ b/src/options.rs @@ -38,13 +38,19 @@ pub(crate) struct LycheeOptions { #[options(help = "Exclude URLs from checking (supports regex)")] pub exclude: Vec, - #[options(help = "Exclude private IP address ranges from checking")] + #[options( + help = "Exclude all private IPs from checking, equivalent to `--exclude-private --exclude-link-local --exclude--loopback`", + short = "E" + )] + pub exclude_all_private: bool, + + #[options(help = "Exclude private IP address ranges from checking", no_short)] pub exclude_private: bool, - #[options(help = "Exclude link-local IP address range from checking")] + #[options(help = "Exclude link-local IP address range from checking", no_short)] pub exclude_link_local: bool, - #[options(help = "Exclude loopback IP address range from checking")] + #[options(help = "Exclude loopback IP address range from checking", no_short)] pub exclude_loopback: bool, // Accumulate all headers in a vector diff --git a/tests/cli.rs b/tests/cli.rs new file mode 100644 index 0000000..b9ae43b --- /dev/null +++ b/tests/cli.rs @@ -0,0 +1,30 @@ +#[cfg(test)] +mod cli { + use assert_cmd::Command; + use predicates::str::contains; + use std::path::Path; + + #[test] + fn test_exclude_all_private() { + let mut cmd = + Command::cargo_bin(env!("CARGO_PKG_NAME")).expect("Couldn't get cargo package name"); + + let test_all_private_path = Path::new(module_path!()) + .parent() + .unwrap() + .join("fixtures") + .join("TEST_ALL_PRIVATE.md"); + + // assert that the command runs OK, and that it excluded all the links + cmd.env("GITHUB_TOKEN", "invalid-token") + .arg("--exclude-all-private") + .arg("--verbose") + .arg(test_all_private_path) + .assert() + .success() + .stdout(contains("Found: 7")) + .stdout(contains("Excluded: 7")) + .stdout(contains("Successful: 0")) + .stdout(contains("Errors: 0")); + } +}