From 21ea0fd033a9dad7319231978af3a6d549fad8c1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 12 Sep 2021 18:10:23 +0200 Subject: [PATCH] Add support for tokio-console (#318) This allows troubleshooting and improving async Rust code. It is an optional feature that is still experimental (but can be quite helpful) --- Cargo.lock | 395 +++++++++++++++++++++++++++++++++++++++-- Makefile | 2 +- README.md | 23 +++ lychee-bin/Cargo.toml | 11 +- lychee-bin/src/main.rs | 4 +- lychee-lib/src/lib.rs | 2 +- 6 files changed, 420 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d13478f..d5ddae2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -202,7 +202,7 @@ dependencies = [ "fast_chemail", "hostname 0.1.5", "log", - "nom", + "nom 5.1.2", "pin-project", "pin-utils", "serde", @@ -253,6 +253,27 @@ dependencies = [ "trust-dns-resolver", ] +[[package]] +name = "async-stream" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "171374e7e3b2504e0e5236e3b59260560f9fe94bfe9ac39ba5e4e929c5590625" +dependencies = [ + "async-stream-impl", + "futures-core", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "648ed8c8d2ce5409ccd57453d9d1b214b342a0d69376a6feda1fd6cae3299308" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "async-task" version = "4.0.3" @@ -311,6 +332,18 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +[[package]] +name = "bitvec" +version = "0.19.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8942c8d352ae1838c9dda0b0ca2ab657696ef2232a20147cf1b30ae1a9cb4321" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "block-buffer" version = "0.9.0" @@ -482,7 +515,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19b076e143e1d9538dde65da30f8481c2a6c44040edb8e02b9bf1351edb92ce3" dependencies = [ "lazy_static", - "nom", + "nom 5.1.2", "serde", ] @@ -501,6 +534,37 @@ dependencies = [ "winapi", ] +[[package]] +name = "console-api" +version = "0.1.0" +source = "git+https://github.com/tokio-rs/console?rev=926de99ce4cbfd02c87190f9ec5f1c60b5c305d5#926de99ce4cbfd02c87190f9ec5f1c60b5c305d5" +dependencies = [ + "prost", + "prost-types", + "tonic", + "tonic-build", + "tracing-core", +] + +[[package]] +name = "console-subscriber" +version = "0.1.0" +source = "git+https://github.com/tokio-rs/console?rev=926de99ce4cbfd02c87190f9ec5f1c60b5c305d5#926de99ce4cbfd02c87190f9ec5f1c60b5c305d5" +dependencies = [ + "console-api", + "futures", + "hdrhistogram", + "serde", + "serde_json", + "thread_local", + "tokio", + "tokio-stream", + "tonic", + "tracing", + "tracing-core", + "tracing-subscriber", +] + [[package]] name = "core-foundation" version = "0.9.1" @@ -707,6 +771,12 @@ dependencies = [ "instant", ] +[[package]] +name = "fixedbitset" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d" + [[package]] name = "flate2" version = "1.0.20" @@ -759,6 +829,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "funty" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7" + [[package]] name = "futf" version = "0.1.4" @@ -969,6 +1045,19 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" +[[package]] +name = "hdrhistogram" +version = "7.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faa51471caf8069812385974ce947bf4b71a806d7e5a0d1f710af57d6a9a45ad" +dependencies = [ + "base64 0.13.0", + "byteorder", + "flate2", + "nom 6.2.1", + "num-traits", +] + [[package]] name = "headers" version = "0.3.4" @@ -1060,9 +1149,9 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.1" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfb77c123b4e2f72a2069aeae0b4b4949cc7e966df277813fc16347e7549737" +checksum = "399c583b2979440c60be0821a6199eca73bc3c8dcd9d070d75ac726e2c6186e5" dependencies = [ "bytes", "http", @@ -1147,6 +1236,18 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + [[package]] name = "hyper-tls" version = "0.5.0" @@ -1371,6 +1472,7 @@ dependencies = [ "anyhow", "assert_cmd", "console", + "console-subscriber", "headers", "http", "indicatif", @@ -1389,6 +1491,7 @@ dependencies = [ "tempfile", "tokio", "toml", + "tracing-subscriber", "uuid", "wiremock", ] @@ -1475,6 +1578,15 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" +[[package]] +name = "matchers" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f099785f7595cc4b4553a174ce30dd7589ef93391ff414dbb67f62392b9e0ce1" +dependencies = [ + "regex-automata", +] + [[package]] name = "matches" version = "0.1.8" @@ -1525,6 +1637,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "multimap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" + [[package]] name = "native-tls" version = "0.2.7" @@ -1570,6 +1688,19 @@ dependencies = [ "version_check", ] +[[package]] +name = "nom" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c5c51b9083a3c620fa67a2a635d1ce7d95b897e957d6b28ff9a5da960a103a6" +dependencies = [ + "bitvec", + "funty", + "lexical-core", + "memchr", + "version_check", +] + [[package]] name = "normalize-line-endings" version = "0.3.0" @@ -1758,6 +1889,16 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" +[[package]] +name = "petgraph" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "467d164a6de56270bd7c4d070df81d07beace25012d5103ced4e9ff08d6afdb7" +dependencies = [ + "fixedbitset", + "indexmap", +] + [[package]] name = "phf" version = "0.8.0" @@ -1956,6 +2097,57 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "prost" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de5e2533f59d08fcf364fd374ebda0692a70bd6d7e66ef97f306f45c6c5d8020" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "355f634b43cdd80724ee7848f95770e7e70eefa6dcf14fea676216573b8fd603" +dependencies = [ + "bytes", + "heck", + "itertools", + "log", + "multimap", + "petgraph", + "prost", + "prost-types", + "tempfile", + "which", +] + +[[package]] +name = "prost-derive" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "600d2f334aa05acb02a755e217ef1ab6dea4d51b58b7846588b747edec04efba" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-types" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "603bbd6394701d13f3f25aada59c7de9d35a6a5887cfc156181234a44002771b" +dependencies = [ + "bytes", + "prost", +] + [[package]] name = "pulldown-cmark" version = "0.8.0" @@ -1983,6 +2175,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "radium" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "941ba9d78d8e2f7ce474c015eea4d9c6d25b6a3327f9832ee29a4de27f91bbb8" + [[package]] name = "rand" version = "0.7.3" @@ -2111,6 +2309,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae1ded71d66a4a97f5e961fd0cb25a5f366a42a41570d16a763a69c092c26ae4" dependencies = [ "byteorder", + "regex-syntax", ] [[package]] @@ -2130,9 +2329,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.3" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2296f2fac53979e8ccbc4a1136b25dcefd37be9ed7e4a1f6b05a6029c84ff124" +checksum = "246e9f61b9bb77df069a947682be06e31ac43ea37862e244a69f177694ea6d22" dependencies = [ "async-compression", "base64 0.13.0", @@ -2303,6 +2502,15 @@ dependencies = [ "opaque-debug", ] +[[package]] +name = "sharded-slab" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "740223c51853f3145fe7c90360d2d4232f2b62e3449489c207eccde818979982" +dependencies = [ + "lazy_static", +] + [[package]] name = "shellexpand" version = "2.1.0" @@ -2467,6 +2675,12 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "tempfile" version = "3.2.0" @@ -2531,6 +2745,15 @@ dependencies = [ "syn", ] +[[package]] +name = "thread_local" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8018d24e04c95ac8790716a5987d0fec4f8b27249ffa0f7d33f1369bdfb88cbd" +dependencies = [ + "once_cell", +] + [[package]] name = "time" version = "0.1.43" @@ -2558,9 +2781,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.6.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd3076b5c8cc18138b8f8814895c11eb4de37114a5d127bafdc5e55798ceef37" +checksum = "b4efe6fc2395938c8155973d7be49fe8d03a843726e285e100a8a383cc0154ce" dependencies = [ "autocfg", "bytes", @@ -2573,9 +2796,20 @@ dependencies = [ "pin-project-lite", "signal-hook-registry", "tokio-macros", + "tracing", "winapi", ] +[[package]] +name = "tokio-io-timeout" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90c49f106be240de154571dd31fbe48acb10ba6c6dd6f6517ad603abffa42de9" +dependencies = [ + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-macros" version = "1.1.0" @@ -2609,6 +2843,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2f3f698253f03119ac0102beaa64f67a67e08074d03a22d18784104543727f" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-util" version = "0.6.6" @@ -2632,6 +2877,75 @@ dependencies = [ "serde", ] +[[package]] +name = "tonic" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "796c5e1cd49905e65dd8e700d4cb1dffcbfdb4fc9d017de08c1a537afd83627c" +dependencies = [ + "async-stream", + "async-trait", + "base64 0.13.0", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost", + "prost-derive", + "tokio", + "tokio-stream", + "tokio-util", + "tower", + "tower-layer", + "tower-service", + "tracing", + "tracing-futures", +] + +[[package]] +name = "tonic-build" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12b52d07035516c2b74337d2ac7746075e7dcae7643816c1b12c5ff8a7484c08" +dependencies = [ + "proc-macro2", + "prost-build", + "quote", + "syn", +] + +[[package]] +name = "tower" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60422bc7fefa2f3ec70359b8ff1caff59d785877eb70595904605bcc412470f" +dependencies = [ + "futures-core", + "futures-util", + "indexmap", + "pin-project", + "rand 0.8.3", + "slab", + "tokio", + "tokio-stream", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343bc9466d3fe6b0f960ef45960509f84480bf4fd96f92901afe7ff3df9d3a62" + [[package]] name = "tower-service" version = "0.3.1" @@ -2640,24 +2954,62 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" -version = "0.1.25" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01ebdc2bb4498ab1ab5f5b73c5803825e60199229ccba0698170e3be0e7f959f" +checksum = "09adeb8c97449311ccd28a427f96fb563e7fd31aabf994189879d9da2394b89d" dependencies = [ "cfg-if", + "log", "pin-project-lite", + "tracing-attributes", "tracing-core", ] [[package]] -name = "tracing-core" -version = "0.1.17" +name = "tracing-attributes" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f50de3927f93d202783f4513cda820ab47ef17f624b03c096e86ef00c67e6b5f" +checksum = "c42e6fa53307c8a17e4ccd4dc81cf5ec38db9209f59b222210375b54ee40d1e2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ca517f43f0fb96e0c3072ed5c275fe5eece87e8cb52f4a77b69226d3b1c9df8" dependencies = [ "lazy_static", ] +[[package]] +name = "tracing-futures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" +dependencies = [ + "pin-project", + "tracing", +] + +[[package]] +name = "tracing-subscriber" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9cbe87a2fa7e35900ce5de20220a582a9483a7063811defce79d7cbd59d4cfe" +dependencies = [ + "lazy_static", + "matchers", + "regex", + "sharded-slab", + "thread_local", + "tracing", + "tracing-core", +] + [[package]] name = "treeline" version = "0.1.0" @@ -2967,6 +3319,17 @@ dependencies = [ "cc", ] +[[package]] +name = "which" +version = "4.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea187a8ef279bc014ec368c27a920da2024d2a711109bfbe3440585d5cf27ad9" +dependencies = [ + "either", + "lazy_static", + "libc", +] + [[package]] name = "widestring" version = "0.4.3" @@ -3042,6 +3405,12 @@ dependencies = [ "tokio", ] +[[package]] +name = "wyz" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85e60b0d1b5f99db2556934e21937020776a5d31520bf169e851ac44e6420214" + [[package]] name = "xml5ever" version = "0.16.1" diff --git a/Makefile b/Makefile index a0b985d..109dddc 100644 --- a/Makefile +++ b/Makefile @@ -27,5 +27,5 @@ install: ## Install project locally cargo install --path lychee-bin .PHONY: run -run: ## Run Rust code locally +run: ## Run project locally cargo run diff --git a/README.md b/README.md index 587ae39..21289b4 100644 --- a/README.md +++ b/README.md @@ -326,6 +326,29 @@ cargo install cargo-publish-all cargo-publish-all --dry-run --yes # dry run release ``` +## Debugging and improving async code + +Lychee makes heavy use of async code to be resource-friendly while still being performant. +Async code can be difficult to troubleshoot with most tools, however. +Therefore we provide experimental support for [tokio-console](https://github.com/tokio-rs/console). +It provides a top(1)-like overview for async tasks! + +If you want to give it a spin, download and start the console: + +```sh +git clone https://github.com/tokio-rs/console +cd console +cargo run +``` + +Then run lychee with some special flags and features enabled. + +```sh +RUSTFLAGS="--cfg tokio_unstable" cargo run --features tokio-console -- ... +``` + +If you find a way to make lychee faster, please do reach out. + ## Troubleshooting and workarounds We collect a list of common workarounds for various websites in our [troubleshooting guide](./TROUBLESHOOTING.md). diff --git a/lychee-bin/Cargo.toml b/lychee-bin/Cargo.toml index 8e78b38..152a2df 100644 --- a/lychee-bin/Cargo.toml +++ b/lychee-bin/Cargo.toml @@ -27,7 +27,7 @@ lazy_static = "1.4.0" openssl-sys = "0.9.63" pad = "0.1.6" regex = "1.4.6" -reqwest = { version = "0.11.3", features = ["gzip"] } +reqwest = { version = "0.11.4", features = ["gzip"] } # Make build work on Apple Silicon. # See https://github.com/briansmith/ring/issues/1163 # This is necessary for the homebrew build @@ -46,6 +46,15 @@ pretty_assertions = "0.7.2" tempfile = "3.2.0" uuid = { version = "0.8.2", features = ["v4"] } wiremock = "0.5.7" +tracing-subscriber = { version = "0.2.17", default-features = false, features = ["fmt", "registry", "env-filter"] } + +[dependencies.console-subscriber] +optional = true +# console-subscriber is not yet published to crates.io +# pin to a specific git revision +git = "https://github.com/tokio-rs/console" +rev = "926de99ce4cbfd02c87190f9ec5f1c60b5c305d5" [features] +tokio-console = ["console-subscriber", "tracing-subscriber/registry"] vendored-openssl = ["openssl-sys/vendored"] diff --git a/lychee-bin/src/main.rs b/lychee-bin/src/main.rs index 6c115e3..e5636a7 100644 --- a/lychee-bin/src/main.rs +++ b/lychee-bin/src/main.rs @@ -46,7 +46,7 @@ #![warn(clippy::all, clippy::pedantic)] #![warn( absolute_paths_not_starting_with_crate, - invalid_html_tags, + rustdoc::invalid_html_tags, missing_copy_implementations, missing_debug_implementations, semicolon_in_expressions_from_macros, @@ -97,6 +97,8 @@ enum ExitCode { } fn main() -> Result<()> { + #[cfg(feature = "tokio-console")] + console_subscriber::init(); // std::process::exit doesn't guarantee that all destructors will be ran, // therefore we wrap "main" code in another function to guarantee that. // See: https://doc.rust-lang.org/stable/std/process/fn.exit.html diff --git a/lychee-lib/src/lib.rs b/lychee-lib/src/lib.rs index 22b76f8..58c0a6a 100644 --- a/lychee-lib/src/lib.rs +++ b/lychee-lib/src/lib.rs @@ -29,7 +29,7 @@ #![warn(clippy::all, clippy::pedantic)] #![warn( absolute_paths_not_starting_with_crate, - invalid_html_tags, + rustdoc::invalid_html_tags, missing_copy_implementations, missing_debug_implementations, semicolon_in_expressions_from_macros,