Add support for tokio-console (#318)

This allows troubleshooting and improving async Rust code.
It is an optional feature that is still
experimental (but can be quite helpful)
This commit is contained in:
Matthias 2021-09-12 18:10:23 +02:00 committed by GitHub
parent 9b5fc399ed
commit 21ea0fd033
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 420 additions and 17 deletions

395
Cargo.lock generated
View file

@ -202,7 +202,7 @@ dependencies = [
"fast_chemail",
"hostname 0.1.5",
"log",
"nom",
"nom 5.1.2",
"pin-project",
"pin-utils",
"serde",
@ -253,6 +253,27 @@ dependencies = [
"trust-dns-resolver",
]
[[package]]
name = "async-stream"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "171374e7e3b2504e0e5236e3b59260560f9fe94bfe9ac39ba5e4e929c5590625"
dependencies = [
"async-stream-impl",
"futures-core",
]
[[package]]
name = "async-stream-impl"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "648ed8c8d2ce5409ccd57453d9d1b214b342a0d69376a6feda1fd6cae3299308"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "async-task"
version = "4.0.3"
@ -311,6 +332,18 @@ version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
[[package]]
name = "bitvec"
version = "0.19.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8942c8d352ae1838c9dda0b0ca2ab657696ef2232a20147cf1b30ae1a9cb4321"
dependencies = [
"funty",
"radium",
"tap",
"wyz",
]
[[package]]
name = "block-buffer"
version = "0.9.0"
@ -482,7 +515,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19b076e143e1d9538dde65da30f8481c2a6c44040edb8e02b9bf1351edb92ce3"
dependencies = [
"lazy_static",
"nom",
"nom 5.1.2",
"serde",
]
@ -501,6 +534,37 @@ dependencies = [
"winapi",
]
[[package]]
name = "console-api"
version = "0.1.0"
source = "git+https://github.com/tokio-rs/console?rev=926de99ce4cbfd02c87190f9ec5f1c60b5c305d5#926de99ce4cbfd02c87190f9ec5f1c60b5c305d5"
dependencies = [
"prost",
"prost-types",
"tonic",
"tonic-build",
"tracing-core",
]
[[package]]
name = "console-subscriber"
version = "0.1.0"
source = "git+https://github.com/tokio-rs/console?rev=926de99ce4cbfd02c87190f9ec5f1c60b5c305d5#926de99ce4cbfd02c87190f9ec5f1c60b5c305d5"
dependencies = [
"console-api",
"futures",
"hdrhistogram",
"serde",
"serde_json",
"thread_local",
"tokio",
"tokio-stream",
"tonic",
"tracing",
"tracing-core",
"tracing-subscriber",
]
[[package]]
name = "core-foundation"
version = "0.9.1"
@ -707,6 +771,12 @@ dependencies = [
"instant",
]
[[package]]
name = "fixedbitset"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d"
[[package]]
name = "flate2"
version = "1.0.20"
@ -759,6 +829,12 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "funty"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7"
[[package]]
name = "futf"
version = "0.1.4"
@ -969,6 +1045,19 @@ version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
[[package]]
name = "hdrhistogram"
version = "7.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "faa51471caf8069812385974ce947bf4b71a806d7e5a0d1f710af57d6a9a45ad"
dependencies = [
"base64 0.13.0",
"byteorder",
"flate2",
"nom 6.2.1",
"num-traits",
]
[[package]]
name = "headers"
version = "0.3.4"
@ -1060,9 +1149,9 @@ dependencies = [
[[package]]
name = "http-body"
version = "0.4.1"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5dfb77c123b4e2f72a2069aeae0b4b4949cc7e966df277813fc16347e7549737"
checksum = "399c583b2979440c60be0821a6199eca73bc3c8dcd9d070d75ac726e2c6186e5"
dependencies = [
"bytes",
"http",
@ -1147,6 +1236,18 @@ dependencies = [
"want",
]
[[package]]
name = "hyper-timeout"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1"
dependencies = [
"hyper",
"pin-project-lite",
"tokio",
"tokio-io-timeout",
]
[[package]]
name = "hyper-tls"
version = "0.5.0"
@ -1371,6 +1472,7 @@ dependencies = [
"anyhow",
"assert_cmd",
"console",
"console-subscriber",
"headers",
"http",
"indicatif",
@ -1389,6 +1491,7 @@ dependencies = [
"tempfile",
"tokio",
"toml",
"tracing-subscriber",
"uuid",
"wiremock",
]
@ -1475,6 +1578,15 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4"
[[package]]
name = "matchers"
version = "0.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f099785f7595cc4b4553a174ce30dd7589ef93391ff414dbb67f62392b9e0ce1"
dependencies = [
"regex-automata",
]
[[package]]
name = "matches"
version = "0.1.8"
@ -1525,6 +1637,12 @@ dependencies = [
"winapi",
]
[[package]]
name = "multimap"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a"
[[package]]
name = "native-tls"
version = "0.2.7"
@ -1570,6 +1688,19 @@ dependencies = [
"version_check",
]
[[package]]
name = "nom"
version = "6.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c5c51b9083a3c620fa67a2a635d1ce7d95b897e957d6b28ff9a5da960a103a6"
dependencies = [
"bitvec",
"funty",
"lexical-core",
"memchr",
"version_check",
]
[[package]]
name = "normalize-line-endings"
version = "0.3.0"
@ -1758,6 +1889,16 @@ version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
[[package]]
name = "petgraph"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "467d164a6de56270bd7c4d070df81d07beace25012d5103ced4e9ff08d6afdb7"
dependencies = [
"fixedbitset",
"indexmap",
]
[[package]]
name = "phf"
version = "0.8.0"
@ -1956,6 +2097,57 @@ dependencies = [
"unicode-xid",
]
[[package]]
name = "prost"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "de5e2533f59d08fcf364fd374ebda0692a70bd6d7e66ef97f306f45c6c5d8020"
dependencies = [
"bytes",
"prost-derive",
]
[[package]]
name = "prost-build"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "355f634b43cdd80724ee7848f95770e7e70eefa6dcf14fea676216573b8fd603"
dependencies = [
"bytes",
"heck",
"itertools",
"log",
"multimap",
"petgraph",
"prost",
"prost-types",
"tempfile",
"which",
]
[[package]]
name = "prost-derive"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "600d2f334aa05acb02a755e217ef1ab6dea4d51b58b7846588b747edec04efba"
dependencies = [
"anyhow",
"itertools",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "prost-types"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "603bbd6394701d13f3f25aada59c7de9d35a6a5887cfc156181234a44002771b"
dependencies = [
"bytes",
"prost",
]
[[package]]
name = "pulldown-cmark"
version = "0.8.0"
@ -1983,6 +2175,12 @@ dependencies = [
"proc-macro2",
]
[[package]]
name = "radium"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "941ba9d78d8e2f7ce474c015eea4d9c6d25b6a3327f9832ee29a4de27f91bbb8"
[[package]]
name = "rand"
version = "0.7.3"
@ -2111,6 +2309,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae1ded71d66a4a97f5e961fd0cb25a5f366a42a41570d16a763a69c092c26ae4"
dependencies = [
"byteorder",
"regex-syntax",
]
[[package]]
@ -2130,9 +2329,9 @@ dependencies = [
[[package]]
name = "reqwest"
version = "0.11.3"
version = "0.11.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2296f2fac53979e8ccbc4a1136b25dcefd37be9ed7e4a1f6b05a6029c84ff124"
checksum = "246e9f61b9bb77df069a947682be06e31ac43ea37862e244a69f177694ea6d22"
dependencies = [
"async-compression",
"base64 0.13.0",
@ -2303,6 +2502,15 @@ dependencies = [
"opaque-debug",
]
[[package]]
name = "sharded-slab"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "740223c51853f3145fe7c90360d2d4232f2b62e3449489c207eccde818979982"
dependencies = [
"lazy_static",
]
[[package]]
name = "shellexpand"
version = "2.1.0"
@ -2467,6 +2675,12 @@ dependencies = [
"unicode-xid",
]
[[package]]
name = "tap"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
[[package]]
name = "tempfile"
version = "3.2.0"
@ -2531,6 +2745,15 @@ dependencies = [
"syn",
]
[[package]]
name = "thread_local"
version = "1.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8018d24e04c95ac8790716a5987d0fec4f8b27249ffa0f7d33f1369bdfb88cbd"
dependencies = [
"once_cell",
]
[[package]]
name = "time"
version = "0.1.43"
@ -2558,9 +2781,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
[[package]]
name = "tokio"
version = "1.6.0"
version = "1.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd3076b5c8cc18138b8f8814895c11eb4de37114a5d127bafdc5e55798ceef37"
checksum = "b4efe6fc2395938c8155973d7be49fe8d03a843726e285e100a8a383cc0154ce"
dependencies = [
"autocfg",
"bytes",
@ -2573,9 +2796,20 @@ dependencies = [
"pin-project-lite",
"signal-hook-registry",
"tokio-macros",
"tracing",
"winapi",
]
[[package]]
name = "tokio-io-timeout"
version = "1.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90c49f106be240de154571dd31fbe48acb10ba6c6dd6f6517ad603abffa42de9"
dependencies = [
"pin-project-lite",
"tokio",
]
[[package]]
name = "tokio-macros"
version = "1.1.0"
@ -2609,6 +2843,17 @@ dependencies = [
"tokio",
]
[[package]]
name = "tokio-stream"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b2f3f698253f03119ac0102beaa64f67a67e08074d03a22d18784104543727f"
dependencies = [
"futures-core",
"pin-project-lite",
"tokio",
]
[[package]]
name = "tokio-util"
version = "0.6.6"
@ -2632,6 +2877,75 @@ dependencies = [
"serde",
]
[[package]]
name = "tonic"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "796c5e1cd49905e65dd8e700d4cb1dffcbfdb4fc9d017de08c1a537afd83627c"
dependencies = [
"async-stream",
"async-trait",
"base64 0.13.0",
"bytes",
"futures-core",
"futures-util",
"h2",
"http",
"http-body",
"hyper",
"hyper-timeout",
"percent-encoding",
"pin-project",
"prost",
"prost-derive",
"tokio",
"tokio-stream",
"tokio-util",
"tower",
"tower-layer",
"tower-service",
"tracing",
"tracing-futures",
]
[[package]]
name = "tonic-build"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12b52d07035516c2b74337d2ac7746075e7dcae7643816c1b12c5ff8a7484c08"
dependencies = [
"proc-macro2",
"prost-build",
"quote",
"syn",
]
[[package]]
name = "tower"
version = "0.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f60422bc7fefa2f3ec70359b8ff1caff59d785877eb70595904605bcc412470f"
dependencies = [
"futures-core",
"futures-util",
"indexmap",
"pin-project",
"rand 0.8.3",
"slab",
"tokio",
"tokio-stream",
"tokio-util",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
name = "tower-layer"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "343bc9466d3fe6b0f960ef45960509f84480bf4fd96f92901afe7ff3df9d3a62"
[[package]]
name = "tower-service"
version = "0.3.1"
@ -2640,24 +2954,62 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6"
[[package]]
name = "tracing"
version = "0.1.25"
version = "0.1.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "01ebdc2bb4498ab1ab5f5b73c5803825e60199229ccba0698170e3be0e7f959f"
checksum = "09adeb8c97449311ccd28a427f96fb563e7fd31aabf994189879d9da2394b89d"
dependencies = [
"cfg-if",
"log",
"pin-project-lite",
"tracing-attributes",
"tracing-core",
]
[[package]]
name = "tracing-core"
version = "0.1.17"
name = "tracing-attributes"
version = "0.1.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f50de3927f93d202783f4513cda820ab47ef17f624b03c096e86ef00c67e6b5f"
checksum = "c42e6fa53307c8a17e4ccd4dc81cf5ec38db9209f59b222210375b54ee40d1e2"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "tracing-core"
version = "0.1.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2ca517f43f0fb96e0c3072ed5c275fe5eece87e8cb52f4a77b69226d3b1c9df8"
dependencies = [
"lazy_static",
]
[[package]]
name = "tracing-futures"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2"
dependencies = [
"pin-project",
"tracing",
]
[[package]]
name = "tracing-subscriber"
version = "0.2.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9cbe87a2fa7e35900ce5de20220a582a9483a7063811defce79d7cbd59d4cfe"
dependencies = [
"lazy_static",
"matchers",
"regex",
"sharded-slab",
"thread_local",
"tracing",
"tracing-core",
]
[[package]]
name = "treeline"
version = "0.1.0"
@ -2967,6 +3319,17 @@ dependencies = [
"cc",
]
[[package]]
name = "which"
version = "4.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea187a8ef279bc014ec368c27a920da2024d2a711109bfbe3440585d5cf27ad9"
dependencies = [
"either",
"lazy_static",
"libc",
]
[[package]]
name = "widestring"
version = "0.4.3"
@ -3042,6 +3405,12 @@ dependencies = [
"tokio",
]
[[package]]
name = "wyz"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85e60b0d1b5f99db2556934e21937020776a5d31520bf169e851ac44e6420214"
[[package]]
name = "xml5ever"
version = "0.16.1"

View file

@ -27,5 +27,5 @@ install: ## Install project locally
cargo install --path lychee-bin
.PHONY: run
run: ## Run Rust code locally
run: ## Run project locally
cargo run

View file

@ -326,6 +326,29 @@ cargo install cargo-publish-all
cargo-publish-all --dry-run --yes # dry run release
```
## Debugging and improving async code
Lychee makes heavy use of async code to be resource-friendly while still being performant.
Async code can be difficult to troubleshoot with most tools, however.
Therefore we provide experimental support for [tokio-console](https://github.com/tokio-rs/console).
It provides a top(1)-like overview for async tasks!
If you want to give it a spin, download and start the console:
```sh
git clone https://github.com/tokio-rs/console
cd console
cargo run
```
Then run lychee with some special flags and features enabled.
```sh
RUSTFLAGS="--cfg tokio_unstable" cargo run --features tokio-console -- <input1> <input2> ...
```
If you find a way to make lychee faster, please do reach out.
## Troubleshooting and workarounds
We collect a list of common workarounds for various websites in our [troubleshooting guide](./TROUBLESHOOTING.md).

View file

@ -27,7 +27,7 @@ lazy_static = "1.4.0"
openssl-sys = "0.9.63"
pad = "0.1.6"
regex = "1.4.6"
reqwest = { version = "0.11.3", features = ["gzip"] }
reqwest = { version = "0.11.4", features = ["gzip"] }
# Make build work on Apple Silicon.
# See https://github.com/briansmith/ring/issues/1163
# This is necessary for the homebrew build
@ -46,6 +46,15 @@ pretty_assertions = "0.7.2"
tempfile = "3.2.0"
uuid = { version = "0.8.2", features = ["v4"] }
wiremock = "0.5.7"
tracing-subscriber = { version = "0.2.17", default-features = false, features = ["fmt", "registry", "env-filter"] }
[dependencies.console-subscriber]
optional = true
# console-subscriber is not yet published to crates.io
# pin to a specific git revision
git = "https://github.com/tokio-rs/console"
rev = "926de99ce4cbfd02c87190f9ec5f1c60b5c305d5"
[features]
tokio-console = ["console-subscriber", "tracing-subscriber/registry"]
vendored-openssl = ["openssl-sys/vendored"]

View file

@ -46,7 +46,7 @@
#![warn(clippy::all, clippy::pedantic)]
#![warn(
absolute_paths_not_starting_with_crate,
invalid_html_tags,
rustdoc::invalid_html_tags,
missing_copy_implementations,
missing_debug_implementations,
semicolon_in_expressions_from_macros,
@ -97,6 +97,8 @@ enum ExitCode {
}
fn main() -> Result<()> {
#[cfg(feature = "tokio-console")]
console_subscriber::init();
// std::process::exit doesn't guarantee that all destructors will be ran,
// therefore we wrap "main" code in another function to guarantee that.
// See: https://doc.rust-lang.org/stable/std/process/fn.exit.html

View file

@ -29,7 +29,7 @@
#![warn(clippy::all, clippy::pedantic)]
#![warn(
absolute_paths_not_starting_with_crate,
invalid_html_tags,
rustdoc::invalid_html_tags,
missing_copy_implementations,
missing_debug_implementations,
semicolon_in_expressions_from_macros,