From 9f75f28d3dfdde22487e709a8530d14995ce4bb9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 30 Apr 2021 13:33:24 +0200 Subject: [PATCH] Add example folder (#241) --- Cargo.lock | 38 ++++++++++++++++++++ Cargo.toml | 1 + README.md | 1 - examples/builder/Cargo.toml | 15 ++++++++ examples/builder/builder.rs | 47 +++++++++++++++++++++++++ examples/client_pool/Cargo.toml | 12 +++++++ examples/client_pool/client_pool.rs | 47 +++++++++++++++++++++++++ examples/collect_links/Cargo.toml | 15 ++++++++ examples/collect_links/collect_links.rs | 26 ++++++++++++++ examples/simple/Cargo.toml | 12 +++++++ examples/simple/simple.rs | 8 +++++ 11 files changed, 221 insertions(+), 1 deletion(-) create mode 100644 examples/builder/Cargo.toml create mode 100644 examples/builder/builder.rs create mode 100644 examples/client_pool/Cargo.toml create mode 100644 examples/client_pool/client_pool.rs create mode 100644 examples/collect_links/Cargo.toml create mode 100644 examples/collect_links/collect_links.rs create mode 100644 examples/simple/Cargo.toml create mode 100644 examples/simple/simple.rs diff --git a/Cargo.lock b/Cargo.lock index 72c7561..a487ee7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -351,6 +351,17 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40e38929add23cdf8a366df9b0e088953150724bcbe5fc330b0d8eb3b328eec8" +[[package]] +name = "builder" +version = "0.1.0" +dependencies = [ + "http", + "lychee-lib", + "regex", + "reqwest", + "tokio", +] + [[package]] name = "bumpalo" version = "3.6.1" @@ -436,6 +447,25 @@ dependencies = [ "vec_map", ] +[[package]] +name = "client_pool" +version = "0.1.0" +dependencies = [ + "lychee-lib", + "tokio", +] + +[[package]] +name = "collect_links" +version = "0.1.0" +dependencies = [ + "http", + "lychee-lib", + "regex", + "reqwest", + "tokio", +] + [[package]] name = "concurrent-queue" version = "1.2.2" @@ -2265,6 +2295,14 @@ dependencies = [ "libc", ] +[[package]] +name = "simple" +version = "0.1.0" +dependencies = [ + "lychee-lib", + "tokio", +] + [[package]] name = "simple_asn1" version = "0.4.1" diff --git a/Cargo.toml b/Cargo.toml index 863013f..7c228c6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ members = [ "lychee-bin", "lychee-lib", + "examples/*", ] [patch.crates-io] diff --git a/README.md b/README.md index d400582..576528f 100644 --- a/README.md +++ b/README.md @@ -256,7 +256,6 @@ let client = lychee_lib::ClientBuilder::builder() .custom_headers(headers) .method(method) .timeout(timeout) - .verbose(cfg.verbose) .github_token(cfg.github_token) .scheme(cfg.scheme) .accepted(accepted) diff --git a/examples/builder/Cargo.toml b/examples/builder/Cargo.toml new file mode 100644 index 0000000..f94cea6 --- /dev/null +++ b/examples/builder/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "builder" +version = "0.1.0" +edition = "2018" + +[[example]] +name = "builder" +path = "builder.rs" + +[dependencies] +lychee-lib = { path = "../../lychee-lib" } +tokio = { version = "1.5.0", features = ["full"] } +regex = "1.4.6" +http = "0.2.4" +reqwest = { version = "0.11.3", features = ["gzip"] } \ No newline at end of file diff --git a/examples/builder/builder.rs b/examples/builder/builder.rs new file mode 100644 index 0000000..08f189e --- /dev/null +++ b/examples/builder/builder.rs @@ -0,0 +1,47 @@ +use http::header::{self, HeaderMap}; +use http::StatusCode; +use lychee_lib::{ClientBuilder, Result}; +use regex::RegexSet; +use reqwest::Method; +use std::iter::FromIterator; +use std::{collections::HashSet, time::Duration}; + +#[tokio::main] +#[allow(clippy::trivial_regex)] +async fn main() -> Result<()> { + // Excludes + let excludes = Some(RegexSet::new(&[r"example"]).unwrap()); + // Includes take precedence over excludes + let includes = Some(RegexSet::new(&[r"example.org"]).unwrap()); + + // Set custom request headers + let mut headers = HeaderMap::new(); + headers.insert(header::ACCEPT, "text/html".parse().unwrap()); + + let accepted = Some(HashSet::from_iter(vec![ + StatusCode::OK, + StatusCode::NO_CONTENT, + ])); + + let client = ClientBuilder::builder() + .excludes(excludes) + .includes(includes) + .max_redirects(3u8) + .user_agent("custom useragent") + .allow_insecure(true) + .custom_headers(headers) + .method(Method::HEAD) + .timeout(Duration::from_secs(5)) + .schemes(HashSet::from_iter(vec![ + "http".to_string(), + "https".to_string(), + ])) + .accepted(accepted) + .build() + .client()?; + + let response = client.check("http://example.org").await?; + dbg!(&response); + assert!(response.status().is_success()); + Ok(()) +} diff --git a/examples/client_pool/Cargo.toml b/examples/client_pool/Cargo.toml new file mode 100644 index 0000000..811f2fe --- /dev/null +++ b/examples/client_pool/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "client_pool" +version = "0.1.0" +edition = "2018" + +[[example]] +name = "client_pool" +path = "client_pool.rs" + +[dependencies] +lychee-lib = { path = "../../lychee-lib" } +tokio = { version = "1.5.0", features = ["full"] } diff --git a/examples/client_pool/client_pool.rs b/examples/client_pool/client_pool.rs new file mode 100644 index 0000000..ef4bc8d --- /dev/null +++ b/examples/client_pool/client_pool.rs @@ -0,0 +1,47 @@ +use lychee_lib::{ClientBuilder, ClientPool, Input, Request, Result, Uri}; +use std::convert::TryFrom; +use tokio::sync::mpsc; + +const CONCURRENT_REQUESTS: usize = 4; + +#[tokio::main] +#[allow(clippy::trivial_regex)] +async fn main() -> Result<()> { + // These channels are used to send requests and receive responses to and + // from the lychee client pool + let (send_req, recv_req) = mpsc::channel(CONCURRENT_REQUESTS); + let (send_resp, mut recv_resp) = mpsc::channel(CONCURRENT_REQUESTS); + + // Add as many requests as you like + let requests = vec![Request::new( + Uri::try_from("https://example.org")?, + Input::Stdin, + )]; + + // Send requests to pool + tokio::spawn(async move { + for request in requests { + println!("Sending {}", request); + send_req.send(request).await.unwrap(); + } + }); + + // Create a default lychee client + let client = ClientBuilder::default().client()?; + + // Create a pool with four lychee clients + let clients = vec![client; CONCURRENT_REQUESTS]; + let mut clients = ClientPool::new(send_resp, recv_req, clients); + + // Handle requests in a client pool + tokio::spawn(async move { + clients.listen().await; + }); + + // Finally, listen to incoming responses from lychee + while let Some(response) = recv_resp.recv().await { + println!("{}", response); + } + + Ok(()) +} diff --git a/examples/collect_links/Cargo.toml b/examples/collect_links/Cargo.toml new file mode 100644 index 0000000..ab9cc01 --- /dev/null +++ b/examples/collect_links/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "collect_links" +version = "0.1.0" +edition = "2018" + +[[example]] +name = "collect_links" +path = "collect_links.rs" + +[dependencies] +lychee-lib = { path = "../../lychee-lib" } +tokio = { version = "1.5.0", features = ["full"] } +regex = "1.4.6" +http = "0.2.4" +reqwest = { version = "0.11.3", features = ["gzip"] } \ No newline at end of file diff --git a/examples/collect_links/collect_links.rs b/examples/collect_links/collect_links.rs new file mode 100644 index 0000000..5e7d908 --- /dev/null +++ b/examples/collect_links/collect_links.rs @@ -0,0 +1,26 @@ +use lychee_lib::{Input, Result}; +use reqwest::Url; +use std::path::PathBuf; + +#[tokio::main] +#[allow(clippy::trivial_regex)] +async fn main() -> Result<()> { + // Collect all links from the following inputs + let inputs: &[Input] = &[ + Input::RemoteUrl(Box::new( + Url::parse("https://github.com/lycheeverse/lychee").unwrap(), + )), + Input::FsPath(PathBuf::from("fixtures/TEST.md")), + ]; + + let links = lychee_lib::collector::collect_links( + inputs, None, // base_url + false, // don't skip missing inputs + 10, // max concurrency + ) + .await?; + + dbg!(links); + + Ok(()) +} diff --git a/examples/simple/Cargo.toml b/examples/simple/Cargo.toml new file mode 100644 index 0000000..f5d35a7 --- /dev/null +++ b/examples/simple/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "simple" +version = "0.1.0" +edition = "2018" + +[[example]] +name = "simple" +path = "simple.rs" + +[dependencies] +lychee-lib = { path = "../../lychee-lib" } +tokio = { version = "1.5.0", features = ["full"] } \ No newline at end of file diff --git a/examples/simple/simple.rs b/examples/simple/simple.rs new file mode 100644 index 0000000..bb711e6 --- /dev/null +++ b/examples/simple/simple.rs @@ -0,0 +1,8 @@ +use lychee_lib::Result; + +#[tokio::main] +async fn main() -> Result<()> { + let response = lychee_lib::check("https://github.com/lycheeverse/lychee").await?; + dbg!("{}", response); + Ok(()) +}