Add example folder (#241)

Cargo.lock

@@ -351,6 +351,17 @@ version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "40e38929add23cdf8a366df9b0e088953150724bcbe5fc330b0d8eb3b328eec8"
 
+[[package]]
+name = "builder"
+version = "0.1.0"
+dependencies = [
+ "http",
+ "lychee-lib",
+ "regex",
+ "reqwest",
+ "tokio",
+]
+
 [[package]]
 name = "bumpalo"
 version = "3.6.1"
@@ -436,6 +447,25 @@ dependencies = [
  "vec_map",
 ]
 
+[[package]]
+name = "client_pool"
+version = "0.1.0"
+dependencies = [
+ "lychee-lib",
+ "tokio",
+]
+
+[[package]]
+name = "collect_links"
+version = "0.1.0"
+dependencies = [
+ "http",
+ "lychee-lib",
+ "regex",
+ "reqwest",
+ "tokio",
+]
+
 [[package]]
 name = "concurrent-queue"
 version = "1.2.2"
@@ -2265,6 +2295,14 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "simple"
+version = "0.1.0"
+dependencies = [
+ "lychee-lib",
+ "tokio",
+]
+
 [[package]]
 name = "simple_asn1"
 version = "0.4.1"

Cargo.toml

@@ -2,6 +2,7 @@
 members = [
     "lychee-bin",
     "lychee-lib",
+    "examples/*",
 ]
 
 [patch.crates-io]

README.md

@@ -256,7 +256,6 @@ let client = lychee_lib::ClientBuilder::builder()
     .custom_headers(headers)
     .method(method)
     .timeout(timeout)
-    .verbose(cfg.verbose)
     .github_token(cfg.github_token)
     .scheme(cfg.scheme)
     .accepted(accepted)

examples/builder/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "builder"
+version = "0.1.0"
+edition = "2018"
+
+[[example]]
+name = "builder"
+path = "builder.rs"
+
+[dependencies]
+lychee-lib = { path = "../../lychee-lib" }
+tokio = { version = "1.5.0", features = ["full"] }
+regex = "1.4.6"
+http = "0.2.4"
+reqwest = { version = "0.11.3", features = ["gzip"] }

examples/builder/builder.rs

@@ -0,0 +1,47 @@
+use http::header::{self, HeaderMap};
+use http::StatusCode;
+use lychee_lib::{ClientBuilder, Result};
+use regex::RegexSet;
+use reqwest::Method;
+use std::iter::FromIterator;
+use std::{collections::HashSet, time::Duration};
+
+#[tokio::main]
+#[allow(clippy::trivial_regex)]
+async fn main() -> Result<()> {
+    // Excludes
+    let excludes = Some(RegexSet::new(&[r"example"]).unwrap());
+    // Includes take precedence over excludes
+    let includes = Some(RegexSet::new(&[r"example.org"]).unwrap());
+
+    // Set custom request headers
+    let mut headers = HeaderMap::new();
+    headers.insert(header::ACCEPT, "text/html".parse().unwrap());
+
+    let accepted = Some(HashSet::from_iter(vec![
+        StatusCode::OK,
+        StatusCode::NO_CONTENT,
+    ]));
+
+    let client = ClientBuilder::builder()
+        .excludes(excludes)
+        .includes(includes)
+        .max_redirects(3u8)
+        .user_agent("custom useragent")
+        .allow_insecure(true)
+        .custom_headers(headers)
+        .method(Method::HEAD)
+        .timeout(Duration::from_secs(5))
+        .schemes(HashSet::from_iter(vec![
+            "http".to_string(),
+            "https".to_string(),
+        ]))
+        .accepted(accepted)
+        .build()
+        .client()?;
+
+    let response = client.check("http://example.org").await?;
+    dbg!(&response);
+    assert!(response.status().is_success());
+    Ok(())
+}

examples/client_pool/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "client_pool"
+version = "0.1.0"
+edition = "2018"
+
+[[example]]
+name = "client_pool"
+path = "client_pool.rs"
+
+[dependencies]
+lychee-lib = { path = "../../lychee-lib" }
+tokio = { version = "1.5.0", features = ["full"] }

examples/client_pool/client_pool.rs

@@ -0,0 +1,47 @@
+use lychee_lib::{ClientBuilder, ClientPool, Input, Request, Result, Uri};
+use std::convert::TryFrom;
+use tokio::sync::mpsc;
+
+const CONCURRENT_REQUESTS: usize = 4;
+
+#[tokio::main]
+#[allow(clippy::trivial_regex)]
+async fn main() -> Result<()> {
+    // These channels are used to send requests and receive responses to and
+    // from the lychee client pool
+    let (send_req, recv_req) = mpsc::channel(CONCURRENT_REQUESTS);
+    let (send_resp, mut recv_resp) = mpsc::channel(CONCURRENT_REQUESTS);
+
+    // Add as many requests as you like
+    let requests = vec![Request::new(
+        Uri::try_from("https://example.org")?,
+        Input::Stdin,
+    )];
+
+    // Send requests to pool
+    tokio::spawn(async move {
+        for request in requests {
+            println!("Sending {}", request);
+            send_req.send(request).await.unwrap();
+        }
+    });
+
+    // Create a default lychee client
+    let client = ClientBuilder::default().client()?;
+
+    // Create a pool with four lychee clients
+    let clients = vec![client; CONCURRENT_REQUESTS];
+    let mut clients = ClientPool::new(send_resp, recv_req, clients);
+
+    // Handle requests in a client pool
+    tokio::spawn(async move {
+        clients.listen().await;
+    });
+
+    // Finally, listen to incoming responses from lychee
+    while let Some(response) = recv_resp.recv().await {
+        println!("{}", response);
+    }
+
+    Ok(())
+}

examples/collect_links/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "collect_links"
+version = "0.1.0"
+edition = "2018"
+
+[[example]]
+name = "collect_links"
+path = "collect_links.rs"
+
+[dependencies]
+lychee-lib = { path = "../../lychee-lib" }
+tokio = { version = "1.5.0", features = ["full"] }
+regex = "1.4.6"
+http = "0.2.4"
+reqwest = { version = "0.11.3", features = ["gzip"] }

examples/collect_links/collect_links.rs

@@ -0,0 +1,26 @@
+use lychee_lib::{Input, Result};
+use reqwest::Url;
+use std::path::PathBuf;
+
+#[tokio::main]
+#[allow(clippy::trivial_regex)]
+async fn main() -> Result<()> {
+    // Collect all links from the following inputs
+    let inputs: &[Input] = &[
+        Input::RemoteUrl(Box::new(
+            Url::parse("https://github.com/lycheeverse/lychee").unwrap(),
+        )),
+        Input::FsPath(PathBuf::from("fixtures/TEST.md")),
+    ];
+
+    let links = lychee_lib::collector::collect_links(
+        inputs, None,  // base_url
+        false, // don't skip missing inputs
+        10,    // max concurrency
+    )
+    .await?;
+
+    dbg!(links);
+
+    Ok(())
+}

examples/simple/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "simple"
+version = "0.1.0"
+edition = "2018"
+
+[[example]]
+name = "simple"
+path = "simple.rs"
+
+[dependencies]
+lychee-lib = { path = "../../lychee-lib" }
+tokio = { version = "1.5.0", features = ["full"] }

examples/simple/simple.rs

@@ -0,0 +1,8 @@
+use lychee_lib::Result;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let response = lychee_lib::check("https://github.com/lycheeverse/lychee").await?;
+    dbg!("{}", response);
+    Ok(())
+}