Hopiu/lychee
1
0
Fork 0
mirror of https://github.com/Hopiu/lychee.git synced 2026-05-01 10:24:47 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

wip

Browse source
This commit is contained in:
Matthias Endler 2021-02-25 21:22:25 +01:00
parent 959d2d51b3
commit 93161a8c9a
2 changed files with 47 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
src/client.rs
View file

@ -245,27 +245,17 @@ impl Client {
}
async fn check_default(&self, url: &Url) -> Status {
let mut req_method = self.method.clone();
let mut req_url = url.to_owned();
let mut req_headers = None;
let request = match self
.reqwest_client
.request(self.method.clone(), url.to_owned())
.build()
{
Ok(r) => r,
Err(e) => return e.into(),
};
let request = self.quirks.apply(request);
for quirk in self.quirks.matching(url) {
println!("Applying quirk: {:?}", quirk);
if let Some(rewrite) = quirk.rewrite {
req_url = rewrite(url.to_owned());
}
if let Some(method) = quirk.method {
req_method = method;
}
req_headers = quirk.headers;
}
let mut request = self.reqwest_client.request(req_method, req_url);
if let Some(headers) = req_headers {
request = request.headers(headers);
}
match request.send().await {
match self.reqwest_client.execute(request).await {
Ok(response) => Status::new(response.status(), self.accepted.clone()),
Err(e) => e.into(),
}

39
src/quirks/mod.rs
View file

@ -4,7 +4,7 @@ mod headers_macro;
use headers::HeaderMap;
use http::{header::USER_AGENT, Method};
use regex::Regex;
use reqwest::Url;
use reqwest::{Request, Url};
/// Sadly some pages only return plaintext results if Google is trying to crawl them.
const GOOGLEBOT: &'static str =
@ -18,6 +18,17 @@ pub struct Quirk {
pub rewrite: Option<fn(Url) -> Url>,
}
impl Quirk {
fn matches(&self, url: &Url) -> bool {
self.pattern.is_match(url.as_str())
}
fn apply(&self, request: &mut Request) -> &mut Request {
&mut request
}
}
#[derive(Debug, Clone)]
pub struct Quirks {
quirks: Vec<Quirk>,
@ -51,7 +62,7 @@ impl Quirks {
Self { quirks }
}
pub fn matching(&self, url: &Url) -> Vec<Quirk> {
fn matches(&self, url: &Url) -> Vec<Quirk> {
let mut matching = vec![];
for quirk in &self.quirks {
if quirk.pattern.is_match(url.as_str()) {
@ -60,4 +71,28 @@ impl Quirks {
}
matching
}
pub fn apply(&self, request: Request) -> Request {
let mut request = request.clone();
// let mut req_method = self.method.clone();
// let mut req_url = url.to_owned();
// let mut req_headers = None;
for quirk in self.matches(request.url()) {
println!("Applying quirk: {:?}", quirk);
request = quirk.apply(request);
// if let Some(rewrite) = quirk.rewrite {
// req_url = rewrite(url.to_owned());
// }
// if let Some(method) = quirk.method {
// req_method = method;
// }
// req_headers = quirk.headers;
}
// if let Some(headers) = req_headers {
// request = request.headers(headers);
// }
request
}
}