Use ureq until AWC IO error is fixed
This commit is contained in:
parent
1fd688eeed
commit
2385fa33ec
57
Cargo.lock
generated
57
Cargo.lock
generated
@ -31,7 +31,7 @@ dependencies = [
|
|||||||
"futures-util",
|
"futures-util",
|
||||||
"http",
|
"http",
|
||||||
"log",
|
"log",
|
||||||
"rustls",
|
"rustls 0.18.1",
|
||||||
"tokio-rustls",
|
"tokio-rustls",
|
||||||
"trust-dns-proto",
|
"trust-dns-proto",
|
||||||
"trust-dns-resolver",
|
"trust-dns-resolver",
|
||||||
@ -193,10 +193,10 @@ dependencies = [
|
|||||||
"actix-service",
|
"actix-service",
|
||||||
"actix-utils",
|
"actix-utils",
|
||||||
"futures-util",
|
"futures-util",
|
||||||
"rustls",
|
"rustls 0.18.1",
|
||||||
"tokio-rustls",
|
"tokio-rustls",
|
||||||
"webpki",
|
"webpki",
|
||||||
"webpki-roots",
|
"webpki-roots 0.20.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@ -249,7 +249,7 @@ dependencies = [
|
|||||||
"mime",
|
"mime",
|
||||||
"pin-project 1.0.4",
|
"pin-project 1.0.4",
|
||||||
"regex",
|
"regex",
|
||||||
"rustls",
|
"rustls 0.18.1",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"serde_urlencoded",
|
"serde_urlencoded",
|
||||||
@ -393,7 +393,7 @@ dependencies = [
|
|||||||
"mime",
|
"mime",
|
||||||
"percent-encoding",
|
"percent-encoding",
|
||||||
"rand",
|
"rand",
|
||||||
"rustls",
|
"rustls 0.18.1",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"serde_urlencoded",
|
"serde_urlencoded",
|
||||||
@ -529,6 +529,12 @@ version = "1.0.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "chunked_transfer"
|
||||||
|
version = "1.3.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7477065d45a8fe57167bf3cf8bcd3729b54cfcb81cca49bda2d038ea89ae82ca"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "const_fn"
|
name = "const_fn"
|
||||||
version = "0.4.5"
|
version = "0.4.5"
|
||||||
@ -986,6 +992,7 @@ dependencies = [
|
|||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"time",
|
"time",
|
||||||
|
"ureq",
|
||||||
"url",
|
"url",
|
||||||
]
|
]
|
||||||
|
|
||||||
@ -1413,6 +1420,19 @@ dependencies = [
|
|||||||
"webpki",
|
"webpki",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rustls"
|
||||||
|
version = "0.19.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "064fd21ff87c6e87ed4506e68beb42459caa4a0e2eb144932e6776768556980b"
|
||||||
|
dependencies = [
|
||||||
|
"base64 0.13.0",
|
||||||
|
"log",
|
||||||
|
"ring",
|
||||||
|
"sct",
|
||||||
|
"webpki",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "ryu"
|
name = "ryu"
|
||||||
version = "1.0.5"
|
version = "1.0.5"
|
||||||
@ -1749,7 +1769,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "e12831b255bcfa39dc0436b01e19fea231a37db570686c06ee72c423479f889a"
|
checksum = "e12831b255bcfa39dc0436b01e19fea231a37db570686c06ee72c423479f889a"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"futures-core",
|
"futures-core",
|
||||||
"rustls",
|
"rustls 0.18.1",
|
||||||
"tokio",
|
"tokio",
|
||||||
"webpki",
|
"webpki",
|
||||||
]
|
]
|
||||||
@ -1890,6 +1910,22 @@ version = "0.7.1"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
|
checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "ureq"
|
||||||
|
version = "2.0.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "96014ded8c85822677daee4f909d18acccca744810fd4f8ffc492c284f2324bc"
|
||||||
|
dependencies = [
|
||||||
|
"base64 0.13.0",
|
||||||
|
"chunked_transfer",
|
||||||
|
"log",
|
||||||
|
"once_cell",
|
||||||
|
"rustls 0.19.0",
|
||||||
|
"url",
|
||||||
|
"webpki",
|
||||||
|
"webpki-roots 0.21.0",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "url"
|
name = "url"
|
||||||
version = "2.2.0"
|
version = "2.2.0"
|
||||||
@ -1997,6 +2033,15 @@ dependencies = [
|
|||||||
"webpki",
|
"webpki",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "webpki-roots"
|
||||||
|
version = "0.21.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "82015b7e0b8bad8185994674a13a93306bea76cf5a16c5a181382fd3a5ec2376"
|
||||||
|
dependencies = [
|
||||||
|
"webpki",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "widestring"
|
name = "widestring"
|
||||||
version = "0.4.3"
|
version = "0.4.3"
|
||||||
|
@ -11,6 +11,7 @@ edition = "2018"
|
|||||||
base64 = "0.13.0"
|
base64 = "0.13.0"
|
||||||
actix-web = { version = "3.3.2", features = ["rustls"] }
|
actix-web = { version = "3.3.2", features = ["rustls"] }
|
||||||
askama = "0.10.5"
|
askama = "0.10.5"
|
||||||
|
ureq = "2.0.1"
|
||||||
serde = { version = "1.0.118", default_features = false, features = ["derive"] }
|
serde = { version = "1.0.118", default_features = false, features = ["derive"] }
|
||||||
serde_json = "1.0"
|
serde_json = "1.0"
|
||||||
async-recursion = "0.3.1"
|
async-recursion = "0.3.1"
|
||||||
|
152
src/utils.rs
152
src/utils.rs
@ -363,85 +363,85 @@ pub async fn request(path: &str) -> Result<Value, String> {
|
|||||||
let url = format!("https://www.reddit.com{}", path);
|
let url = format!("https://www.reddit.com{}", path);
|
||||||
|
|
||||||
// Send request using awc
|
// Send request using awc
|
||||||
async fn send(url: &str) -> Result<String, (bool, String)> {
|
// async fn send(url: &str) -> Result<String, (bool, String)> {
|
||||||
let client = actix_web::client::Client::default();
|
// let client = actix_web::client::Client::default();
|
||||||
let response = client.get(url).header("User-Agent", format!("web:libreddit:{}", env!("CARGO_PKG_VERSION"))).send().await;
|
// let response = client.get(url).header("User-Agent", format!("web:libreddit:{}", env!("CARGO_PKG_VERSION"))).send().await;
|
||||||
|
|
||||||
match response {
|
// match response {
|
||||||
Ok(mut payload) => {
|
// Ok(mut payload) => {
|
||||||
// Get first number of response HTTP status code
|
// // Get first number of response HTTP status code
|
||||||
match payload.status().to_string().chars().next() {
|
// match payload.status().to_string().chars().next() {
|
||||||
// If success
|
// // If success
|
||||||
Some('2') => Ok(String::from_utf8(payload.body().limit(20_000_000).await.unwrap_or_default().to_vec()).unwrap_or_default()),
|
// Some('2') => Ok(String::from_utf8(payload.body().limit(20_000_000).await.unwrap_or_default().to_vec()).unwrap_or_default()),
|
||||||
// If redirection
|
// // If redirection
|
||||||
Some('3') => match payload.headers().get("location") {
|
// Some('3') => match payload.headers().get("location") {
|
||||||
Some(location) => Err((true, location.to_str().unwrap_or_default().to_string())),
|
// Some(location) => Err((true, location.to_str().unwrap_or_default().to_string())),
|
||||||
None => Err((false, "Page not found".to_string())),
|
// None => Err((false, "Page not found".to_string())),
|
||||||
},
|
// },
|
||||||
// Otherwise
|
// // Otherwise
|
||||||
_ => Err((false, "Page not found".to_string())),
|
// _ => Err((false, "Page not found".to_string())),
|
||||||
}
|
|
||||||
}
|
|
||||||
Err(e) => { dbg!(e); Err((false, "Couldn't send request to Reddit, this instance may be being rate-limited. Try another.".to_string())) },
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Print error if debugging then return error based on error message
|
|
||||||
fn err(url: String, msg: String) -> Result<Value, String> {
|
|
||||||
// #[cfg(debug_assertions)]
|
|
||||||
dbg!(format!("{} - {}", url, msg));
|
|
||||||
Err(msg)
|
|
||||||
};
|
|
||||||
|
|
||||||
// Parse JSON from body. If parsing fails, return error
|
|
||||||
fn json(url: String, body: String) -> Result<Value, String> {
|
|
||||||
match from_str(body.as_str()) {
|
|
||||||
Ok(json) => Ok(json),
|
|
||||||
Err(_) => err(url, "Failed to parse page JSON data".to_string()),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Make request to Reddit using send function
|
|
||||||
match send(&url).await {
|
|
||||||
// If success, parse and return body
|
|
||||||
Ok(body) => json(url, body),
|
|
||||||
// Follow any redirects
|
|
||||||
Err((true, location)) => match send(location.as_str()).await {
|
|
||||||
// If success, parse and return body
|
|
||||||
Ok(body) => json(url, body),
|
|
||||||
// Follow any redirects again
|
|
||||||
Err((true, location)) => err(url, location),
|
|
||||||
// Return errors if request fails
|
|
||||||
Err((_, msg)) => err(url, msg),
|
|
||||||
},
|
|
||||||
// Return errors if request fails
|
|
||||||
Err((_, msg)) => err(url, msg),
|
|
||||||
}
|
|
||||||
|
|
||||||
// Send request using ureq
|
|
||||||
// match ureq::get(&url).call() {
|
|
||||||
// // If response is success
|
|
||||||
// Ok(response) => {
|
|
||||||
// // Parse the response from Reddit as JSON
|
|
||||||
// match from_str(&response.into_string().unwrap()) {
|
|
||||||
// Ok(json) => Ok(json),
|
|
||||||
// Err(_) => {
|
|
||||||
// #[cfg(debug_assertions)]
|
|
||||||
// dbg!(format!("{} - Failed to parse page JSON data", url));
|
|
||||||
// Err("Failed to parse page JSON data".to_string())
|
|
||||||
// }
|
// }
|
||||||
// }
|
// }
|
||||||
// }
|
// Err(e) => { dbg!(e); Err((false, "Couldn't send request to Reddit, this instance may be being rate-limited. Try another.".to_string())) },
|
||||||
// // If response is error
|
|
||||||
// Err(ureq::Error::Status(_, _)) => {
|
|
||||||
// #[cfg(debug_assertions)]
|
|
||||||
// dbg!(format!("{} - Page not found", url));
|
|
||||||
// Err("Page not found".to_string())
|
|
||||||
// }
|
|
||||||
// // If failed to send request
|
|
||||||
// Err(e) => {
|
|
||||||
// dbg!(format!("{} - {}", url, e));
|
|
||||||
// Err("Couldn't send request to Reddit, this instance may be being rate-limited. Try another.".to_string())
|
|
||||||
// }
|
// }
|
||||||
// }
|
// }
|
||||||
|
|
||||||
|
// // Print error if debugging then return error based on error message
|
||||||
|
// fn err(url: String, msg: String) -> Result<Value, String> {
|
||||||
|
// // #[cfg(debug_assertions)]
|
||||||
|
// dbg!(format!("{} - {}", url, msg));
|
||||||
|
// Err(msg)
|
||||||
|
// };
|
||||||
|
|
||||||
|
// // Parse JSON from body. If parsing fails, return error
|
||||||
|
// fn json(url: String, body: String) -> Result<Value, String> {
|
||||||
|
// match from_str(body.as_str()) {
|
||||||
|
// Ok(json) => Ok(json),
|
||||||
|
// Err(_) => err(url, "Failed to parse page JSON data".to_string()),
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
|
// // Make request to Reddit using send function
|
||||||
|
// match send(&url).await {
|
||||||
|
// // If success, parse and return body
|
||||||
|
// Ok(body) => json(url, body),
|
||||||
|
// // Follow any redirects
|
||||||
|
// Err((true, location)) => match send(location.as_str()).await {
|
||||||
|
// // If success, parse and return body
|
||||||
|
// Ok(body) => json(url, body),
|
||||||
|
// // Follow any redirects again
|
||||||
|
// Err((true, location)) => err(url, location),
|
||||||
|
// // Return errors if request fails
|
||||||
|
// Err((_, msg)) => err(url, msg),
|
||||||
|
// },
|
||||||
|
// // Return errors if request fails
|
||||||
|
// Err((_, msg)) => err(url, msg),
|
||||||
|
// }
|
||||||
|
|
||||||
|
// Send request using ureq
|
||||||
|
match ureq::get(&url).call() {
|
||||||
|
// If response is success
|
||||||
|
Ok(response) => {
|
||||||
|
// Parse the response from Reddit as JSON
|
||||||
|
match from_str(&response.into_string().unwrap()) {
|
||||||
|
Ok(json) => Ok(json),
|
||||||
|
Err(_) => {
|
||||||
|
#[cfg(debug_assertions)]
|
||||||
|
dbg!(format!("{} - Failed to parse page JSON data", url));
|
||||||
|
Err("Failed to parse page JSON data".to_string())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// If response is error
|
||||||
|
Err(ureq::Error::Status(_, _)) => {
|
||||||
|
#[cfg(debug_assertions)]
|
||||||
|
dbg!(format!("{} - Page not found", url));
|
||||||
|
Err("Page not found".to_string())
|
||||||
|
}
|
||||||
|
// If failed to send request
|
||||||
|
Err(e) => {
|
||||||
|
dbg!(format!("{} - {}", url, e));
|
||||||
|
Err("Couldn't send request to Reddit, this instance may be being rate-limited. Try another.".to_string())
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user