iridium/redsunlib
iridium/redsunlib
2
2
Fork 0
Code Issues 2 Pull Requests Packages Projects Releases 4 Wiki Activity

Switch back to ureq temporarily

Browse Source
This commit is contained in:
spikecodes 2021-01-15 15:28:51 -08:00
parent 078d6fe25b
commit 59021b9331
3 changed files with 144 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

89
Cargo.lock generated
View File

@ -31,7 +31,7 @@ dependencies = [
"futures-util", "futures-util",
"http", "http",
"log", "log",
"rustls", "rustls 0.18.1",
"tokio-rustls", "tokio-rustls",
"trust-dns-proto", "trust-dns-proto",
"trust-dns-resolver", "trust-dns-resolver",
@ -193,10 +193,10 @@ dependencies = [
"actix-service", "actix-service",
"actix-utils", "actix-utils",
"futures-util", "futures-util",
"rustls", "rustls 0.18.1",
"tokio-rustls", "tokio-rustls",
"webpki", "webpki",
"webpki-roots", "webpki-roots 0.20.0",
] ]
[[package]] [[package]]
@ -249,7 +249,7 @@ dependencies = [
"mime", "mime",
"pin-project 1.0.4", "pin-project 1.0.4",
"regex", "regex",
"rustls", "rustls 0.18.1",
"serde", "serde",
"serde_json", "serde_json",
"serde_urlencoded", "serde_urlencoded",
@ -393,7 +393,7 @@ dependencies = [
"mime", "mime",
"percent-encoding", "percent-encoding",
"rand", "rand",
"rustls", "rustls 0.18.1",
"serde", "serde",
"serde_json", "serde_json",
"serde_urlencoded", "serde_urlencoded",
@ -529,6 +529,12 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "chunked_transfer"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7477065d45a8fe57167bf3cf8bcd3729b54cfcb81cca49bda2d038ea89ae82ca"
[[package]] [[package]]
name = "const_fn" name = "const_fn"
version = "0.4.5" version = "0.4.5"
@ -672,9 +678,9 @@ checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7"
[[package]] [[package]]
name = "futures" name = "futures"
version = "0.3.10" version = "0.3.12"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "309f13e3f4be6d5917178c84db67c0b9a09177ac16d4f9a7313a767a68adaa77" checksum = "da9052a1a50244d8d5aa9bf55cbc2fb6f357c86cc52e46c62ed390a7180cf150"
dependencies = [ dependencies = [
"futures-channel", "futures-channel",
"futures-core", "futures-core",
@ -686,9 +692,9 @@ dependencies = [
[[package]] [[package]]
name = "futures-channel" name = "futures-channel"
version = "0.3.10" version = "0.3.12"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a3b03bd32f6ec7885edeb99acd1e47e20e34fd4dfd3c6deed6fcac8a9d28f6a" checksum = "f2d31b7ec7efab6eefc7c57233bb10b847986139d88cc2f5a02a1ae6871a1846"
dependencies = [ dependencies = [
"futures-core", "futures-core",
"futures-sink", "futures-sink",
@ -696,21 +702,21 @@ dependencies = [
[[package]] [[package]]
name = "futures-core" name = "futures-core"
version = "0.3.10" version = "0.3.12"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed8aeae2b6ab243ebabe6f54cd4cf53054d98883d5d326128af7d57a9ca5cd3d" checksum = "79e5145dde8da7d1b3892dad07a9c98fc04bc39892b1ecc9692cf53e2b780a65"
[[package]] [[package]]
name = "futures-io" name = "futures-io"
version = "0.3.10" version = "0.3.12"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d41234e71d5e8ca73d01563974ef6f50e516d71e18f1a2f1184742e31f5d469f" checksum = "28be053525281ad8259d47e4de5de657b25e7bac113458555bb4b70bc6870500"
[[package]] [[package]]
name = "futures-macro" name = "futures-macro"
version = "0.3.10" version = "0.3.12"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3520e0eb4e704e88d771b92d51273ee212997f0d8282f17f5d8ff1cb39104e42" checksum = "c287d25add322d9f9abdcdc5927ca398917996600182178774032e9f8258fedd"
dependencies = [ dependencies = [
"proc-macro-hack", "proc-macro-hack",
"proc-macro2", "proc-macro2",
@ -720,24 +726,24 @@ dependencies = [
[[package]] [[package]]
name = "futures-sink" name = "futures-sink"
version = "0.3.10" version = "0.3.12"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c72d188479368953c6c8c7140e40d7a4401674ab3b98a41e60e515d6cbdbe5de" checksum = "caf5c69029bda2e743fddd0582d1083951d65cc9539aebf8812f36c3491342d6"
[[package]] [[package]]
name = "futures-task" name = "futures-task"
version = "0.3.10" version = "0.3.12"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08944cea9021170d383287169859c0ca8147d9ec285978393109954448f33cc7" checksum = "13de07eb8ea81ae445aca7b69f5f7bf15d7bf4912d8ca37d6645c77ae8a58d86"
dependencies = [ dependencies = [
"once_cell", "once_cell",
] ]
[[package]] [[package]]
name = "futures-util" name = "futures-util"
version = "0.3.10" version = "0.3.12"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3dd206efbe2ca683b2ce138ccdf61e1b0a63f5816dcedc9d8654c500ba0cea6" checksum = "632a8cd0f2a4b3fdea1657f08bde063848c3bd00f9bbf6e256b8be78802e624b"
dependencies = [ dependencies = [
"futures-channel", "futures-channel",
"futures-core", "futures-core",
@ -986,6 +992,7 @@ dependencies = [
"serde", "serde",
"serde_json", "serde_json",
"time", "time",
"ureq",
"url", "url",
] ]
@ -1413,6 +1420,19 @@ dependencies = [
"webpki", "webpki",
] ]
[[package]]
name = "rustls"
version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "064fd21ff87c6e87ed4506e68beb42459caa4a0e2eb144932e6776768556980b"
dependencies = [
"base64 0.13.0",
"log",
"ring",
"sct",
"webpki",
]
[[package]] [[package]]
name = "ryu" name = "ryu"
version = "1.0.5" version = "1.0.5"
@ -1749,7 +1769,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e12831b255bcfa39dc0436b01e19fea231a37db570686c06ee72c423479f889a" checksum = "e12831b255bcfa39dc0436b01e19fea231a37db570686c06ee72c423479f889a"
dependencies = [ dependencies = [
"futures-core", "futures-core",
"rustls", "rustls 0.18.1",
"tokio", "tokio",
"webpki", "webpki",
] ]
@ -1890,6 +1910,22 @@ version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
[[package]]
name = "ureq"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96014ded8c85822677daee4f909d18acccca744810fd4f8ffc492c284f2324bc"
dependencies = [
"base64 0.13.0",
"chunked_transfer",
"log",
"once_cell",
"rustls 0.19.0",
"url",
"webpki",
"webpki-roots 0.21.0",
]
[[package]] [[package]]
name = "url" name = "url"
version = "2.2.0" version = "2.2.0"
@ -1997,6 +2033,15 @@ dependencies = [
"webpki", "webpki",
] ]
[[package]]
name = "webpki-roots"
version = "0.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "82015b7e0b8bad8185994674a13a93306bea76cf5a16c5a181382fd3a5ec2376"
dependencies = [
"webpki",
]
[[package]] [[package]]
name = "widestring" name = "widestring"
version = "0.4.3" version = "0.4.3"

1
Cargo.toml
View File

@ -11,6 +11,7 @@ edition = "2018"
base64 = "0.13.0" base64 = "0.13.0"
actix-web = { version = "3.3.2", features = ["rustls"] } actix-web = { version = "3.3.2", features = ["rustls"] }
askama = "0.10.5" askama = "0.10.5"
ureq = "2.0.1"
serde = { version = "1.0.118", default_features = false, features = ["derive"] } serde = { version = "1.0.118", default_features = false, features = ["derive"] }
serde_json = "1.0" serde_json = "1.0"
async-recursion = "0.3.1" async-recursion = "0.3.1"

122
src/utils.rs
View File

@ -363,58 +363,86 @@ pub async fn request(path: &str) -> Result<Value, String> {
let url = format!("https://www.reddit.com{}", path); let url = format!("https://www.reddit.com{}", path);
// Send request using awc // Send request using awc
async fn send(url: &str) -> Result<String, (bool, String)> { // async fn send(url: &str) -> Result<String, (bool, String)> {
let client = actix_web::client::Client::default(); // let client = actix_web::client::Client::default();
let response = client.get(url).send().await; // let response = client.get(url).send().await;
match response { // match response {
Ok(mut payload) => { // Ok(mut payload) => {
// Get first number of response HTTP status code // // Get first number of response HTTP status code
match payload.status().to_string().chars().next() { // match payload.status().to_string().chars().next() {
// If success // // If success
Some('2') => Ok(String::from_utf8(payload.body().limit(20_000_000).await.unwrap_or_default().to_vec()).unwrap_or_default()), // Some('2') => Ok(String::from_utf8(payload.body().limit(20_000_000).await.unwrap_or_default().to_vec()).unwrap_or_default()),
// If redirection // // If redirection
Some('3') => match payload.headers().get("location") { // Some('3') => match payload.headers().get("location") {
Some(location) => Err((true, location.to_str().unwrap_or_default().to_string())), // Some(location) => Err((true, location.to_str().unwrap_or_default().to_string())),
None => Err((false, "Page not found".to_string())), // None => Err((false, "Page not found".to_string())),
}, // },
// Otherwise // // Otherwise
_ => Err((false, "Page not found".to_string())), // _ => Err((false, "Page not found".to_string())),
} // }
} // }
Err(_) => Err((false, "Couldn't send request to Reddit, this instance may be being rate-limited. Try another.".to_string())), // Err(_) => Err((false, "Couldn't send request to Reddit, this instance may be being rate-limited. Try another.".to_string())),
} // }
} // }
// Print error if debugging then return error based on error message // // Print error if debugging then return error based on error message
fn err(url: String, msg: String) -> Result<Value, String> { // fn err(url: String, msg: String) -> Result<Value, String> {
// #[cfg(debug_assertions)] // // #[cfg(debug_assertions)]
dbg!(format!("{} - {}", url, msg)); // dbg!(format!("{} - {}", url, msg));
Err(msg) // Err(msg)
}; // };
// Parse JSON from body. If parsing fails, return error // // Parse JSON from body. If parsing fails, return error
fn json(url: String, body: String) -> Result<Value, String> { // fn json(url: String, body: String) -> Result<Value, String> {
match from_str(body.as_str()) { // match from_str(body.as_str()) {
// Ok(json) => Ok(json),
// Err(_) => err(url, "Failed to parse page JSON data".to_string()),
// }
// }
// // Make request to Reddit using send function
// match send(&url).await {
// // If success, parse and return body
// Ok(body) => json(url, body),
// // Follow any redirects
// Err((true, location)) => match send(location.as_str()).await {
// // If success, parse and return body
// Ok(body) => json(url, body),
// // Follow any redirects again
// Err((true, location)) => err(url, location),
// // Return errors if request fails
// Err((_, msg)) => err(url, msg),
// },
// // Return errors if request fails
// Err((_, msg)) => err(url, msg),
// }
// Send request using ureq
match ureq::get(&url).call() {
// If response is success
Ok(response) => {
// Parse the response from Reddit as JSON
match from_str(&response.into_string().unwrap()) {
Ok(json) => Ok(json), Ok(json) => Ok(json),
Err(_) => err(url, "Failed to parse page JSON data".to_string()), Err(_) => {
#[cfg(debug_assertions)]
dbg!(format!("{} - Failed to parse page JSON data", url));
Err("Failed to parse page JSON data".to_string())
} }
} }
}
// Make request to Reddit using send function // If response is error
match send(&url).await { Err(ureq::Error::Status(_, _)) => {
// If success, parse and return body #[cfg(debug_assertions)]
Ok(body) => json(url, body), dbg!(format!("{} - Page not found", url));
// Follow any redirects Err("Page not found".to_string())
Err((true, location)) => match send(location.as_str()).await { }
// If success, parse and return body // If failed to send request
Ok(body) => json(url, body), Err(e) => {
// Follow any redirects again #[cfg(debug_assertions)]
Err((true, location)) => err(url, location), dbg!(format!("{} - {}", url, e));
// Return errors if request fails Err("Couldn't send request to Reddit, this instance may be being rate-limited. Try another.".to_string())
Err((_, msg)) => err(url, msg), }
},
// Return errors if request fails
Err((_, msg)) => err(url, msg),
} }
} }