From 89313f73e683165d3c7b711ae36aa50146980a1e Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Thu, 27 Jun 2024 23:26:31 -0400 Subject: [PATCH 01/17] fix(oauth): atomics to avoid simultaneous token rollover --- src/client.rs | 19 ++++++++++--------- src/oauth.rs | 5 ++++- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/client.rs b/src/client.rs index fff2cf0..f6e0ddf 100644 --- a/src/client.rs +++ b/src/client.rs @@ -11,7 +11,7 @@ use percent_encoding::{percent_encode, CONTROLS}; use serde_json::Value; use std::sync::atomic::Ordering; -use std::sync::atomic::{AtomicU16, Ordering::SeqCst}; +use std::sync::atomic::{AtomicBool, AtomicU16}; use std::{io, result::Result}; use tokio::sync::RwLock; @@ -40,6 +40,8 @@ pub static OAUTH_CLIENT: Lazy> = Lazy::new(|| { pub static OAUTH_RATELIMIT_REMAINING: AtomicU16 = AtomicU16::new(99); +pub static OAUTH_IS_ROLLING_OVER: AtomicBool = AtomicBool::new(false); + /// Gets the canonical path for a resource on Reddit. This is accomplished by /// making a `HEAD` request to Reddit at the path given in `path`. /// @@ -318,11 +320,12 @@ pub async fn json(path: String, quarantine: bool) -> Result { // First, handle rolling over the OAUTH_CLIENT if need be. let current_rate_limit = OAUTH_RATELIMIT_REMAINING.load(Ordering::SeqCst); - if current_rate_limit < 10 { + let is_rolling_over = OAUTH_IS_ROLLING_OVER.load(Ordering::SeqCst); + if current_rate_limit < 10 && !is_rolling_over { warn!("Rate limit {current_rate_limit} is low. Spawning force_refresh_token()"); - OAUTH_RATELIMIT_REMAINING.store(99, Ordering::SeqCst); tokio::spawn(force_refresh_token()); } + OAUTH_RATELIMIT_REMAINING.fetch_sub(1, Ordering::SeqCst); // Fetch the url... match reddit_get(path.clone(), quarantine).await { @@ -331,12 +334,10 @@ pub async fn json(path: String, quarantine: bool) -> Result { // Ratelimit remaining if let Some(Ok(remaining)) = response.headers().get("x-ratelimit-remaining").map(|val| val.to_str()) { - trace!("Ratelimit remaining: {}", remaining); - if let Ok(remaining) = remaining.parse::().map(|f| f.round() as u16) { - OAUTH_RATELIMIT_REMAINING.store(remaining, SeqCst); - } else { - warn!("Failed to parse rate limit {remaining} from header."); - } + trace!( + "Ratelimit remaining: Header says {remaining}, we have {current_rate_limit}. {}", + if is_rolling_over { "Rolling over" } else { "" } + ); } // Ratelimit used diff --git a/src/oauth.rs b/src/oauth.rs index 03b56f6..dd0fe66 100644 --- a/src/oauth.rs +++ b/src/oauth.rs @@ -1,7 +1,7 @@ use std::{collections::HashMap, sync::atomic::Ordering, time::Duration}; use crate::{ - client::{CLIENT, OAUTH_CLIENT, OAUTH_RATELIMIT_REMAINING}, + client::{CLIENT, OAUTH_CLIENT, OAUTH_IS_ROLLING_OVER, OAUTH_RATELIMIT_REMAINING}, oauth_resources::ANDROID_APP_VERSION_LIST, }; use base64::{engine::general_purpose, Engine as _}; @@ -131,8 +131,11 @@ pub async fn token_daemon() { } pub async fn force_refresh_token() { + OAUTH_IS_ROLLING_OVER.store(true, Ordering::SeqCst); trace!("Rolling over refresh token. Current rate limit: {}", OAUTH_RATELIMIT_REMAINING.load(Ordering::SeqCst)); OAUTH_CLIENT.write().await.refresh().await; + OAUTH_RATELIMIT_REMAINING.store(99, Ordering::SeqCst); + OAUTH_IS_ROLLING_OVER.store(false, Ordering::SeqCst); } #[derive(Debug, Clone, Default)] From 4e2ec3fbc9c1776b12e4419e852621e25383063e Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Thu, 27 Jun 2024 23:29:50 -0400 Subject: [PATCH 02/17] fix(oauth): handle case where a rate limit sneaks in --- src/client.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/client.rs b/src/client.rs index f6e0ddf..ab40236 100644 --- a/src/client.rs +++ b/src/client.rs @@ -359,8 +359,13 @@ pub async fn json(path: String, quarantine: bool) -> Result { let has_remaining = body.has_remaining(); if !has_remaining { + // Rate limited, so spawn a force_refresh_token() + tokio::spawn(force_refresh_token()); return match reset { - Some(val) => Err(format!("Reddit rate limit exceeded. Will reset in: {val}")), + Some(val) => Err(format!( + "Reddit rate limit exceeded. Try refreshing in a few seconds.\ + Rate limit will reset in: {val}" + )), None => Err("Reddit rate limit exceeded".to_string()), }; } From 13083e999c936f258080cc5c157711ee2a5b7310 Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Thu, 27 Jun 2024 23:32:17 -0400 Subject: [PATCH 03/17] fix(oauth): handle extremely rare race condition by atomically compare_exchanging --- src/oauth.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/oauth.rs b/src/oauth.rs index dd0fe66..2ed2ff7 100644 --- a/src/oauth.rs +++ b/src/oauth.rs @@ -131,7 +131,11 @@ pub async fn token_daemon() { } pub async fn force_refresh_token() { - OAUTH_IS_ROLLING_OVER.store(true, Ordering::SeqCst); + if !OAUTH_IS_ROLLING_OVER.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst).is_ok() { + trace!("Skipping refresh token roll over, already in progress"); + return; + } + trace!("Rolling over refresh token. Current rate limit: {}", OAUTH_RATELIMIT_REMAINING.load(Ordering::SeqCst)); OAUTH_CLIENT.write().await.refresh().await; OAUTH_RATELIMIT_REMAINING.store(99, Ordering::SeqCst); From 2f8a38d8c7d86a47c2b169179fb8d513d483a70b Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Thu, 27 Jun 2024 23:34:27 -0400 Subject: [PATCH 04/17] chore(clippy): fix lint --- src/oauth.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/oauth.rs b/src/oauth.rs index 2ed2ff7..a3f4dc0 100644 --- a/src/oauth.rs +++ b/src/oauth.rs @@ -131,7 +131,7 @@ pub async fn token_daemon() { } pub async fn force_refresh_token() { - if !OAUTH_IS_ROLLING_OVER.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst).is_ok() { + if OAUTH_IS_ROLLING_OVER.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst).is_err() { trace!("Skipping refresh token roll over, already in progress"); return; } From 3b2ad212d50d9c8122b96e9f54933c4711caa1ca Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Fri, 28 Jun 2024 18:14:47 -0400 Subject: [PATCH 05/17] fix(oauth): arc_swap --- Cargo.lock | 7 +++++++ Cargo.toml | 1 + src/client.rs | 8 ++++---- src/oauth.rs | 23 ++++++++--------------- 4 files changed, 20 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 916093c..30bdf02 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -71,6 +71,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b" +[[package]] +name = "arc-swap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" + [[package]] name = "askama" version = "0.12.1" @@ -1034,6 +1040,7 @@ checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" name = "redlib" version = "0.34.0" dependencies = [ + "arc-swap", "askama", "base64", "brotli", diff --git a/Cargo.toml b/Cargo.toml index e33560e..df90167 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,6 +42,7 @@ fastrand = "2.0.1" log = "0.4.20" pretty_env_logger = "0.5.0" dotenvy = "0.15.7" +arc-swap = "1.7.1" [dev-dependencies] lipsum = "0.9.0" diff --git a/src/client.rs b/src/client.rs index ab40236..7281df1 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1,3 +1,4 @@ +use arc_swap::ArcSwap; use cached::proc_macro::cached; use futures_lite::future::block_on; use futures_lite::{future::Boxed, FutureExt}; @@ -13,7 +14,6 @@ use serde_json::Value; use std::sync::atomic::Ordering; use std::sync::atomic::{AtomicBool, AtomicU16}; use std::{io, result::Result}; -use tokio::sync::RwLock; use crate::dbg_msg; use crate::oauth::{force_refresh_token, token_daemon, Oauth}; @@ -32,10 +32,10 @@ pub static CLIENT: Lazy>> = Lazy::new(|| { client::Client::builder().build(https) }); -pub static OAUTH_CLIENT: Lazy> = Lazy::new(|| { +pub static OAUTH_CLIENT: Lazy> = Lazy::new(|| { let client = block_on(Oauth::new()); tokio::spawn(token_daemon()); - RwLock::new(client) + ArcSwap::new(client.into()) }); pub static OAUTH_RATELIMIT_REMAINING: AtomicU16 = AtomicU16::new(99); @@ -177,7 +177,7 @@ fn request(method: &'static Method, path: String, redirect: bool, quarantine: bo let client: Client<_, Body> = CLIENT.clone(); let (token, vendor_id, device_id, user_agent, loid) = { - let client = block_on(OAUTH_CLIENT.read()); + let client = OAUTH_CLIENT.load_full(); ( client.token.clone(), client.headers_map.get("Client-Vendor-Id").cloned().unwrap_or_default(), diff --git a/src/oauth.rs b/src/oauth.rs index a3f4dc0..efdf41e 100644 --- a/src/oauth.rs +++ b/src/oauth.rs @@ -98,21 +98,13 @@ impl Oauth { Some(()) } - - async fn refresh(&mut self) -> Option<()> { - // Refresh is actually just a subsequent login with the same headers (without the old token - // or anything). This logic is handled in login, so we just call login again. - let refresh = self.login().await; - info!("Refreshing OAuth token... {}", if refresh.is_some() { "success" } else { "failed" }); - refresh - } } pub async fn token_daemon() { // Monitor for refreshing token loop { // Get expiry time - be sure to not hold the read lock - let expires_in = { OAUTH_CLIENT.read().await.expires_in }; + let expires_in = { OAUTH_CLIENT.load_full().expires_in }; // sleep for the expiry time minus 2 minutes let duration = Duration::from_secs(expires_in - 120); @@ -125,7 +117,7 @@ pub async fn token_daemon() { // Refresh token - in its own scope { - OAUTH_CLIENT.write().await.refresh().await; + force_refresh_token().await; } } } @@ -137,7 +129,8 @@ pub async fn force_refresh_token() { } trace!("Rolling over refresh token. Current rate limit: {}", OAUTH_RATELIMIT_REMAINING.load(Ordering::SeqCst)); - OAUTH_CLIENT.write().await.refresh().await; + let new_client = Oauth::new().await; + OAUTH_CLIENT.swap(new_client.into()); OAUTH_RATELIMIT_REMAINING.store(99, Ordering::SeqCst); OAUTH_IS_ROLLING_OVER.store(false, Ordering::SeqCst); } @@ -187,21 +180,21 @@ fn choose(list: &[T]) -> T { #[tokio::test(flavor = "multi_thread")] async fn test_oauth_client() { - assert!(!OAUTH_CLIENT.read().await.token.is_empty()); + assert!(!OAUTH_CLIENT.load_full().token.is_empty()); } #[tokio::test(flavor = "multi_thread")] async fn test_oauth_client_refresh() { - OAUTH_CLIENT.write().await.refresh().await.unwrap(); + force_refresh_token().await; } #[tokio::test(flavor = "multi_thread")] async fn test_oauth_token_exists() { - assert!(!OAUTH_CLIENT.read().await.token.is_empty()); + assert!(!OAUTH_CLIENT.load_full().token.is_empty()); } #[tokio::test(flavor = "multi_thread")] async fn test_oauth_headers_len() { - assert!(OAUTH_CLIENT.read().await.headers_map.len() >= 3); + assert!(OAUTH_CLIENT.load_full().headers_map.len() >= 3); } #[test] From ea87ec33a1a277c53e073dc1f80c211699989aaa Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Fri, 28 Jun 2024 22:28:58 -0400 Subject: [PATCH 06/17] fix(subreddit): handle plus-encoding errors even better (#163) * fix(subreddit): handle plus-encoding errors even better * chore(clippy): fix lint --- src/subreddit.rs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/subreddit.rs b/src/subreddit.rs index 00edcae..0560e1b 100644 --- a/src/subreddit.rs +++ b/src/subreddit.rs @@ -64,7 +64,7 @@ pub async fn community(req: Request) -> Result, String> { let post_sort = req.cookie("post_sort").map_or_else(|| "hot".to_string(), |c| c.value().to_string()); let sort = req.param("sort").unwrap_or_else(|| req.param("id").unwrap_or(post_sort)); - let mut sub_name = req.param("sub").unwrap_or(if front_page == "default" || front_page.is_empty() { + let sub_name = req.param("sub").unwrap_or(if front_page == "default" || front_page.is_empty() { if subscribed.is_empty() { "popular".to_string() } else { @@ -84,11 +84,6 @@ pub async fn community(req: Request) -> Result, String> { return Ok(redirect(&["/user/", &sub_name[2..]].concat())); } - // If multi-sub, replace + with url encoded + - if sub_name.contains('+') { - sub_name = sub_name.replace('+', "%2B"); - } - // Request subreddit metadata let sub = if !sub_name.contains('+') && sub_name != subscribed && sub_name != "popular" && sub_name != "all" { // Regular subreddit @@ -124,7 +119,7 @@ pub async fn community(req: Request) -> Result, String> { params.push_str(&format!("&geo_filter={geo_filter}")); } - let path = format!("/r/{sub_name}/{sort}.json?{}{params}", req.uri().query().unwrap_or_default()); + let path = format!("/r/{}/{sort}.json?{}{params}", sub_name.replace('+', "%2B"), req.uri().query().unwrap_or_default()); let url = String::from(req.uri().path_and_query().map_or("", |val| val.as_str())); let redirect_url = url[1..].replace('?', "%3F").replace('&', "%26").replace('+', "%2B"); let filters = get_filters(&req); From 0f7eba717e5ec478ed4e78f7ba9e638b59962d42 Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Fri, 28 Jun 2024 22:39:42 -0400 Subject: [PATCH 07/17] fix(client): Handle invalid reddit response of base URL location --- src/client.rs | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/client.rs b/src/client.rs index 7281df1..d6473ca 100644 --- a/src/client.rs +++ b/src/client.rs @@ -3,6 +3,7 @@ use cached::proc_macro::cached; use futures_lite::future::block_on; use futures_lite::{future::Boxed, FutureExt}; use hyper::client::HttpConnector; +use hyper::header::HeaderValue; use hyper::{body, body::Buf, client, header, Body, Client, Method, Request, Response, Uri}; use hyper_rustls::HttpsConnector; use libflate::gzip; @@ -21,6 +22,7 @@ use crate::server::RequestExt; use crate::utils::format_url; const REDDIT_URL_BASE: &str = "https://oauth.reddit.com"; +const ALTERNATIVE_REDDIT_URL_BASE: &str = "https://www.reddit.com"; pub static CLIENT: Lazy>> = Lazy::new(|| { let https = hyper_rustls::HttpsConnectorBuilder::new() @@ -221,12 +223,13 @@ fn request(method: &'static Method, path: String, redirect: bool, quarantine: bo if !redirect { return Ok(response); }; - + let location_header = response.headers().get(header::LOCATION); + if location_header == Some(&HeaderValue::from_static("https://www.reddit.com/")) { + return Err("Reddit response was invalid".to_string()); + } return request( method, - response - .headers() - .get(header::LOCATION) + location_header .map(|val| { // We need to make adjustments to the URI // we get back from Reddit. Namely, we @@ -239,7 +242,11 @@ fn request(method: &'static Method, path: String, redirect: bool, quarantine: bo // required. // // 2. Percent-encode the path. - let new_path = percent_encode(val.as_bytes(), CONTROLS).to_string().trim_start_matches(REDDIT_URL_BASE).to_string(); + let new_path = percent_encode(val.as_bytes(), CONTROLS) + .to_string() + .trim_start_matches(REDDIT_URL_BASE) + .trim_start_matches(ALTERNATIVE_REDDIT_URL_BASE) + .to_string(); format!("{new_path}{}raw_json=1", if new_path.contains('?') { "&" } else { "?" }) }) .unwrap_or_default() @@ -298,7 +305,7 @@ fn request(method: &'static Method, path: String, redirect: bool, quarantine: bo } } Err(e) => { - dbg_msg!("{} {}: {}", method, path, e); + dbg_msg!("{method} {REDDIT_URL_BASE}{path}: {}", e); Err(e.to_string()) } From 459a8e1245b9ebca20897dd83bf8d2fef5426677 Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Sat, 29 Jun 2024 00:20:19 -0400 Subject: [PATCH 08/17] refactor(log): shorten some logs --- src/client.rs | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/src/client.rs b/src/client.rs index d6473ca..39ec63c 100644 --- a/src/client.rs +++ b/src/client.rs @@ -339,23 +339,19 @@ pub async fn json(path: String, quarantine: bool) -> Result { Ok(response) => { let status = response.status(); - // Ratelimit remaining - if let Some(Ok(remaining)) = response.headers().get("x-ratelimit-remaining").map(|val| val.to_str()) { + let reset: Option = if let (Some(remaining), Some(reset), Some(used)) = ( + response.headers().get("x-ratelimit-remaining").and_then(|val| val.to_str().ok().map(|s| s.to_string())), + response.headers().get("x-ratelimit-reset").and_then(|val| val.to_str().ok().map(|s| s.to_string())), + response.headers().get("x-ratelimit-used").and_then(|val| val.to_str().ok().map(|s| s.to_string())), + ) { trace!( - "Ratelimit remaining: Header says {remaining}, we have {current_rate_limit}. {}", - if is_rolling_over { "Rolling over" } else { "" } + "Ratelimit remaining: Header says {}, we have {}. Rollover: {}. Ratelimit used: {}", + remaining, + current_rate_limit, + if is_rolling_over { "yes" } else { "no" }, + used, ); - } - - // Ratelimit used - if let Some(Ok(used)) = response.headers().get("x-ratelimit-used").map(|val| val.to_str()) { - trace!("Ratelimit used: {}", used); - } - - // Ratelimit reset - let reset = if let Some(Ok(reset)) = response.headers().get("x-ratelimit-reset").map(|val| val.to_str()) { - trace!("Ratelimit reset: {}", reset); - Some(reset.to_string()) + Some(reset) } else { None }; From c565ebfb01a696beccdc51ff9cd296141e781ba9 Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Sat, 29 Jun 2024 10:44:33 -0400 Subject: [PATCH 09/17] refactor(log): update some logs --- src/client.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/client.rs b/src/client.rs index 39ec63c..bc07168 100644 --- a/src/client.rs +++ b/src/client.rs @@ -345,11 +345,8 @@ pub async fn json(path: String, quarantine: bool) -> Result { response.headers().get("x-ratelimit-used").and_then(|val| val.to_str().ok().map(|s| s.to_string())), ) { trace!( - "Ratelimit remaining: Header says {}, we have {}. Rollover: {}. Ratelimit used: {}", - remaining, - current_rate_limit, + "Ratelimit remaining: Header says {remaining}, we have {current_rate_limit}. Resets in {reset}. Rollover: {}. Ratelimit used: {used}", if is_rolling_over { "yes" } else { "no" }, - used, ); Some(reset) } else { From beb4cf193b3fb03e1a93d462eb39930c3d7d0d68 Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Sat, 29 Jun 2024 11:48:42 -0400 Subject: [PATCH 10/17] fix(posts): manually sort by created date (#166) --- src/utils.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/utils.rs b/src/utils.rs index 4d412cc..52c66d1 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -309,6 +309,7 @@ pub struct Post { pub domain: String, pub rel_time: String, pub created: String, + pub created_ts: u64, pub num_duplicates: u64, pub comments: (String, String), pub gallery: Vec, @@ -340,6 +341,7 @@ impl Post { let data = &post["data"]; let (rel_time, created) = time(data["created_utc"].as_f64().unwrap_or_default()); + let created_ts = data["created_utc"].as_f64().unwrap_or_default().round() as u64; let score = data["score"].as_i64().unwrap_or_default(); let ratio: f64 = data["upvote_ratio"].as_f64().unwrap_or(1.0) * 100.0; let title = val(post, "title"); @@ -412,6 +414,7 @@ impl Post { poll: Poll::parse(&data["poll_data"]), rel_time, created, + created_ts, num_duplicates: post["data"]["num_duplicates"].as_u64().unwrap_or(0), comments: format_num(data["num_comments"].as_i64().unwrap_or_default()), gallery, @@ -420,7 +423,7 @@ impl Post { ws_url: val(post, "websocket_url"), }); } - + posts.sort_by(|a, b| b.created_ts.cmp(&a.created_ts)); Ok((posts, res["data"]["after"].as_str().unwrap_or_default().to_string())) } } @@ -673,6 +676,8 @@ pub async fn parse_post(post: &Value) -> Post { // Determine the type of media along with the media URL let (post_type, media, gallery) = Media::parse(&post["data"]).await; + let created_ts = post["data"]["created_utc"].as_f64().unwrap_or_default().round() as u64; + let awards: Awards = Awards::parse(&post["data"]["all_awardings"]); let permalink = val(post, "permalink"); @@ -743,6 +748,7 @@ pub async fn parse_post(post: &Value) -> Post { domain: val(post, "domain"), rel_time, created, + created_ts, num_duplicates: post["data"]["num_duplicates"].as_u64().unwrap_or(0), comments: format_num(post["data"]["num_comments"].as_i64().unwrap_or_default()), gallery, From f44638a2cb94c3cd98aaf01bbeb38753e6965d41 Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Sat, 29 Jun 2024 12:00:34 -0400 Subject: [PATCH 11/17] v0.35.0 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 30bdf02..66c0b66 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1038,7 +1038,7 @@ checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" [[package]] name = "redlib" -version = "0.34.0" +version = "0.35.0" dependencies = [ "arc-swap", "askama", diff --git a/Cargo.toml b/Cargo.toml index df90167..b7413e0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ name = "redlib" description = " Alternative private front-end to Reddit" license = "AGPL-3.0" repository = "https://github.com/redlib-org/redlib" -version = "0.34.0" +version = "0.35.0" authors = [ "Matthew Esposito ", "spikecodes <19519553+spikecodes@users.noreply.github.com>", From f74d1affb6fc2aa37c2205e4cd8cb835caabec68 Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Sat, 29 Jun 2024 13:26:09 -0400 Subject: [PATCH 12/17] fix(posts): manually sort by flags (#168) * fix(posts): manually sort by flags * fix(posts): shorten sort call --- src/utils.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/utils.rs b/src/utils.rs index 52c66d1..52f81a3 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -424,6 +424,7 @@ impl Post { }); } posts.sort_by(|a, b| b.created_ts.cmp(&a.created_ts)); + posts.sort_by(|a, b| b.flags.stickied.cmp(&a.flags.stickied)); Ok((posts, res["data"]["after"].as_str().unwrap_or_default().to_string())) } } From d9e768100460dabb8016288ca17f22bdbada3a53 Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Sat, 29 Jun 2024 13:28:18 -0400 Subject: [PATCH 13/17] v0.35.1 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 66c0b66..4080ec0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1038,7 +1038,7 @@ checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" [[package]] name = "redlib" -version = "0.35.0" +version = "0.35.1" dependencies = [ "arc-swap", "askama", diff --git a/Cargo.toml b/Cargo.toml index b7413e0..5f8b339 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ name = "redlib" description = " Alternative private front-end to Reddit" license = "AGPL-3.0" repository = "https://github.com/redlib-org/redlib" -version = "0.35.0" +version = "0.35.1" authors = [ "Matthew Esposito ", "spikecodes <19519553+spikecodes@users.noreply.github.com>", From 366bc17f97385ab6e6d38036c88c40cca19e85c1 Mon Sep 17 00:00:00 2001 From: Pim Date: Mon, 1 Jul 2024 23:15:50 +0200 Subject: [PATCH 14/17] feat: show post link title for comments on user page (#169) --- src/utils.rs | 3 +++ static/style.css | 20 +++++++++++++++++++- templates/user.html | 6 ++++-- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/utils.rs b/src/utils.rs index 52f81a3..ea0045d 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -298,6 +298,7 @@ pub struct Post { pub body: String, pub author: Author, pub permalink: String, + pub link_title: String, pub poll: Option, pub score: (String, String), pub upvote_ratio: i64, @@ -411,6 +412,7 @@ impl Post { stickied: data["stickied"].as_bool().unwrap_or_default() || data["pinned"].as_bool().unwrap_or_default(), }, permalink: val(post, "permalink"), + link_title: val(post, "link_title"), poll: Poll::parse(&data["poll_data"]), rel_time, created, @@ -715,6 +717,7 @@ pub async fn parse_post(post: &Value) -> Post { distinguished: val(post, "distinguished"), }, permalink, + link_title: val(post, "link_title"), poll, score: format_num(score), upvote_ratio: ratio as i64, diff --git a/static/style.css b/static/style.css index 05b4e4b..f7e8854 100644 --- a/static/style.css +++ b/static/style.css @@ -1255,10 +1255,28 @@ a.search_subreddit:hover { min-width: 0; } -.comment_data > * { +.comment:has([id]) .comment_data > * { margin-right: 5px; } +.comment:not([id]) .comment_data { + display: inline-flex; + max-width: 100%; +} + +.comment:not([id]) .comment_data > * { + flex: 0 0 auto; +} + +.comment:not([id]) .comment_data > .comment_link { + display: -webkit-box; + -webkit-line-clamp: 1; + -webkit-box-orient: vertical; + word-break: break-all; + overflow: hidden; + flex: 0 1 auto; +} + .comment_image { max-width: 500px; align-self: center; diff --git a/templates/user.html b/templates/user.html index 42019e7..35e58b2 100644 --- a/templates/user.html +++ b/templates/user.html @@ -63,8 +63,10 @@
- Comment on r/{{ post.community }} - {{ post.rel_time }} + {{ post.link_title }} +  in  + r/{{ post.community }} +  {{ post.rel_time }}

{{ post.body|safe }}

From 67a890cab30e899650d40aa5c3d5416d3958c723 Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Tue, 2 Jul 2024 08:04:27 -0400 Subject: [PATCH 15/17] fix(posts): fix sort call on new (#171) --- src/subreddit.rs | 4 ++++ src/utils.rs | 2 -- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/subreddit.rs b/src/subreddit.rs index 0560e1b..8aea21b 100644 --- a/src/subreddit.rs +++ b/src/subreddit.rs @@ -145,6 +145,10 @@ pub async fn community(req: Request) -> Result, String> { let (_, all_posts_filtered) = filter_posts(&mut posts, &filters); let no_posts = posts.is_empty(); let all_posts_hidden_nsfw = !no_posts && (posts.iter().all(|p| p.flags.nsfw) && setting(&req, "show_nsfw") != "on"); + if sort == "new" { + posts.sort_by(|a, b| b.created_ts.cmp(&a.created_ts)); + posts.sort_by(|a, b| b.flags.stickied.cmp(&a.flags.stickied)); + } Ok(template(&SubredditTemplate { sub, posts, diff --git a/src/utils.rs b/src/utils.rs index ea0045d..5e8d83c 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -425,8 +425,6 @@ impl Post { ws_url: val(post, "websocket_url"), }); } - posts.sort_by(|a, b| b.created_ts.cmp(&a.created_ts)); - posts.sort_by(|a, b| b.flags.stickied.cmp(&a.flags.stickied)); Ok((posts, res["data"]["after"].as_str().unwrap_or_default().to_string())) } } From 8a917fcde3f933107ad2899186d4026de6dd8114 Mon Sep 17 00:00:00 2001 From: Pim Date: Fri, 5 Jul 2024 03:32:12 +0200 Subject: [PATCH 16/17] feat: add download button on image/gif/video posts (#173) * feat: add download button on image/gif/video posts * chore: fix formatting * chore: dont create reference --- src/utils.rs | 43 +++++++++++++++++++++++++++++++++++++++++++ static/style.css | 9 +++++---- templates/utils.html | 27 ++++++++++++++++++++------- 3 files changed, 68 insertions(+), 11 deletions(-) diff --git a/src/utils.rs b/src/utils.rs index 5e8d83c..b11096f 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -169,6 +169,7 @@ pub struct Media { pub width: i64, pub height: i64, pub poster: String, + pub download_name: String, } impl Media { @@ -235,6 +236,15 @@ impl Media { let alt_url = alt_url_val.map_or(String::new(), |val| format_url(val.as_str().unwrap_or_default())); + let download_name = if post_type == "image" || post_type == "gif" || post_type == "video" { + let permalink_base = url_path_basename(data["permalink"].as_str().unwrap_or_default()); + let media_url_base = url_path_basename(url_val.as_str().unwrap_or_default()); + + format!("redlib_{permalink_base}_{media_url_base}") + } else { + String::new() + }; + ( post_type.to_string(), Self { @@ -245,6 +255,7 @@ impl Media { width: source["width"].as_i64().unwrap_or_default(), height: source["height"].as_i64().unwrap_or_default(), poster: format_url(source["url"].as_str().unwrap_or_default()), + download_name, }, gallery, ) @@ -389,6 +400,7 @@ impl Post { width: data["thumbnail_width"].as_i64().unwrap_or_default(), height: data["thumbnail_height"].as_i64().unwrap_or_default(), poster: String::new(), + download_name: String::new(), }, media, domain: val(post, "domain"), @@ -727,6 +739,7 @@ pub async fn parse_post(post: &Value) -> Post { width: post["data"]["thumbnail_width"].as_i64().unwrap_or_default(), height: post["data"]["thumbnail_height"].as_i64().unwrap_or_default(), poster: String::new(), + download_name: String::new(), }, flair: Flair { flair_parts: FlairPart::parse( @@ -1110,6 +1123,20 @@ pub async fn nsfw_landing(req: Request, req_url: String) -> Result String { + let url_result = Url::parse(format!("https://libredd.it/{path}").as_str()); + + if url_result.is_err() { + path.to_string() + } else { + let mut url = url_result.unwrap(); + url.path_segments_mut().unwrap().pop_if_empty(); + + url.path_segments().unwrap().last().unwrap().to_string() + } +} + #[cfg(test)] mod tests { use super::{format_num, format_url, rewrite_urls}; @@ -1218,3 +1245,19 @@ fn test_rewriting_image_links() { let output = r#"

caption 1
li.desktop_item { +.desktop_item { display: auto; } @media screen and (min-width: 481px) { - #post_links > li.mobile_item { + .mobile_item { display: none; } } @@ -1770,10 +1770,11 @@ td, th { } #post_links > li { margin-right: 10px } - #post_links > li.desktop_item { display: none } - #post_links > li.mobile_item { display: auto } .post_footer > p > span#upvoted { display: none } + .desktop_item { display: none } + .mobile_item { display: auto } + .popup { width: auto; } diff --git a/templates/utils.html b/templates/utils.html index 8edb55b..e1d317a 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -164,13 +164,28 @@ Upvotes

{{ post.upvote_ratio }}% Upvoted

@@ -178,8 +193,7 @@ {%- endmacro %} {% macro external_reddit_link(permalink) %} -{% for dev_type in ["desktop", "mobile"] %} -
  • +
  • -{% endfor %} {% endmacro %} {% macro post_in_list(post) -%} From 4f213886436423c935961b46d9da8388fa95fbfb Mon Sep 17 00:00:00 2001 From: Pim Date: Fri, 5 Jul 2024 22:33:06 +0200 Subject: [PATCH 17/17] fix: also use hls if possible for gifs in post_in_list macro (#177) --- templates/utils.html | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/templates/utils.html b/templates/utils.html index e1d317a..c5ba45f 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -262,11 +262,7 @@ {% endif %} - {% else if (prefs.layout.is_empty() || prefs.layout == "card") && post.post_type == "gif" %} -
    - -
    - {% else if (prefs.layout.is_empty() || prefs.layout == "card") && post.post_type == "video" %} + {% else if (prefs.layout.is_empty() || prefs.layout == "card") && (post.post_type == "gif" || post.post_type == "video") %} {% if prefs.use_hls == "on" && !post.media.alt_url.is_empty() %}