From 6a18ea17eca8347ac3931ffe8b55c62304afb5e1 Mon Sep 17 00:00:00 2001 From: pratclot Date: Mon, 11 Nov 2024 02:19:40 +0100 Subject: [PATCH 01/16] Use quotes for kaniko to expand ARG in Dockerfile (#314) --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 314a784..8b275e7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ ARG TARGET RUN apk add --no-cache curl -RUN curl -L https://github.com/redlib-org/redlib/releases/latest/download/redlib-${TARGET}.tar.gz | \ +RUN curl -L "https://github.com/redlib-org/redlib/releases/latest/download/redlib-${TARGET}.tar.gz" | \ tar xz -C /usr/local/bin/ RUN adduser --home /nonexistent --no-create-home --disabled-password redlib From a301afc3837319ed7ef49a068309b4112d0ba836 Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Wed, 13 Nov 2024 16:43:41 -0500 Subject: [PATCH 02/16] fix(scraper): truncate to post count --- src/scraper/main.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/scraper/main.rs b/src/scraper/main.rs index f59b145..bc154ff 100644 --- a/src/scraper/main.rs +++ b/src/scraper/main.rs @@ -65,6 +65,8 @@ async fn main() { std::io::stdout().flush().unwrap(); } + posts.truncate(final_count); + match format { Format::Json => { let filename: String = output.unwrap_or_else(|| format!("{sub}.json")); From 62717ef6b2a4f82f92e8ee14e58cb179e2f9ebfa Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Thu, 14 Nov 2024 11:49:47 -0500 Subject: [PATCH 03/16] fix: update error template --- static/check_update.js | 2 ++ templates/error.html | 2 ++ 2 files changed, 4 insertions(+) diff --git a/static/check_update.js b/static/check_update.js index 7741edb..b68c508 100644 --- a/static/check_update.js +++ b/static/check_update.js @@ -21,8 +21,10 @@ async function checkInstanceUpdateStatus() { statusMessage = '✅ Instance is up to date.'; } else if (commitIndex > 0) { statusMessage = `⚠️ This instance is not up to date and is ${commitIndex} commits old. Test and confirm on an up-to-date instance before reporting.`; + document.getElementById('error-318').remove(); } else { statusMessage = `⚠️ This instance is not up to date and is at least ${commitHashes.length} commits old. Test and confirm on an up-to-date instance before reporting.`; + document.getElementById('error-318').remove(); } } else { statusMessage = '⚠️ Unable to fetch commit information.'; diff --git a/templates/error.html b/templates/error.html index f7a2d5d..8f2f44d 100644 --- a/templates/error.html +++ b/templates/error.html @@ -15,6 +15,8 @@ href="https://github.com/redlib-org/redlib/issues/new?assignees=&labels=bug&projects=&template=bug_report.md&title=%F0%9F%90%9B+Bug+Report%3A+{{ msg }}">Report an issue
+

If you're getting a "Failed to parse page JSON data" error, please check #318

+

Head back home?

{% endblock %} From 6c64ebd56b98f5616c2014e2e0567fa37791844c Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Fri, 15 Nov 2024 16:53:00 -0500 Subject: [PATCH 04/16] fix(scraper): additionally grab common words --- Cargo.lock | 46 ++++++++++++++++++++++++++ Cargo.toml | 1 + src/scraper/main.rs | 79 ++++++++++++++++++++++++++++++++++++++------- 3 files changed, 114 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d4a1a05..6447c2c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -71,6 +71,12 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8365de52b16c035ff4fcafe0092ba9390540e3e352870ac09933bebcaa2c8c56" +[[package]] +name = "anyhow" +version = "1.0.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775" + [[package]] name = "arc-swap" version = "1.7.1" @@ -307,6 +313,18 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" +[[package]] +name = "common-words-all" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84a6ff47eb813c9e315610ceca0ddd247827e22f2cdadc4189e4676a81470c77" +dependencies = [ + "anyhow", + "csv", + "glob", + "serde", +] + [[package]] name = "cookie" version = "0.18.1" @@ -370,6 +388,27 @@ dependencies = [ "typenum", ] +[[package]] +name = "csv" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" +dependencies = [ + "memchr", +] + [[package]] name = "darling" version = "0.20.10" @@ -642,6 +681,12 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + [[package]] name = "globset" version = "0.4.15" @@ -1160,6 +1205,7 @@ dependencies = [ "build_html", "cached", "clap", + "common-words-all", "cookie", "dotenvy", "fastrand", diff --git a/Cargo.toml b/Cargo.toml index 2fe1456..dac3f48 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,6 +48,7 @@ rss = "2.0.7" arc-swap = "1.7.1" serde_json_path = "0.6.7" async-recursion = "1.1.1" +common-words-all = { version = "0.0.2", default-features = false, features = ["english", "one"] } [dev-dependencies] diff --git a/src/scraper/main.rs b/src/scraper/main.rs index bc154ff..f2e48d6 100644 --- a/src/scraper/main.rs +++ b/src/scraper/main.rs @@ -1,6 +1,7 @@ -use std::{fmt::Display, io::Write}; +use std::{collections::HashMap, fmt::Display, io::Write}; use clap::{Parser, ValueEnum}; +use common_words_all::{get_top, Language, NgramSize}; use redlib::utils::Post; #[derive(Parser)] @@ -10,9 +11,6 @@ struct Cli { #[arg(short = 's', long = "sub")] sub: String, - #[arg(short = 'c', long = "count")] - count: usize, - #[arg(long = "sort")] sort: SortOrder, @@ -50,28 +48,85 @@ enum Format { #[tokio::main] async fn main() { + pretty_env_logger::init(); let cli = Cli::parse(); - let (sub, final_count, sort, format, output) = (cli.sub, cli.count, cli.sort, cli.format, cli.output); + let (sub, sort, format, output) = (cli.sub, cli.sort, cli.format, cli.output); let initial = format!("/r/{sub}/{sort}.json?&raw_json=1"); - let (mut posts, mut after) = Post::fetch(&initial, false).await.unwrap(); - while posts.len() < final_count { + let (posts, mut after) = Post::fetch(&initial, false).await.unwrap(); + let mut hashmap = HashMap::new(); + hashmap.extend(posts.into_iter().map(|post| (post.id.clone(), post))); + loop { print!("\r"); let path = format!("/r/{sub}/{sort}.json?sort={sort}&t=&after={after}&raw_json=1"); let (new_posts, new_after) = Post::fetch(&path, false).await.unwrap(); - posts.extend(new_posts); + let old_len = hashmap.len(); + // convert to hashmap and extend hashmap + let new_posts = new_posts.into_iter().map(|post| (post.id.clone(), post)).collect::>(); + let len = new_posts.len(); + hashmap.extend(new_posts); + if hashmap.len() - old_len < 3 { + break; + } + + let x = hashmap.len() - old_len; after = new_after; // Print number of posts fetched - print!("Fetched {} posts", posts.len()); + print!("Fetched {len} posts (+{x})",); std::io::stdout().flush().unwrap(); } + println!("\n\n"); + // additionally search if final count not reached - posts.truncate(final_count); + for word in get_top(Language::English, 10_000, NgramSize::One) { + let mut retrieved_posts_from_search = 0; + let initial = format!("/r/{sub}/search.json?q={word}&restrict_sr=on&include_over_18=on&raw_json=1&sort={sort}"); + println!("Grabbing posts with word {word}."); + let (posts, mut after) = Post::fetch(&initial, false).await.unwrap(); + hashmap.extend(posts.into_iter().map(|post| (post.id.clone(), post))); + 'search: loop { + let path = format!("/r/{sub}/search.json?q={word}&restrict_sr=on&include_over_18=on&raw_json=1&sort={sort}&after={after}"); + let (new_posts, new_after) = Post::fetch(&path, false).await.unwrap(); + if new_posts.is_empty() || new_after.is_empty() { + println!("No more posts for word {word}"); + break 'search; + } + retrieved_posts_from_search += new_posts.len(); + let old_len = hashmap.len(); + let new_posts = new_posts.into_iter().map(|post| (post.id.clone(), post)).collect::>(); + let len = new_posts.len(); + hashmap.extend(new_posts); + let delta = hashmap.len() - old_len; + after = new_after; + // Print number of posts fetched + println!("Fetched {len} posts (+{delta})",); + if retrieved_posts_from_search > 1000 { + println!("Reached 1000 posts from search"); + break 'search; + } + } + // Need to save incrementally. atomic save + move + let tmp_file = output.clone().unwrap_or_else(|| format!("{sub}.json.tmp")); + let perm_file = output.clone().unwrap_or_else(|| format!("{sub}.json")); + write_posts(&hashmap.values().collect(), tmp_file.clone()); + // move file + std::fs::rename(tmp_file, perm_file).unwrap(); + } + + println!("\n\n"); + + println!("Size of hashmap: {}", hashmap.len()); + + let posts: Vec<&Post> = hashmap.values().collect(); match format { Format::Json => { let filename: String = output.unwrap_or_else(|| format!("{sub}.json")); - let json = serde_json::to_string(&posts).unwrap(); - std::fs::write(filename, json).unwrap(); + write_posts(&posts, filename); } } } + +fn write_posts(posts: &Vec<&Post>, filename: String) { + let json = serde_json::to_string(&posts).unwrap(); + std::fs::write(filename, json).unwrap(); +} From a96bebb0991558ff278b4cabf3c54678d7d8b208 Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Tue, 19 Nov 2024 11:08:00 -0500 Subject: [PATCH 05/16] fix(client): switch to hyper-tls --- Cargo.lock | 209 ++++++++++++++++++++++++-------------------------- Cargo.toml | 4 +- src/client.rs | 9 +-- 3 files changed, 107 insertions(+), 115 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6447c2c..6906b65 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "addr2line" @@ -139,12 +139,6 @@ dependencies = [ "windows-targets", ] -[[package]] -name = "base64" -version = "0.21.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" - [[package]] name = "base64" version = "0.22.1" @@ -253,9 +247,9 @@ checksum = "ade8366b8bd5ba243f0a58f036cc0ca8a2f069cff1a2351ef1cac6b083e16fc0" [[package]] name = "cc" -version = "1.1.31" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2e7962b54006dcfcc61cb72735f4d89bb97061dd6a7ed882ec6b8ee53714c6f" +checksum = "fd9de9f2205d5ef3fd67e685b0df337994ddd4495e2a28d185500d0e1edfea47" dependencies = [ "shlex", ] @@ -565,6 +559,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -818,19 +827,16 @@ dependencies = [ ] [[package]] -name = "hyper-rustls" -version = "0.24.2" +name = "hyper-tls" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ - "futures-util", - "http", + "bytes", "hyper", - "log", - "rustls", - "rustls-native-certs", + "native-tls", "tokio", - "tokio-rustls", + "tokio-native-tls", ] [[package]] @@ -1002,6 +1008,23 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "native-tls" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "never" version = "0.1.0" @@ -1057,12 +1080,50 @@ version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" +[[package]] +name = "openssl" +version = "0.10.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "openssl-probe" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "openssl-sys" +version = "0.9.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45abf306cbf99debc8195b66b7346498d7b10c210de50418b5ccd7ceba08c741" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "parking" version = "2.2.1" @@ -1110,6 +1171,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkg-config" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" + [[package]] name = "powerfmt" version = "0.2.0" @@ -1200,7 +1267,7 @@ version = "0.35.1" dependencies = [ "arc-swap", "async-recursion", - "base64 0.22.1", + "base64", "brotli", "build_html", "cached", @@ -1211,7 +1278,7 @@ dependencies = [ "fastrand", "futures-lite", "hyper", - "hyper-rustls", + "hyper-tls", "libflate", "lipsum", "log", @@ -1273,21 +1340,6 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" -[[package]] -name = "ring" -version = "0.17.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" -dependencies = [ - "cc", - "cfg-if", - "getrandom", - "libc", - "spin", - "untrusted", - "windows-sys 0.52.0", -] - [[package]] name = "rinja" version = "0.3.5" @@ -1408,49 +1460,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "rustls" -version = "0.21.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" -dependencies = [ - "log", - "ring", - "rustls-webpki", - "sct", -] - -[[package]] -name = "rustls-native-certs" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" -dependencies = [ - "openssl-probe", - "rustls-pemfile", - "schannel", - "security-framework", -] - -[[package]] -name = "rustls-pemfile" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" -dependencies = [ - "base64 0.21.7", -] - -[[package]] -name = "rustls-webpki" -version = "0.101.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "rusty-forkfork" version = "0.4.0" @@ -1480,9 +1489,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01227be5826fa0690321a2ba6c5cd57a19cf3f6a09e76973b58e61de6ab9d1c1" +checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" dependencies = [ "windows-sys 0.59.0", ] @@ -1493,16 +1502,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "sct" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "sealed_test" version = "1.1.0" @@ -1540,9 +1539,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.12.0" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea4a292869320c0272d7bc55a5a6aafaff59b4f63404a003887b679a2e05b4b6" +checksum = "fa39c7303dc58b5543c94d22c1766b0d31f2ee58306363ea622b10bbc075eaa2" dependencies = [ "core-foundation-sys", "libc", @@ -1706,12 +1705,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "spin" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" - [[package]] name = "strsim" version = "0.11.1" @@ -1849,12 +1842,12 @@ dependencies = [ ] [[package]] -name = "tokio-rustls" -version = "0.24.1" +name = "tokio-native-tls" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" dependencies = [ - "rustls", + "native-tls", "tokio", ] @@ -1975,12 +1968,6 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" -[[package]] -name = "untrusted" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" - [[package]] name = "url" version = "2.5.2" @@ -2001,6 +1988,12 @@ dependencies = [ "getrandom", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" diff --git a/Cargo.toml b/Cargo.toml index dac3f48..96c7952 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,8 +23,7 @@ regex = "1.10.2" serde = { version = "1.0.193", features = ["derive"] } cookie = "0.18.0" futures-lite = "2.2.0" -hyper = { version = "0.14.28", features = ["full"] } -hyper-rustls = { version = "0.24.2", features = [ "http2" ] } +hyper = { version = "0.14.31", features = ["full"] } percent-encoding = "2.3.1" route-recognizer = "0.3.1" serde_json = "1.0.108" @@ -49,6 +48,7 @@ arc-swap = "1.7.1" serde_json_path = "0.6.7" async-recursion = "1.1.1" common-words-all = { version = "0.0.2", default-features = false, features = ["english", "one"] } +hyper-tls = "=0.5.0" [dev-dependencies] diff --git a/src/client.rs b/src/client.rs index 3d20db0..a0a01d2 100644 --- a/src/client.rs +++ b/src/client.rs @@ -4,8 +4,9 @@ use futures_lite::future::block_on; use futures_lite::{future::Boxed, FutureExt}; use hyper::client::HttpConnector; use hyper::header::HeaderValue; -use hyper::{body, body::Buf, header, Body, Client, Method, Request, Response, Uri}; -use hyper_rustls::HttpsConnector; +use hyper::Client; +use hyper::{body, body::Buf, header, Body, Method, Request, Response, Uri}; +use hyper_tls::HttpsConnector; use libflate::gzip; use log::{error, trace, warn}; use once_cell::sync::Lazy; @@ -30,8 +31,7 @@ const REDDIT_SHORT_URL_BASE_HOST: &str = "redd.it"; const ALTERNATIVE_REDDIT_URL_BASE: &str = "https://www.reddit.com"; const ALTERNATIVE_REDDIT_URL_BASE_HOST: &str = "www.reddit.com"; -pub static HTTPS_CONNECTOR: Lazy> = - Lazy::new(|| hyper_rustls::HttpsConnectorBuilder::new().with_native_roots().https_only().enable_http2().build()); +pub static HTTPS_CONNECTOR: Lazy> = Lazy::new(HttpsConnector::new); pub static CLIENT: Lazy>> = Lazy::new(|| Client::builder().build::<_, Body>(HTTPS_CONNECTOR.clone())); @@ -242,7 +242,6 @@ fn request(method: &'static Method, path: String, redirect: bool, quarantine: bo .header("Authorization", &format!("Bearer {token}")) .header("Accept-Encoding", if method == Method::GET { "gzip" } else { "identity" }) .header("Accept-Language", "en-US,en;q=0.5") - .header("Connection", "keep-alive") .header( "Cookie", if quarantine { From 0714d58efe0b6d1e1bb94890ea2a402ffe86e944 Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Tue, 19 Nov 2024 11:12:04 -0500 Subject: [PATCH 06/16] fix(ci): install new openssl requirements --- .github/workflows/build-artifacts.yaml | 6 +++--- .github/workflows/main-rust.yml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-artifacts.yaml b/.github/workflows/build-artifacts.yaml index 695d1bf..673ef54 100644 --- a/.github/workflows/build-artifacts.yaml +++ b/.github/workflows/build-artifacts.yaml @@ -38,17 +38,17 @@ jobs: - if: matrix.target == 'x86_64-unknown-linux-musl' run: | sudo apt-get update - sudo apt-get install -y --no-install-recommends musl-tools + sudo apt-get install -y --no-install-recommends musl-tools libssl-dev - if: matrix.target == 'armv7-unknown-linux-musleabihf' run: | sudo apt update - sudo apt install -y gcc-arm-linux-gnueabihf musl-tools + sudo apt install -y gcc-arm-linux-gnueabihf musl-tools libssl-dev - if: matrix.target == 'aarch64-unknown-linux-musl' run: | sudo apt update - sudo apt install -y gcc-aarch64-linux-gnu musl-tools + sudo apt install -y gcc-aarch64-linux-gnu musl-tools libssl-dev - name: Versions id: version diff --git a/.github/workflows/main-rust.yml b/.github/workflows/main-rust.yml index f38c01d..64f30b3 100644 --- a/.github/workflows/main-rust.yml +++ b/.github/workflows/main-rust.yml @@ -30,8 +30,8 @@ jobs: with: toolchain: stable - - name: Install musl-gcc - run: sudo apt-get install musl-tools + - name: Install musl-gcc and libssl-dev + run: sudo apt-get install musl-tools libssl-dev - name: Install cargo musl target run: rustup target add x86_64-unknown-linux-musl From f7240208f1f7204a108af193f7270ad00dc8e402 Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Tue, 19 Nov 2024 11:18:20 -0500 Subject: [PATCH 07/16] fix(tls): vendor native-tls --- Cargo.lock | 10 ++++++++++ Cargo.toml | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 6906b65..fdb2d9b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1112,6 +1112,15 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "openssl-src" +version = "300.4.1+3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faa4eac4138c62414b5622d1b31c5c304f34b406b013c079c2bbc652fdd6678c" +dependencies = [ + "cc", +] + [[package]] name = "openssl-sys" version = "0.9.104" @@ -1120,6 +1129,7 @@ checksum = "45abf306cbf99debc8195b66b7346498d7b10c210de50418b5ccd7ceba08c741" dependencies = [ "cc", "libc", + "openssl-src", "pkg-config", "vcpkg", ] diff --git a/Cargo.toml b/Cargo.toml index 96c7952..e130f8c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,7 +48,7 @@ arc-swap = "1.7.1" serde_json_path = "0.6.7" async-recursion = "1.1.1" common-words-all = { version = "0.0.2", default-features = false, features = ["english", "one"] } -hyper-tls = "=0.5.0" +hyper-tls = { version = "=0.5.0", features = ["vendored"] } [dev-dependencies] From f8a9ad363de6acde0d4ffb9dcadad189fec0e8f0 Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Tue, 19 Nov 2024 11:37:30 -0500 Subject: [PATCH 08/16] chore(deps): updates --- Cargo.lock | 520 ++++++++++++++++++++++++++++++++++++++++++----------- Cargo.toml | 6 +- 2 files changed, 414 insertions(+), 112 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fdb2d9b..846d8dd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -61,15 +61,15 @@ dependencies = [ [[package]] name = "allocator-api2" -version = "0.2.18" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" +checksum = "45862d1c77f2228b9e10bc609d5bc203d86ebc9b87ad8d5d5167a6c9abf739d9" [[package]] name = "anstyle" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8365de52b16c035ff4fcafe0092ba9390540e3e352870ac09933bebcaa2c8c56" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" [[package]] name = "anyhow" @@ -107,9 +107,9 @@ dependencies = [ [[package]] name = "atom_syndication" -version = "0.12.4" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a3a5ed3201df5658d1aa45060c5a57dc9dba8a8ada20d696d67cb0c479ee043" +checksum = "3ee79fb83c725eae67b55218870813d2fc39fd85e4f1583848ef9f4f823cfe7c" dependencies = [ "chrono", "derive_builder", @@ -183,9 +183,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40723b8fb387abc38f4f4a37c09073622e41dd12327033091ef8950659e6dc0c" +checksum = "1a68f1f47cdf0ec8ee4b941b2eee2a80cb796db73118c0dd09ac63fbe405be22" dependencies = [ "memchr", "serde", @@ -197,6 +197,12 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "225eb82ce9e70dcc0cfa6e404d0f353326b6e163bf500ec4711cec317d11935c" +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + [[package]] name = "byteorder" version = "1.5.0" @@ -211,9 +217,9 @@ checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" [[package]] name = "cached" -version = "0.51.4" +version = "0.54.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0feb64151eed3da6107fddd2d717a6ca4b9dbd74e43784c55c841d1abfe5a295" +checksum = "9718806c4a2fe9e8a56fd736f97b340dd10ed1be8ed733ed50449f351dc33cae" dependencies = [ "ahash", "async-trait", @@ -221,17 +227,17 @@ dependencies = [ "cached_proc_macro_types", "futures", "hashbrown 0.14.5", - "instant", "once_cell", "thiserror", "tokio", + "web-time", ] [[package]] name = "cached_proc_macro" -version = "0.21.0" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "771aa57f3b17da6c8bcacb187bb9ec9bc81c8160e72342e67c329e0e1651a669" +checksum = "2f42a145ed2d10dce2191e1dcf30cfccfea9026660e143662ba5eec4017d5daa" dependencies = [ "darling", "proc-macro2", @@ -271,9 +277,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.20" +version = "4.5.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8" +checksum = "fb3b4b9e5a7c7514dfa52869339ee98b3156b0bfb4e8a77c4ff4babb64b1604f" dependencies = [ "clap_builder", "clap_derive", @@ -281,9 +287,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.20" +version = "4.5.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54" +checksum = "b17a95aa67cc7b5ebd32aa5370189aa0d79069ef1c64ce893bd30fb24bff20ec" dependencies = [ "anstyle", "clap_lex", @@ -303,9 +309,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" +checksum = "afb84c814227b90d6895e01398aee0d8033c00e7466aca416fb6a8e0eb19d8a7" [[package]] name = "common-words-all" @@ -356,9 +362,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" +checksum = "0ca741a962e1b0bff6d724a1a0958b686406e853bb14061f218562e1896f95e6" dependencies = [ "libc", ] @@ -496,13 +502,24 @@ dependencies = [ [[package]] name = "diligent-date-parser" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6cf7fe294274a222363f84bcb63cdea762979a0443b4cf1f4f8fd17c86b1182" +checksum = "c8ede7d79366f419921e2e2f67889c12125726692a313bffb474bd5f37a581e9" dependencies = [ "chrono", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "dotenvy" version = "0.15.7" @@ -549,9 +566,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" +checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4" [[package]] name = "fnv" @@ -627,9 +644,9 @@ checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-lite" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f1fa2f9765705486b33fd2acf1577f8ec449c2ba1f318ae5447697b7c08d210" +checksum = "cef40d21ae2c515b51041df9ed313ed21e572df340ea58a922a0aefe7e8891a1" dependencies = [ "fastrand", "futures-core", @@ -740,9 +757,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.0" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" +checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" [[package]] name = "heck" @@ -839,6 +856,124 @@ dependencies = [ "tokio-native-tls", ] +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -847,12 +982,23 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "idna" -version = "0.5.0" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +dependencies = [ + "icu_normalizer", + "icu_properties", ] [[package]] @@ -862,16 +1008,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", - "hashbrown 0.15.0", -] - -[[package]] -name = "instant" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" -dependencies = [ - "cfg-if", + "hashbrown 0.15.1", ] [[package]] @@ -898,10 +1035,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] -name = "libc" -version = "0.2.161" +name = "js-sys" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" +checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.164" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "433bfe06b8c75da9b2e3fbea6e5329ff87748f0b144ef75306e674c3f6f7c13f" [[package]] name = "libflate" @@ -943,6 +1089,12 @@ dependencies = [ "rand_chacha", ] +[[package]] +name = "litemap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" + [[package]] name = "lock_api" version = "0.4.12" @@ -1229,9 +1381,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quick-xml" -version = "0.36.2" +version = "0.37.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7649a7b4df05aed9ea7ec6f628c67c9953a43869b8bc50929569b2999d443fe" +checksum = "f22f29bdff3987b4d8632ef95fd6424ec7e4e0a57e2f4fc63e489e75357f6a03" dependencies = [ "encoding_rs", "memchr", @@ -1335,9 +1487,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", @@ -1400,9 +1552,9 @@ checksum = "afab94fb28594581f62d981211a9a4d53cc8130bbcbbb89a0440d9b8e81a7746" [[package]] name = "rss" -version = "2.0.9" +version = "2.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27e92048f840d98c6d6dd870af9101610ea9ff413f11f1bcebf4f4c31d96d957" +checksum = "554a62b3dd5450fcbb0435b3db809f9dd3c6e9f5726172408f7ad3b57ed59057" dependencies = [ "atom_syndication", "derive_builder", @@ -1459,9 +1611,9 @@ checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" [[package]] name = "rustix" -version = "0.38.38" +version = "0.38.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa260229e6538e52293eeb577aabd09945a09d6d9cc0fc550ed7529056c2e32a" +checksum = "d7f649912bc1495e167a6edee79151c84b1bad49748cb4f1f1167f459f6224f6" dependencies = [ "bitflags", "errno", @@ -1559,18 +1711,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.214" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5" +checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.214" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766" +checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" dependencies = [ "proc-macro2", "quote", @@ -1579,9 +1731,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.132" +version = "1.0.133" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" +checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" dependencies = [ "itoa", "memchr", @@ -1591,13 +1743,12 @@ dependencies = [ [[package]] name = "serde_json_path" -version = "0.6.7" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bc0207b6351893eafa1e39aa9aea452abb6425ca7b02dd64faf29109e7a33ba" +checksum = "e176fbf9bd62f75c2d8be33207fa13af2f800a506635e89759e46f934c520f4d" dependencies = [ "inventory", "nom", - "once_cell", "regex", "serde", "serde_json", @@ -1608,12 +1759,11 @@ dependencies = [ [[package]] name = "serde_json_path_core" -version = "0.1.6" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3d64fe53ce1aaa31bea2b2b46d3b6ab6a37e61854bedcbd9f174e188f3f7d79" +checksum = "ea3bfd54a421bec8328aefede43ac9f18c8c7ded3b2afc8addd44b4813d99fd0" dependencies = [ "inventory", - "once_cell", "serde", "serde_json", "thiserror", @@ -1621,21 +1771,20 @@ dependencies = [ [[package]] name = "serde_json_path_macros" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a31e8177a443fd3e94917f12946ae7891dfb656e6d4c5e79b8c5d202fbcb723" +checksum = "ee05bac728cc5232af5c23896b34fbdd17cf0bb0c113440588aeeb1b57c6ba1f" dependencies = [ "inventory", - "once_cell", "serde_json_path_core", "serde_json_path_macros_internal", ] [[package]] name = "serde_json_path_macros_internal" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75dde5a1d2ed78dfc411fc45592f72d3694436524d3353683ecb3d22009731dc" +checksum = "aafbefbe175fa9bf03ca83ef89beecff7d2a95aaacd5732325b90ac8c3bd7b90" dependencies = [ "proc-macro2", "quote", @@ -1715,6 +1864,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "strsim" version = "0.11.1" @@ -1723,9 +1878,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "syn" -version = "2.0.86" +version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e89275301d38033efb81a6e60e3497e734dfcc62571f2854bf4b16690398824c" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ "proc-macro2", "quote", @@ -1733,10 +1888,21 @@ dependencies = [ ] [[package]] -name = "tempfile" -version = "3.13.0" +name = "synstructure" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tempfile" +version = "3.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" dependencies = [ "cfg-if", "fastrand", @@ -1756,18 +1922,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.65" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d11abd9594d9b38965ef50805c5e469ca9cc6f197f883f717e0269a3057b3d5" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.65" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae71770322cbd277e69d762a16c444af02aa0575ac0d174f0b9562d3b37f8602" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", @@ -1808,25 +1974,20 @@ dependencies = [ ] [[package]] -name = "tinyvec" -version = "1.8.0" +name = "tinystr" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" dependencies = [ - "tinyvec_macros", + "displaydoc", + "zerovec", ] -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - [[package]] name = "tokio" -version = "1.41.0" +version = "1.41.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "145f3413504347a2be84393cc8a7d2fb4d863b375909ea59f2158261aa258bbb" +checksum = "22cfb5bee7a6a52939ca9224d6ac897bb669134078daa8735560897f69de4d33" dependencies = [ "backtrace", "bytes", @@ -1951,27 +2112,12 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e51b68083f157f853b6379db119d1c1be0e6e4dec98101079dec41f6f5cf6df" -[[package]] -name = "unicode-bidi" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" - [[package]] name = "unicode-ident" version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" -[[package]] -name = "unicode-normalization" -version = "0.1.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" -dependencies = [ - "tinyvec", -] - [[package]] name = "unsafe-libyaml" version = "0.2.11" @@ -1980,15 +2126,27 @@ checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" [[package]] name = "url" -version = "2.5.2" +version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" +checksum = "8d157f1b96d14500ffdc1f10ba712e780825526c03d9a49b4d0324b0d9113ada" dependencies = [ "form_urlencoded", "idna", "percent-encoding", ] +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "uuid" version = "1.11.0" @@ -2044,6 +2202,71 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-bindgen" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" +dependencies = [ + "cfg-if", + "once_cell", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "winapi-util" version = "0.1.9" @@ -2144,6 +2367,42 @@ dependencies = [ "memchr", ] +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + +[[package]] +name = "yoke" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.7.35" @@ -2164,3 +2423,46 @@ dependencies = [ "quote", "syn", ] + +[[package]] +name = "zerofrom" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/Cargo.toml b/Cargo.toml index e130f8c..77aec78 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ default-run = "redlib" [dependencies] rinja = { version = "0.3.4", default-features = false } -cached = { version = "0.51.3", features = ["async"] } +cached = { version = "0.54.0", features = ["async"] } clap = { version = "4.4.11", default-features = false, features = [ "std", "env", @@ -26,7 +26,7 @@ futures-lite = "2.2.0" hyper = { version = "0.14.31", features = ["full"] } percent-encoding = "2.3.1" route-recognizer = "0.3.1" -serde_json = "1.0.108" +serde_json = "1.0.133" tokio = { version = "1.35.1", features = ["full"] } time = { version = "0.3.31", features = ["local-offset"] } url = "2.5.0" @@ -45,7 +45,7 @@ pretty_env_logger = "0.5.0" dotenvy = "0.15.7" rss = "2.0.7" arc-swap = "1.7.1" -serde_json_path = "0.6.7" +serde_json_path = "0.7.1" async-recursion = "1.1.1" common-words-all = { version = "0.0.2", default-features = false, features = ["english", "one"] } hyper-tls = { version = "=0.5.0", features = ["vendored"] } From 96e40e888742966e73ff1265918c6d256d8d3726 Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Tue, 19 Nov 2024 11:40:17 -0500 Subject: [PATCH 09/16] style(clippy): small clippy change --- src/utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils.rs b/src/utils.rs index 170f53d..1edb528 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -507,7 +507,7 @@ impl std::ops::Deref for Awards { impl std::fmt::Display for Awards { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - self.iter().fold(Ok(()), |result, award| result.and_then(|()| writeln!(f, "{award}"))) + self.iter().try_fold((), |_, award| writeln!(f, "{award}")) } } From 3e1718bfc971017aacee180d497f398be2d13855 Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Tue, 19 Nov 2024 12:44:20 -0500 Subject: [PATCH 10/16] fix(client): ??? no accept language --- src/client.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/client.rs b/src/client.rs index a0a01d2..978214e 100644 --- a/src/client.rs +++ b/src/client.rs @@ -241,7 +241,6 @@ fn request(method: &'static Method, path: String, redirect: bool, quarantine: bo .header("Host", host) .header("Authorization", &format!("Bearer {token}")) .header("Accept-Encoding", if method == Method::GET { "gzip" } else { "identity" }) - .header("Accept-Language", "en-US,en;q=0.5") .header( "Cookie", if quarantine { From 96ebfd2d3a6d2b7c45c6cdf7dfc65140d2288747 Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Tue, 19 Nov 2024 12:53:36 -0500 Subject: [PATCH 11/16] fix(ci): statically build on artifacts --- .github/workflows/build-artifacts.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-artifacts.yaml b/.github/workflows/build-artifacts.yaml index 673ef54..9dd94c9 100644 --- a/.github/workflows/build-artifacts.yaml +++ b/.github/workflows/build-artifacts.yaml @@ -55,7 +55,7 @@ jobs: run: echo "VERSION=$(cargo metadata --format-version 1 --no-deps | jq .packages[0].version -r | sed 's/^/v/')" >> "$GITHUB_OUTPUT" - name: Build - run: cargo build --release --target ${{ matrix.target }} + run: RUSTFLAGS='-C target-feature=+crt-static' cargo build --release --target ${{ matrix.target }} - name: Package release run: tar czf redlib-${{ matrix.target }}.tar.gz -C target/${{ matrix.target }}/release/ redlib From 0bc36d529cd8208a07acfa65a071119f63cb7a9a Mon Sep 17 00:00:00 2001 From: James Musselman <134752354+musselmandev@users.noreply.github.com> Date: Tue, 19 Nov 2024 12:19:48 -0600 Subject: [PATCH 12/16] Add Quadlet Container File (#319) * Add Quadlet Container File * Update README.md with Quadlet instructions --- README.md | 36 +++++++++++++++++++++++++++++++++++- redlib.container | 16 ++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 redlib.container diff --git a/README.md b/README.md index 16be29e..4026791 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,9 @@ - [Docker](#docker) - [Docker Compose](#docker-compose) - [Docker CLI](#docker-cli) + - Podman + - Quadlets + - [Binary](#binary) - [Running as a systemd service](#running-as-a-systemd-service) - [Building from source](#building-from-source) @@ -180,7 +183,7 @@ For configuration options, see the [Configuration section](#Configuration). [Docker](https://www.docker.com) lets you run containerized applications. Containers are loosely isolated environments that are lightweight and contain everything needed to run the application, so there's no need to rely on what's installed on the host. -Docker images for Redlib are available at [quay.io](https://quay.io/repository/redlib/redlib), with support for `amd64`, `arm64`, and `armv7` platforms. +Container images for Redlib are available at [quay.io](https://quay.io/repository/redlib/redlib), with support for `amd64`, `arm64`, and `armv7` platforms. ### Docker Compose @@ -224,6 +227,37 @@ Stream logs from the Redlib container: ```bash docker logs -f redlib ``` +## Podman + +[Podman](https://podman.io/) lets you run containerized applications in a rootless fashion. Containers are loosely isolated environments that are lightweight and contain everything needed to run the application, so there's no need to rely on what's installed on the host. + +Container images for Redlib are available at [quay.io](https://quay.io/repository/redlib/redlib), with support for `amd64`, `arm64`, and `armv7` platforms. + +### Quadlets + +> [!IMPORTANT] +> These instructions assume that you are on a systemd based distro with [podman](https://podman.io/). If not, follow these [instructions on podman's website](https://podman.io/docs/installation) for how to do so. +> It also assumes you have used `loginctl enable-linger ` to enable the service to start for your user without logging in. + +Copy the `redlib.container` and `.env.example` files to `.config/containers/systemd/` and modify any relevant values (for example, the ports Redlib should listen on, renaming the .env file and editing its values, etc.). + +To start Redlib either reboot or follow the instructions below: + +Notify systemd of the new files +```bash +systemctl --user daemon-reload +``` + +Start the newly generated service file + +```bash +systemctl --user start redlib.service +``` + +You can check the status of your container by using the following command: +```bash +systemctl --user status redlib.service +``` ## Binary diff --git a/redlib.container b/redlib.container new file mode 100644 index 0000000..e66051e --- /dev/null +++ b/redlib.container @@ -0,0 +1,16 @@ +[Install] +WantedBy=default.target + +[Container] +AutoUpdate=registry +ContainerName=redlib +DropCapability=ALL +EnvironmentFile=.env +HealthCmd=["wget","--spider","-q","--tries=1","http://localhost:8080/settings"] +HealthInterval=5m +HealthTimeout=3s +Image=quay.io/redlib/redlib:latest +NoNewPrivileges=true +PublishPort=8080:8080 +ReadOnly=true +User=nobody From 18efb8c714ad10e0082059e1d47e0f95686d04a7 Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Tue, 19 Nov 2024 14:10:59 -0500 Subject: [PATCH 13/16] fix(client): update headers --- src/client.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/client.rs b/src/client.rs index 978214e..f13e864 100644 --- a/src/client.rs +++ b/src/client.rs @@ -229,6 +229,12 @@ fn request(method: &'static Method, path: String, redirect: bool, quarantine: bo ) }; + let (key, value) = match fastrand::u32(0..3) { + 0 => ("X-Reddit-Width", fastrand::u32(300..500).to_string()), + 1 => ("X-Reddit-DPR", "2".to_owned()), + _ => ("Device-Name", format!("Android {}", fastrand::u8(9..=14))), + }; + // Build request to Reddit. When making a GET, request gzip compression. // (Reddit doesn't do brotli yet.) let builder = Request::builder() @@ -241,6 +247,7 @@ fn request(method: &'static Method, path: String, redirect: bool, quarantine: bo .header("Host", host) .header("Authorization", &format!("Bearer {token}")) .header("Accept-Encoding", if method == Method::GET { "gzip" } else { "identity" }) + .header(key, value) .header( "Cookie", if quarantine { From 6ecdedd2ede7fa72a6651bf51f1d535c9e64ec93 Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Tue, 19 Nov 2024 14:54:06 -0500 Subject: [PATCH 14/16] feat(client): additionally randomize headers --- src/client.rs | 49 +++++++++++++++++++++++++++---------------------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/src/client.rs b/src/client.rs index f13e864..98c42fb 100644 --- a/src/client.rs +++ b/src/client.rs @@ -229,34 +229,39 @@ fn request(method: &'static Method, path: String, redirect: bool, quarantine: bo ) }; - let (key, value) = match fastrand::u32(0..3) { - 0 => ("X-Reddit-Width", fastrand::u32(300..500).to_string()), - 1 => ("X-Reddit-DPR", "2".to_owned()), - _ => ("Device-Name", format!("Android {}", fastrand::u8(9..=14))), - }; - // Build request to Reddit. When making a GET, request gzip compression. // (Reddit doesn't do brotli yet.) - let builder = Request::builder() - .method(method) - .uri(&url) - .header("User-Agent", user_agent) - .header("Client-Vendor-Id", vendor_id) - .header("X-Reddit-Device-Id", device_id) - .header("x-reddit-loid", loid) - .header("Host", host) - .header("Authorization", &format!("Bearer {token}")) - .header("Accept-Encoding", if method == Method::GET { "gzip" } else { "identity" }) - .header(key, value) - .header( + let mut headers = vec![ + ("User-Agent", user_agent), + ("Client-Vendor-Id", vendor_id), + ("X-Reddit-Device-Id", device_id), + ("x-reddit-loid", loid), + ("Host", host.to_string()), + ("Authorization", format!("Bearer {token}")), + ("Accept-Encoding", if method == Method::GET { "gzip".into() } else { "identity".into() }), + ( "Cookie", if quarantine { - "_options=%7B%22pref_quarantine_optin%22%3A%20true%2C%20%22pref_gated_sr_optin%22%3A%20true%7D" + "_options=%7B%22pref_quarantine_optin%22%3A%20true%2C%20%22pref_gated_sr_optin%22%3A%20true%7D".into() } else { - "" + "".into() }, - ) - .body(Body::empty()); + ), + ("X-Reddit-Width", fastrand::u32(300..500).to_string()), + ("X-Reddit-DPR", "2".to_owned()), + ("Device-Name", format!("Android {}", fastrand::u8(9..=14))), + ]; + + // shuffle headers: https://github.com/redlib-org/redlib/issues/324 + fastrand::shuffle(&mut headers); + + let mut builder = Request::builder().method(method).uri(&url); + + for (key, value) in headers { + builder = builder.header(key, value); + } + + let builder = builder.body(Body::empty()); async move { match builder { From cb9a2a3c391f2c148521770aa70556fa5a500623 Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Tue, 19 Nov 2024 15:48:42 -0500 Subject: [PATCH 15/16] fix(client): revert to hyper_rustls :P hi SWE :wave: --- .github/workflows/build-artifacts.yaml | 8 +- .github/workflows/main-rust.yml | 4 +- Cargo.lock | 205 ++++++++++++------------- Cargo.toml | 2 +- src/client.rs | 8 +- 5 files changed, 112 insertions(+), 115 deletions(-) diff --git a/.github/workflows/build-artifacts.yaml b/.github/workflows/build-artifacts.yaml index 9dd94c9..695d1bf 100644 --- a/.github/workflows/build-artifacts.yaml +++ b/.github/workflows/build-artifacts.yaml @@ -38,24 +38,24 @@ jobs: - if: matrix.target == 'x86_64-unknown-linux-musl' run: | sudo apt-get update - sudo apt-get install -y --no-install-recommends musl-tools libssl-dev + sudo apt-get install -y --no-install-recommends musl-tools - if: matrix.target == 'armv7-unknown-linux-musleabihf' run: | sudo apt update - sudo apt install -y gcc-arm-linux-gnueabihf musl-tools libssl-dev + sudo apt install -y gcc-arm-linux-gnueabihf musl-tools - if: matrix.target == 'aarch64-unknown-linux-musl' run: | sudo apt update - sudo apt install -y gcc-aarch64-linux-gnu musl-tools libssl-dev + sudo apt install -y gcc-aarch64-linux-gnu musl-tools - name: Versions id: version run: echo "VERSION=$(cargo metadata --format-version 1 --no-deps | jq .packages[0].version -r | sed 's/^/v/')" >> "$GITHUB_OUTPUT" - name: Build - run: RUSTFLAGS='-C target-feature=+crt-static' cargo build --release --target ${{ matrix.target }} + run: cargo build --release --target ${{ matrix.target }} - name: Package release run: tar czf redlib-${{ matrix.target }}.tar.gz -C target/${{ matrix.target }}/release/ redlib diff --git a/.github/workflows/main-rust.yml b/.github/workflows/main-rust.yml index 64f30b3..f38c01d 100644 --- a/.github/workflows/main-rust.yml +++ b/.github/workflows/main-rust.yml @@ -30,8 +30,8 @@ jobs: with: toolchain: stable - - name: Install musl-gcc and libssl-dev - run: sudo apt-get install musl-tools libssl-dev + - name: Install musl-gcc + run: sudo apt-get install musl-tools - name: Install cargo musl target run: rustup target add x86_64-unknown-linux-musl diff --git a/Cargo.lock b/Cargo.lock index 846d8dd..4143b80 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -139,6 +139,12 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + [[package]] name = "base64" version = "0.22.1" @@ -576,21 +582,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - [[package]] name = "form_urlencoded" version = "1.2.1" @@ -844,16 +835,19 @@ dependencies = [ ] [[package]] -name = "hyper-tls" -version = "0.5.0" +name = "hyper-rustls" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ - "bytes", + "futures-util", + "http", "hyper", - "native-tls", + "log", + "rustls", + "rustls-native-certs", "tokio", - "tokio-native-tls", + "tokio-rustls", ] [[package]] @@ -1160,23 +1154,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "native-tls" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" -dependencies = [ - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - [[package]] name = "never" version = "0.1.0" @@ -1232,60 +1209,12 @@ version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" -[[package]] -name = "openssl" -version = "0.10.68" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" -dependencies = [ - "bitflags", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "openssl-probe" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" -[[package]] -name = "openssl-src" -version = "300.4.1+3.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faa4eac4138c62414b5622d1b31c5c304f34b406b013c079c2bbc652fdd6678c" -dependencies = [ - "cc", -] - -[[package]] -name = "openssl-sys" -version = "0.9.104" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45abf306cbf99debc8195b66b7346498d7b10c210de50418b5ccd7ceba08c741" -dependencies = [ - "cc", - "libc", - "openssl-src", - "pkg-config", - "vcpkg", -] - [[package]] name = "parking" version = "2.2.1" @@ -1333,12 +1262,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" -[[package]] -name = "pkg-config" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" - [[package]] name = "powerfmt" version = "0.2.0" @@ -1429,7 +1352,7 @@ version = "0.35.1" dependencies = [ "arc-swap", "async-recursion", - "base64", + "base64 0.22.1", "brotli", "build_html", "cached", @@ -1440,7 +1363,7 @@ dependencies = [ "fastrand", "futures-lite", "hyper", - "hyper-tls", + "hyper-rustls", "libflate", "lipsum", "log", @@ -1502,6 +1425,21 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +[[package]] +name = "ring" +version = "0.17.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +dependencies = [ + "cc", + "cfg-if", + "getrandom", + "libc", + "spin", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "rinja" version = "0.3.5" @@ -1622,6 +1560,49 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustls" +version = "0.21.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +dependencies = [ + "log", + "ring", + "rustls-webpki", + "sct", +] + +[[package]] +name = "rustls-native-certs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +dependencies = [ + "openssl-probe", + "rustls-pemfile", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.7", +] + +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "rusty-forkfork" version = "0.4.0" @@ -1664,6 +1645,16 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "sealed_test" version = "1.1.0" @@ -1864,6 +1855,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -2013,12 +2010,12 @@ dependencies = [ ] [[package]] -name = "tokio-native-tls" -version = "0.3.1" +name = "tokio-rustls" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "native-tls", + "rustls", "tokio", ] @@ -2124,6 +2121,12 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.5.3" @@ -2156,12 +2159,6 @@ dependencies = [ "getrandom", ] -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - [[package]] name = "version_check" version = "0.9.5" diff --git a/Cargo.toml b/Cargo.toml index 77aec78..7bb7e93 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,7 +48,7 @@ arc-swap = "1.7.1" serde_json_path = "0.7.1" async-recursion = "1.1.1" common-words-all = { version = "0.0.2", default-features = false, features = ["english", "one"] } -hyper-tls = { version = "=0.5.0", features = ["vendored"] } +hyper-rustls = { version = "0.24.2", features = [ "http2" ] } [dev-dependencies] diff --git a/src/client.rs b/src/client.rs index 98c42fb..248fc88 100644 --- a/src/client.rs +++ b/src/client.rs @@ -4,9 +4,8 @@ use futures_lite::future::block_on; use futures_lite::{future::Boxed, FutureExt}; use hyper::client::HttpConnector; use hyper::header::HeaderValue; -use hyper::Client; -use hyper::{body, body::Buf, header, Body, Method, Request, Response, Uri}; -use hyper_tls::HttpsConnector; +use hyper::{body, body::Buf, header, Body, Client, Method, Request, Response, Uri}; +use hyper_rustls::HttpsConnector; use libflate::gzip; use log::{error, trace, warn}; use once_cell::sync::Lazy; @@ -31,7 +30,8 @@ const REDDIT_SHORT_URL_BASE_HOST: &str = "redd.it"; const ALTERNATIVE_REDDIT_URL_BASE: &str = "https://www.reddit.com"; const ALTERNATIVE_REDDIT_URL_BASE_HOST: &str = "www.reddit.com"; -pub static HTTPS_CONNECTOR: Lazy> = Lazy::new(HttpsConnector::new); +pub static HTTPS_CONNECTOR: Lazy> = + Lazy::new(|| hyper_rustls::HttpsConnectorBuilder::new().with_native_roots().https_only().enable_http2().build()); pub static CLIENT: Lazy>> = Lazy::new(|| Client::builder().build::<_, Body>(HTTPS_CONNECTOR.clone())); From d3ba5f3efb6825f4b0454523dae472b2adda3066 Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Tue, 19 Nov 2024 16:30:37 -0500 Subject: [PATCH 16/16] feat(error): add new instance buttom --- src/main.rs | 20 ++++++++++++++++++++ static/check_update.js | 18 ++++++++++++++++++ templates/error.html | 2 ++ 3 files changed, 40 insertions(+) diff --git a/src/main.rs b/src/main.rs index 4923921..7342597 100644 --- a/src/main.rs +++ b/src/main.rs @@ -223,6 +223,7 @@ async fn main() { .get(|_| resource(include_str!("../static/check_update.js"), "text/javascript", false).boxed()); app.at("/commits.atom").get(|_| async move { proxy_commit_info().await }.boxed()); + app.at("/instances.json").get(|_| async move { proxy_instances().await }.boxed()); // Proxy media through Redlib app.at("/vid/:id/:size").get(|r| proxy(r, "https://v.redd.it/{id}/DASH_{size}").boxed()); @@ -399,3 +400,22 @@ async fn fetch_commit_info() -> String { hyper::body::to_bytes(resp).await.expect("Failed to read body").iter().copied().map(|x| x as char).collect() } + +pub async fn proxy_instances() -> Result, String> { + Ok( + Response::builder() + .status(200) + .header("content-type", "application/json") + .body(Body::from(fetch_instances().await)) + .unwrap_or_default(), + ) +} + +#[cached(time = 600)] +async fn fetch_instances() -> String { + let uri = Uri::from_str("https://raw.githubusercontent.com/redlib-org/redlib-instances/refs/heads/main/instances.json").expect("Invalid URI"); + + let resp: Body = CLIENT.get(uri).await.expect("Failed to request GitHub").into_body(); + + hyper::body::to_bytes(resp).await.expect("Failed to read body").iter().copied().map(|x| x as char).collect() +} diff --git a/static/check_update.js b/static/check_update.js index b68c508..b747203 100644 --- a/static/check_update.js +++ b/static/check_update.js @@ -37,4 +37,22 @@ async function checkInstanceUpdateStatus() { } } +async function checkOtherInstances() { + try { + const response = await fetch('/instances.json'); + const data = await response.json(); + const randomInstance = data.instances[Math.floor(Math.random() * data.instances.length)]; + const instanceUrl = randomInstance.url; + // Set the href of the tag to the instance URL with path included + document.getElementById('random-instance').href = instanceUrl + window.location.pathname; + document.getElementById('random-instance').innerText = "Visit Random Instance"; + } catch (error) { + console.error('Error fetching instances:', error); + document.getElementById('update-status').innerText = '⚠️ Error checking update status.'; + } +} + +// Set the target URL when the page loads +window.addEventListener('load', checkOtherInstances); + checkInstanceUpdateStatus(); diff --git a/templates/error.html b/templates/error.html index 8f2f44d..2a56f68 100644 --- a/templates/error.html +++ b/templates/error.html @@ -7,6 +7,8 @@

Reddit Status


+
+