change bin name

port update checker to gitea compatible version
"fix" scraper
2024-11-03 10:36:46 +13:00 · 2024-11-03 10:08:26 +13:00 · 2024-11-03 09:36:11 +13:00 · 2024-11-03 09:34:12 +13:00 · 2024-11-01 18:16:25 -04:00 · 2024-11-01 12:28:52 -04:00
21 changed files with 544 additions and 487 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
--- a/Cargo.toml
+++ b/Cargo.toml
@ -9,6 +9,7 @@ authors = [
    "spikecodes <19519553+spikecodes@users.noreply.github.com>",
 ]
 edition = "2021"
+default-run = "redsunlib"

 [dependencies]
 rinja = { version = "0.3.4", default-features = false }
@ -16,13 +17,14 @@ cached = { version = "0.51.3", features = ["async"] }
 clap = { version = "4.4.11", default-features = false, features = [
    "std",
    "env",
+    "derive",
 ] }
 regex = "1.10.2"
 serde = { version = "1.0.193", features = ["derive"] }
 cookie = "0.18.0"
 futures-lite = "2.2.0"
 hyper = { version = "0.14.28", features = ["full"] }
-hyper-rustls = "0.24.2"
+hyper-rustls = { version = "0.24.2", features = [ "http2" ] }
 percent-encoding = "2.3.1"
 route-recognizer = "0.3.1"
 serde_json = "1.0.108"
@ -56,3 +58,11 @@ sealed_test = "1.0.0"
 codegen-units = 1
 lto = true
 strip = "symbols"
+
+[[bin]]
+name = "redsunlib"
+path = "src/main.rs"
+
+[[bin]]
+name = "scraper"
+path = "src/scraper/main.rs"
--- a/README.md
+++ b/README.md
@ -64,8 +64,7 @@ Redlib currently implements most of Reddit's (signed-out) functionalities but st

 **Red sun** in the sky + Red**lib**  = Redsunlib

-And at the time, I was reading an excerpt from Mao Zedong, so the name seemed appropriate. But paradoxically named since Reddit is basically the sinophobia capital of the internet :/
-
+<sup>I do self criticism constantly, because I'm trapped in a Maoist *cult* where comrades (white terrorists) criticize me merciloussly for having a fascist credit card (VISA Silver Signature Rewards) They won't let me order vegan pizza anymore because the phone is fascist and "summoning my pizza slave with bourgeois app" is "bad vibes"</sup>

 ## Built with

--- a/src/client.rs
+++ b/src/client.rs
@ -4,7 +4,7 @@ use futures_lite::future::block_on;
 use futures_lite::{future::Boxed, FutureExt};
 use hyper::client::HttpConnector;
 use hyper::header::HeaderValue;
-use hyper::{body, body::Buf, client, header, Body, Client, Method, Request, Response, Uri};
+use hyper::{body, body::Buf, header, Body, Client, Method, Request, Response, Uri};
 use hyper_rustls::HttpsConnector;
 use libflate::gzip;
 use log::{error, trace, warn};
@ -30,10 +30,10 @@ const REDDIT_SHORT_URL_BASE_HOST: &str = "redd.it";
 const ALTERNATIVE_REDDIT_URL_BASE: &str = "https://www.reddit.com";
 const ALTERNATIVE_REDDIT_URL_BASE_HOST: &str = "www.reddit.com";

-pub static CLIENT: Lazy<Client<HttpsConnector<HttpConnector>>> = Lazy::new(|| {
-	let https = hyper_rustls::HttpsConnectorBuilder::new().with_native_roots().https_only().enable_http1().build();
-	client::Client::builder().build(https)
-});
+pub static HTTPS_CONNECTOR: Lazy<HttpsConnector<HttpConnector>> =
+	Lazy::new(|| hyper_rustls::HttpsConnectorBuilder::new().with_native_roots().https_only().enable_http2().build());
+
+pub static CLIENT: Lazy<Client<HttpsConnector<HttpConnector>>> = Lazy::new(|| Client::builder().build::<_, Body>(HTTPS_CONNECTOR.clone()));

 pub static OAUTH_CLIENT: Lazy<ArcSwap<Oauth>> = Lazy::new(|| {
 	let client = block_on(Oauth::new());
@ -154,7 +154,7 @@ async fn stream(url: &str, req: &Request<Body>) -> Result<Response<Body>, String
 	let parsed_uri = url.parse::<Uri>().map_err(|_| "Couldn't parse URL".to_string())?;

 	// Build the hyper client from the HTTPS connector.
-	let client: Client<_, Body> = CLIENT.clone();
+	let client: &Lazy<Client<_, Body>> = &CLIENT;

 	let mut builder = Request::get(parsed_uri);

@ -216,7 +216,7 @@ fn request(method: &'static Method, path: String, redirect: bool, quarantine: bo
 	let url = format!("{base_path}{path}");

 	// Construct the hyper client from the HTTPS connector.
-	let client: Client<_, Body> = CLIENT.clone();
+	let client: &Lazy<Client<_, Body>> = &CLIENT;

 	let (token, vendor_id, device_id, user_agent, loid) = {
 		let client = OAUTH_CLIENT.load_full();
--- a/src/instance_info.rs
+++ b/src/instance_info.rs
@ -85,7 +85,7 @@ fn info_html(req: &Request<Body>) -> Result<Response<Body>, Error> {
 pub struct InstanceInfo {
 	package_name: String,
 	crate_version: String,
-	git_commit: String,
+	pub git_commit: String,
 	deploy_date: String,
 	compile_mode: String,
 	deploy_unix_ts: i64,
--- a/src/lib.rs
+++ b/src/lib.rs
@ -0,0 +1,13 @@
+pub mod client;
+pub mod config;
+pub mod duplicates;
+pub mod instance_info;
+pub mod oauth;
+pub mod oauth_resources;
+pub mod post;
+pub mod search;
+pub mod server;
+pub mod settings;
+pub mod subreddit;
+pub mod user;
+pub mod utils;
--- a/src/main.rs
+++ b/src/main.rs
@ -2,35 +2,21 @@
 #![forbid(unsafe_code)]
 #![allow(clippy::cmp_owned)]

-// Reference local files
-mod config;
-mod duplicates;
-mod instance_info;
-mod oauth;
-mod oauth_resources;
-mod post;
-mod search;
-mod settings;
-mod subreddit;
-mod user;
-mod utils;
-
-// Import Crates
+use cached::proc_macro::cached;
 use clap::{Arg, ArgAction, Command};
+use std::str::FromStr;

 use futures_lite::FutureExt;
+use hyper::Uri;
 use hyper::{header::HeaderValue, Body, Request, Response};
-
-mod client;
-use client::{canonical_path, proxy};
 use log::info;
 use once_cell::sync::Lazy;
-use server::RequestExt;
-use utils::{error, redirect, ThemeAssets, MascotAssets};
+use redsunlib::client::{canonical_path, proxy, CLIENT};
+use redsunlib::server::{self, RequestExt};
+use redsunlib::utils::{error, redirect, ThemeAssets, MascotAssets};
+use redsunlib::{config, duplicates, headers, instance_info, post, search, settings, subreddit, user};

-use crate::client::OAUTH_CLIENT;
-
-mod server;
+use redsunlib::client::OAUTH_CLIENT;

 // Create Services

@ -257,6 +243,12 @@ async fn main() {
 	app
 		.at("/highlighted.js")
 		.get(|_| resource(include_str!("../static/highlighted.js"), "text/javascript", false).boxed());
+	app
+		.at("/check_update.js")
+		.get(|_| resource(include_str!("../static/check_update.js"), "text/javascript", false).boxed());
+
+	app.at("/commits.json").get(|_| async move { proxy_commit_info().await }.boxed());
+	
 	// FFmpeg
 	app
 		.at("/ffmpeg/814.ffmpeg.js")
@ -437,3 +429,22 @@ async fn main() {
 		eprintln!("Server error: {e}");
 	}
 }
+
+pub async fn proxy_commit_info() -> Result<Response<Body>, String> {
+	Ok(
+		Response::builder()
+			.status(200)
+			.header("content-type", "application/atom+xml")
+			.body(Body::from(fetch_commit_info().await))
+			.unwrap_or_default(),
+	)
+}
+
+#[cached(time = 600)]
+async fn fetch_commit_info() -> String {
+	let uri = Uri::from_str("https://git.stardust.wtf/api/v1/repos/iridium/redsunlib/commits?verification=false&stat=false").expect("Invalid URI");
+
+	let resp: Body = CLIENT.get(uri).await.expect("Failed to request git.stardust.wtf").into_body();
+
+	hyper::body::to_bytes(resp).await.expect("Failed to read body").iter().copied().map(|x| x as char).collect()
+}
--- a/src/oauth.rs
+++ b/src/oauth.rs
@ -94,7 +94,7 @@ impl Oauth {
 		trace!("Sending token request...");

 		// Send request
-		let client: client::Client<_, Body> = CLIENT.clone();
+		let client: &once_cell::sync::Lazy<client::Client<_, Body>> = &CLIENT;
 		let resp = client.request(request).await.ok()?;

 		trace!("Received response with status {} and length {:?}", resp.status(), resp.headers().get("content-length"));
--- a/src/post.rs
+++ b/src/post.rs
@ -1,3 +1,5 @@
+#![allow(clippy::cmp_owned)]
+
 // CRATES
 use crate::client::json;
 use crate::config::get_setting;
--- a/src/scraper/main.rs
+++ b/src/scraper/main.rs
@ -0,0 +1,75 @@
+use std::{fmt::Display, io::Write};
+
+use clap::{Parser, ValueEnum};
+use redsunlib::utils::Post;
+
+#[derive(Parser)]
+#[command(name = "my_cli")]
+#[command(about = "A simple CLI example", long_about = None)]
+struct Cli {
+	#[arg(short = 's', long = "sub")]
+	sub: String,
+
+	#[arg(short = 'c', long = "count")]
+	count: usize,
+
+	#[arg(long = "sort")]
+	sort: SortOrder,
+
+	#[arg(short = 'f', long = "format", value_enum)]
+	format: Format,
+	#[arg(short = 'o', long = "output")]
+	output: Option<String>,
+}
+
+#[derive(Debug, Clone, ValueEnum)]
+enum SortOrder {
+	Hot,
+	Rising,
+	New,
+	Top,
+	Controversial,
+}
+
+impl Display for SortOrder {
+	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+		match self {
+			SortOrder::Hot => write!(f, "hot"),
+			SortOrder::Rising => write!(f, "rising"),
+			SortOrder::New => write!(f, "new"),
+			SortOrder::Top => write!(f, "top"),
+			SortOrder::Controversial => write!(f, "controversial"),
+		}
+	}
+}
+
+#[derive(Debug, Clone, ValueEnum)]
+enum Format {
+	Json,
+}
+
+#[tokio::main]
+async fn main() {
+	let cli = Cli::parse();
+	let (sub, final_count, sort, format, output) = (cli.sub, cli.count, cli.sort, cli.format, cli.output);
+	let initial = format!("/r/{sub}/{sort}.json?&raw_json=1");
+	let (mut posts, mut after) = Post::fetch(&initial, false).await.unwrap();
+	while posts.len() < final_count {
+		print!("\r");
+		let path = format!("/r/{sub}/{sort}.json?sort={sort}&t=&after={after}&raw_json=1");
+		let (new_posts, new_after) = Post::fetch(&path, false).await.unwrap();
+		posts.extend(new_posts);
+		after = new_after;
+		// Print number of posts fetched
+		print!("Fetched {} posts", posts.len());
+		std::io::stdout().flush().unwrap();
+	}
+
+	match format {
+		Format::Json => {
+			let filename: String = output.unwrap_or_else(|| format!("{sub}.json"));
+			let json = serde_json::to_string(&posts).unwrap();
+			std::fs::write(filename, json).unwrap();
+		}
+	}
+}
--- a/src/search.rs
+++ b/src/search.rs
@ -1,9 +1,11 @@
+#![allow(clippy::cmp_owned)]
+
 // CRATES
 use crate::utils::{self, catch_random, error, filter_posts, format_num, format_url, get_filters, param, redirect, setting, template, val, Post, Preferences};
 use crate::{
 	client::json,
+	server::RequestExt,
 	subreddit::{can_access_quarantine, quarantine},
-	RequestExt,
 };
 use hyper::{Body, Request, Response};
 use once_cell::sync::Lazy;
--- a/src/server.rs
+++ b/src/server.rs
@ -1,4 +1,5 @@
 #![allow(dead_code)]
+#![allow(clippy::cmp_owned)]

 use brotli::enc::{BrotliCompress, BrotliEncoderParams};
 use cached::proc_macro::cached;
@ -195,6 +196,12 @@ impl Route<'_> {
 	}
 }

+impl Default for Server {
+	fn default() -> Self {
+		Self::new()
+	}
+}
+
 impl Server {
 	pub fn new() -> Self {
 		Self {
@ -723,7 +730,7 @@ mod tests {

 				CompressionType::Brotli => Box::new(BrotliDecompressor::new(body_cursor, expected_lorem_ipsum.len())),

-				_ => panic!("no decompressor for {}", expected_encoding.to_string()),
+				_ => panic!("no decompressor for {}", expected_encoding),
 			};

 			let mut decompressed = Vec::<u8>::new();
--- a/src/settings.rs
+++ b/src/settings.rs
@ -1,3 +1,5 @@
+#![allow(clippy::cmp_owned)]
+
 use std::collections::HashMap;

 // CRATES
@ -19,7 +21,7 @@ struct SettingsTemplate {

 // CONSTANTS

-const PREFS: [&str; 19] = [
+const PREFS: [&str; 20] = [
 	"theme",
 	"mascot",
 	"front_page",
@ -39,6 +41,7 @@ const PREFS: [&str; 19] = [
 	"hide_awards",
 	"hide_score",
 	"disable_visit_reddit_confirmation",
+	"video_quality",
 ];

 // FUNCTIONS
--- a/src/subreddit.rs
+++ b/src/subreddit.rs
@ -1,9 +1,11 @@
+#![allow(clippy::cmp_owned)]
+
 use crate::{config, utils};
 // CRATES
 use crate::utils::{
 	catch_random, error, filter_posts, format_num, format_url, get_filters, nsfw_landing, param, redirect, rewrite_urls, setting, template, val, Post, Preferences, Subreddit,
 };
-use crate::{client::json, server::ResponseExt, RequestExt};
+use crate::{client::json, server::RequestExt, server::ResponseExt};
 use cookie::Cookie;
 use hyper::{Body, Request, Response};
 use rinja::Template;
--- a/src/user.rs
+++ b/src/user.rs
@ -1,3 +1,5 @@
+#![allow(clippy::cmp_owned)]
+
 // CRATES
 use crate::client::json;
 use crate::server::RequestExt;
--- a/src/utils.rs
+++ b/src/utils.rs
@ -1,4 +1,6 @@
 #![allow(dead_code)]
+#![allow(clippy::cmp_owned)]
+
 use crate::config::{self, get_setting};
 //
 // CRATES
@ -11,6 +13,7 @@ use once_cell::sync::Lazy;
 use regex::Regex;
 use rinja::Template;
 use rust_embed::RustEmbed;
+use serde::Serialize;
 use serde_json::Value;
 use serde_json_path::{JsonPath, JsonPathExt};
 use std::collections::{HashMap, HashSet};
@ -46,6 +49,7 @@ pub enum ResourceType {
 }

 // Post flair with content, background color and foreground color
+#[derive(Serialize)]
 pub struct Flair {
 	pub flair_parts: Vec<FlairPart>,
 	pub text: String,
@ -54,7 +58,7 @@ pub struct Flair {
 }

 // Part of flair, either emoji or text
-#[derive(Clone)]
+#[derive(Clone, Serialize)]
 pub struct FlairPart {
 	pub flair_part_type: String,
 	pub value: String,
@ -96,12 +100,14 @@ impl FlairPart {
 	}
 }

+#[derive(Serialize)]
 pub struct Author {
 	pub name: String,
 	pub flair: Flair,
 	pub distinguished: String,
 }

+#[derive(Serialize)]
 pub struct Poll {
 	pub poll_options: Vec<PollOption>,
 	pub voting_end_timestamp: (String, String),
@ -129,6 +135,7 @@ impl Poll {
 	}
 }

+#[derive(Serialize)]
 pub struct PollOption {
 	pub id: u64,
 	pub text: String,
@ -158,13 +165,14 @@ impl PollOption {
 }

 // Post flags with nsfw and stickied
+#[derive(Serialize)]
 pub struct Flags {
 	pub spoiler: bool,
 	pub nsfw: bool,
 	pub stickied: bool,
 }

-#[derive(Debug)]
+#[derive(Debug, Serialize)]
 pub struct Media {
 	pub url: String,
 	pub alt_url: String,
@ -264,6 +272,7 @@ impl Media {
 	}
 }

+#[derive(Serialize)]
 pub struct GalleryMedia {
 	pub url: String,
 	pub width: i64,
@ -304,6 +313,7 @@ impl GalleryMedia {
 }

 // Post containing content, metadata and media
+#[derive(Serialize)]
 pub struct Post {
 	pub id: String,
 	pub title: String,
@ -470,7 +480,7 @@ pub struct Comment {
 	pub prefs: Preferences,
 }

-#[derive(Default, Clone)]
+#[derive(Default, Clone, Serialize)]
 pub struct Award {
 	pub name: String,
 	pub icon_url: String,
@ -484,6 +494,7 @@ impl std::fmt::Display for Award {
 	}
 }

+#[derive(Serialize)]
 pub struct Awards(pub Vec<Award>);

 impl std::ops::Deref for Awards {
@ -603,6 +614,7 @@ pub struct Preferences {
 	pub show_nsfw: String,
 	pub blur_nsfw: String,
 	pub hide_hls_notification: String,
+	pub video_quality: String,
 	pub hide_sidebar_and_summary: String,
 	pub use_hls: String,
 	pub ffmpeg_video_downloads: String,
@ -659,6 +671,7 @@ impl Preferences {
 			use_hls: setting(req, "use_hls"),
 			ffmpeg_video_downloads: setting(req, "ffmpeg_video_downloads"),
 			hide_hls_notification: setting(req, "hide_hls_notification"),
+			video_quality: setting(req, "video_quality"),
 			autoplay_videos: setting(req, "autoplay_videos"),
 			fixed_navbar: setting_or_default(req, "fixed_navbar", "on".to_string()),
 			disable_visit_reddit_confirmation: setting(req, "disable_visit_reddit_confirmation"),
--- a/static/check_update.js
+++ b/static/check_update.js
@ -0,0 +1,35 @@
+async function checkInstanceUpdateStatus() {
+    try {
+        const response = await fetch('/commits.json');
+        const text = await response.text();
+        const entries = JSON.parse(text);
+        const localCommit = document.getElementById('git_commit').dataset.value;
+
+        let statusMessage = '';
+
+        if (entries.length > 0) {
+            const commitHashes = Array.from(entries).map(entry => {
+                return entry.sha
+            });
+
+            const commitIndex = commitHashes.indexOf(localCommit);
+
+            if (commitIndex === 0) {
+                statusMessage = '✅ Instance is up to date.';
+            } else if (commitIndex > 0) {
+                statusMessage = `⚠️ This instance is not up to date and is ${commitIndex} commits old. Test and confirm on an up-to-date instance before reporting.`;
+            } else {
+                statusMessage = `⚠️ This instance is not up to date and is at least ${commitHashes.length} commits old. Test and confirm on an up-to-date instance before reporting.`;
+            }
+        } else {
+            statusMessage = '⚠️ Unable to fetch commit information.';
+        }
+
+        document.getElementById('update-status').innerText = statusMessage;
+    } catch (error) {
+        console.error('Error fetching commits:', error);
+        document.getElementById('update-status').innerText = '⚠️ Error checking update status.';
+    }
+}
+
+checkInstanceUpdateStatus();
--- a/static/style.css
+++ b/static/style.css
@ -1965,6 +1965,13 @@ th {
    color: var(--accent);
 }

+#issue_warning {
+    color: var(--popup-toreddit-text);
+    background: var(--popup-background);
+    border-radius: 5px;
+    padding: 10px;
+}
+
 /* Messages */

 #duplicates_msg h3 {
--- a/templates/base.html
+++ b/templates/base.html
@ -27,6 +27,8 @@
 		<link rel="manifest" type="application/json" href="/manifest.json">
 		<link rel="shortcut icon" type="image/x-icon" href="/favicon.ico"> 
 		<link rel="stylesheet" type="text/css" href="/style.css?v={{ env!("CARGO_PKG_VERSION") }}">
+		<!-- Video quality -->
+		<div id="video_quality" data-value="{{ prefs.video_quality }}"></div>
 		{% endblock %}
 		</head>
 	<body class="
--- a/templates/error.html
+++ b/templates/error.html
@ -6,9 +6,14 @@
 	<h1>{{ msg }}</h1>
 	<h3><a href="https://www.redditstatus.com/">Reddit Status</a></h3>
 	<br />
-	<h3>Expected something to work? <a
-			href="https://github.com/redlib-org/redlib/issues/new?assignees=&labels=bug&projects=&template=bug_report.md&title=%F0%9F%90%9B+Bug+Report%3A+{{ msg }}">Report
-			an issue</a></h3>
+	<h3 id="update-status"></h3>
+	<br>
+	<div id="git_commit" data-value="{{ crate::instance_info::INSTANCE_INFO.git_commit }}"></div>
+	<script src="/check_update.js"></script>
+	
+	<h3>Expected something to work? Try a <a href="https://github.com/redlib-org/redlib/">upstream</a> instance.</h3>
+	<br />
+	<h3 id="issue_warning" >!! Do <b>NOT</b> open an issue on the redlib repository with a redsunlib specific issue !!</h3>
 	<br />
 	<h3>Head back <a href="/">home</a>?</h3>
 </div>
--- a/templates/settings.html
+++ b/templates/settings.html
@ -68,6 +68,12 @@
 			</fieldset>
 			<fieldset>
 				<legend>Content</legend>
+				<div class="prefs-group">
+					<label for="video_quality">Video quality:</label>
+					<select name="video_quality" id="video_quality"> 
+						{% call utils::options(prefs.video_quality, ["best", "medium", "worst"], "best") %}
+					</select>
+				</div>
 				<div class="prefs-group">
 					<label for="post_sort" title="Applies only to subreddit feeds">Default subreddit post sort:</label>
 					<select name="post_sort">
Author	SHA1	Message	Date
ayaka	feedc572cd	change bin name	2024-11-03 10:36:46 +13:00
ayaka	a3bc16f7d2	port update checker to gitea compatible version	2024-11-03 10:08:26 +13:00
ayaka	bd4cb96c0f	"fix" scraper	2024-11-03 09:36:11 +13:00
ayaka	a9c99cc752	Merge remote-tracking branch 'upstream/main'	2024-11-03 09:34:12 +13:00
Matthew Esposito	f03bdcf472	feat: display whether or not the instance is up to date on error (#310 )	2024-11-01 18:16:25 -04:00
Matthew Esposito	2fd358f3ed	feat(hls): add video quality preference (#306 )	2024-11-01 12:28:52 -04:00
Matthew Esposito	5ef57812f8	style: fix clippy	2024-11-01 11:39:05 -04:00
Nolan Poe	d17d097b12	Fix parts of CI (#304 ) * Run cargo fmt, hide clippy::cmp_owned errors * Bump deps * Fix failing test * Update src/client.rs --------- Co-authored-by: Matthew Esposito <matt@matthew.science>	2024-10-31 22:50:50 -04:00
Alex	a96894c743	enables http2 crate feature, replaces http1 protocol with http2 on co… (#305 )	2024-10-31 22:48:19 -04:00
Matthew Esposito	9aea9c90a2	fix: reduce to minimum patch, fix clippy	2024-10-31 16:09:35 -04:00
Matthew Esposito	efdf1848ac	fix: emergency patch for 403	2024-10-31 16:06:29 -04:00
Matthew Esposito	bc9530821d	feat(scraper): add output file	2024-10-30 15:15:38 -04:00
Matthew Esposito	f3d2f0cc59	feat(scraper): add scraper CLI	2024-10-21 20:54:05 -04:00
Matthew Esposito	49ef59e000	chore: make library	2024-10-21 20:46:03 -04:00
Ayaka	6773e3756b	Update README.md	2024-10-20 21:16:03 +13:00