Add option to disable all indexing.

README.md

@@ -213,6 +213,7 @@ Assign a default value for each instance-specific setting by passing environment
 |-|-|-|-|
 | `SFW_ONLY` | `["on", "off"]` | `off`   | Enables SFW-only mode for the instance, i.e. all NSFW content is filtered. |
 | `BANNER`   | String          | (empty) | Allows the server to set a banner to be displayed. Currently this is displayed on the instance info page. | 
+| `ROBOTS_DISABLE_INDEXING` | `["on", "off"]` | `off` | Disables indexing of the instance by search engines. |
 
 ## Default User Settings
 

app.json

@@ -50,6 +50,9 @@
     "LIBREDDIT_BANNER": {
       "required": false
     },
+    "LIBREDDIT_ROBOTS_DISABLE_INDEXING":  {
+        "required": false
+    },
     "LIBREDDIT_DEFAULT_SUBSCRIPTIONS": {
       "required": false
     },

src/config.rs

@@ -60,6 +60,9 @@ pub struct Config {
 
 	#[serde(rename = "LIBREDDIT_BANNER")]
 	pub(crate) banner: Option<String>,
+
+	#[serde(rename = "LIBREDDIT_ROBOTS_DISABLE_INDEXING")]
+	pub(crate) robots_disable_indexing: Option<String>,
 }
 
 impl Config {
@@ -90,6 +93,7 @@ impl Config {
 			default_subscriptions: parse("LIBREDDIT_DEFAULT_SUBSCRIPTIONS"),
 			default_disable_visit_reddit_confirmation: parse("LIBREDDIT_DEFAULT_DISABLE_VISIT_REDDIT_CONFIRMATION"),
 			banner: parse("LIBREDDIT_BANNER"),
+			robots_disable_indexing: parse("LIBREDDIT_ROBOTS_DISABLE_INDEXING"),
 		}
 	}
 }
@@ -111,6 +115,7 @@ fn get_setting_from_config(name: &str, config: &Config) -> Option<String> {
 		"LIBREDDIT_DEFAULT_SUBSCRIPTIONS" => config.default_subscriptions.clone(),
 		"LIBREDDIT_DEFAULT_DISABLE_VISIT_REDDIT_CONFIRMATION" => config.default_disable_visit_reddit_confirmation.clone(),
 		"LIBREDDIT_BANNER" => config.banner.clone(),
+		"LIBREDDIT_ROBOTS_DISABLE_INDEXING" => config.robots_disable_indexing.clone(),
 		_ => None,
 	}
 }

src/main.rs

@@ -186,9 +186,21 @@ async fn main() {
 	app
 		.at("/manifest.json")
 		.get(|_| resource(include_str!("../static/manifest.json"), "application/json", false).boxed());
-	app
+	app.at("/robots.txt").get(|_| {
-		.at("/robots.txt")
+		resource(
-		.get(|_| resource("User-agent: *\nDisallow: /u/\nDisallow: /user/", "text/plain", true).boxed());
+			if match config::get_setting("LIBREDDIT_ROBOTS_DISABLE_INDEXING") {
+				Some(val) => val == "on",
+				None => false,
+			} {
+				"User-agent: *\nDisallow: /"
+			} else {
+				"User-agent: *\nDisallow: /u/\nDisallow: /user/"
+			},
+			"text/plain",
+			true,
+		)
+		.boxed()
+	});
 	app.at("/favicon.ico").get(|_| favicon().boxed());
 	app.at("/logo.png").get(|_| pwa_logo().boxed());
 	app.at("/Inter.var.woff2").get(|_| font().boxed());