From 89b30bf4c1cc9cea227ae23a87b23fff703e03d5 Mon Sep 17 00:00:00 2001
From: holly sparkles <sparkles@holly.sh>
Date: Fri, 16 Feb 2024 20:30:21 +0100
Subject: [PATCH] feat: add robots.txt support

---
 rgit.conf.example        |  2 ++
 src/configuration/mod.rs |  8 ++++++++
 src/main.rs              | 58 +++++++++++++++++++++++++++++++++++++++++++++++-----------
 statics/robots.txt       |  2 ++
 4 files changed, 59 insertions(+), 11 deletions(-)
 create mode 100644 statics/robots.txt

diff --git a/rgit.conf.example b/rgit.conf.example
index 3ea3274..e436bac 100644
--- a/rgit.conf.example
+++ b/rgit.conf.example
@@ -34,5 +34,7 @@ max_commit_message_length = 80
 max_repo_desc_length = 80
 # Use custom css on all pages
 css = "/custom.scss"
+# Set a custom file to serve at `/robots.txt`
+robots_txt = "/robots.txt"
 # Change the header style
 header = "Image"
\ No newline at end of file
diff --git a/src/configuration/mod.rs b/src/configuration/mod.rs
index 7305fb2..ee78100 100644
--- a/src/configuration/mod.rs
+++ b/src/configuration/mod.rs
@@ -98,6 +98,13 @@ pub struct AppConfig {
     /// Default value: `None`
     pub css: Option<String>,
 
+    /// Specifies an absolute path on the filesystem to the file to serve at `/robots.txt`.
+    ///
+    /// If set, the default `robots.txt` will be replaced with this one.
+    ///
+    /// Default value: `None`
+    pub robots_txt: Option<String>,
+
     /// Specifies the style of header to use.
     ///
     /// `Default` is the default style of an emoji and title text
@@ -133,6 +140,7 @@ impl ::std::default::Default for AppConfig {
             max_commit_message_length: 0,
             max_repo_desc_length: 0,
             css: None,
+            robots_txt: None,
             header: Some(HeaderStyle::default()),
         }
     }
diff --git a/src/main.rs b/src/main.rs
index 59da25a..681e541 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -62,6 +62,8 @@ static GLOBAL_CSS_HASH: Lazy<Box<str>> = Lazy::new(|| build_asset_hash(GLOBAL_CS
 static HIGHLIGHT_CSS_HASH: OnceCell<Box<str>> = OnceCell::new();
 static DARK_HIGHLIGHT_CSS_HASH: OnceCell<Box<str>> = OnceCell::new();
 
+static GLOBAL_ROBOTS_TXT: &[u8] = include_bytes!("../statics/robots.txt");
+
 #[derive(Parser, Debug)]
 #[clap(author, version, about)]
 pub struct Args {
@@ -201,28 +203,30 @@ async fn main() -> Result<(), anyhow::Error> {
         }
     };
 
-    let static_file = |file: String| {
+    let static_file = |file: Option<String>, fallback_data: Option<&'static [u8]>| {
         move || async move {
-            let mime_type = mime_guess::from_path(&file)
+            let mime_type = mime_guess::from_path(&file.clone().unwrap_or_default())
                 .first()
                 .map_or("text/plain".to_string(), |mime| {
                     mime.essence_str().to_string()
                 });
 
-            let content = std::fs::read(&PathBuf::from(&file)).unwrap_or_else(|_| {
-                if file.is_empty() {
-                    warn!("Requested filename is blank.");
-                }
-                error!("Unable to load file: {}", file);
-                Vec::new()
-            });
+            let mut content = Vec::new();
+            match file.clone() {
+                Some(path) => content = read_file_contents(path.clone(), fallback_data).await,
+                None => (),
+            };
 
             //TODO: create a 410 (gone) response here when content is empty or file is invalid.
             let mut resp = Response::new(Body::from(content));
             resp.headers_mut().insert(
                 http::header::CONTENT_TYPE,
                 HeaderValue::from_str(&mime_type).unwrap_or_else(|_| {
-                    warn!("Invalid MIME type: {} for file: {}", mime_type, file);
+                    warn!(
+                        "Invalid MIME type: {} for file: {}",
+                        mime_type,
+                        file.unwrap_or_default()
+                    );
                     HeaderValue::from_static("text/plain")
                 }),
             );
@@ -257,7 +261,14 @@ async fn main() -> Result<(), anyhow::Error> {
         .route("/about", get(methods::about::handle))
         .route(
             &format!("/{}", get_logo_filename(config.logo.clone())),
-            get(static_file(config.logo.clone())),
+            get(static_file(Some(config.logo.clone()), None)),
+        )
+        .route(
+            "/robots.txt",
+            get(static_file(
+                Some(config.robots_txt.clone().unwrap_or_default()),
+                Some(GLOBAL_ROBOTS_TXT),
+            )),
         )
         .fallback(methods::repo::service)
         .layer(TimeoutLayer::new(args.request_timeout.into()))
@@ -434,3 +445,28 @@ pub fn get_logo_filename(filename: String) -> String {
         .to_string();
     format!("logo.{}", extension)
 }
+
+async fn read_file_contents(file: String, fallback_data: Option<&'static [u8]>) -> Vec<u8> {
+    let content = tokio::fs::read(&PathBuf::from(&file))
+        .await
+        .unwrap_or_else(|_| {
+            let data = match (file.is_empty(), fallback_data) {
+                (true, Some(fallback)) => {
+                    // file is empty, there is fallback data
+                    fallback.into()
+                }
+                (true, None) => {
+                    // file is empty, there is no fallback data
+                    warn!("Fallback data missing.");
+                    Vec::new()
+                }
+                (false, _) => {
+                    // return the data
+                    file.into()
+                }
+            };
+
+            data
+        });
+    content
+}
diff --git a/statics/robots.txt b/statics/robots.txt
new file mode 100644
index 0000000..c2a49f4
--- /dev/null
+++ b/statics/robots.txt
@@ -0,0 +1,2 @@
+User-agent: *
+Allow: /
--
libgit2 1.7.2