index : sparkle-git.git

89b30bf4c1cc9cea227ae23a87b23fff703e03d5 [patch]

5f97b061ddfbc6d4bbbe388923e355e92183f65c

756b4dae0065ae88254b0fdb04fd518aef2f99b2

89b30bf4c1cc9cea227ae23a87b23fff703e03d5.tar.gz

feat: add robots.txt support

Diff

 rgit.conf.example        |  2 ++-
 src/configuration/mod.rs |  8 +++++++-
 src/main.rs              | 58 +++++++++++++++++++++++++++++++++++++++----------
 statics/robots.txt       |  2 ++-
 4 files changed, 59 insertions(+), 11 deletions(-)

diff --git a/rgit.conf.example b/rgit.conf.example
index 3ea3274..e436bac 100644
--- a/rgit.conf.example
+++ b/rgit.conf.example
@@ -34,5 +34,7 @@ max_commit_message_length = 80
max_repo_desc_length = 80
# Use custom css on all pages
css = "/custom.scss"
# Set a custom file to serve at `/robots.txt`
robots_txt = "/robots.txt"
# Change the header style
header = "Image"
\ No newline at end of file
diff --git a/src/configuration/mod.rs b/src/configuration/mod.rs
index 7305fb2..ee78100 100644
--- a/src/configuration/mod.rs
+++ b/src/configuration/mod.rs
@@ -98,6 +98,13 @@ pub struct AppConfig {
    /// Default value: `None`
    pub css: Option<String>,

    /// Specifies an absolute path on the filesystem to the file to serve at `/robots.txt`.
    ///
    /// If set, the default `robots.txt` will be replaced with this one.
    ///
    /// Default value: `None`
    pub robots_txt: Option<String>,

    /// Specifies the style of header to use.
    ///
    /// `Default` is the default style of an emoji and title text
@@ -133,6 +140,7 @@ impl ::std::default::Default for AppConfig {
            max_commit_message_length: 0,
            max_repo_desc_length: 0,
            css: None,
            robots_txt: None,
            header: Some(HeaderStyle::default()),
        }
    }
diff --git a/src/main.rs b/src/main.rs
index 59da25a..681e541 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -62,6 +62,8 @@ static GLOBAL_CSS_HASH: Lazy<Box<str>> = Lazy::new(|| build_asset_hash(GLOBAL_CS
static HIGHLIGHT_CSS_HASH: OnceCell<Box<str>> = OnceCell::new();
static DARK_HIGHLIGHT_CSS_HASH: OnceCell<Box<str>> = OnceCell::new();

static GLOBAL_ROBOTS_TXT: &[u8] = include_bytes!("../statics/robots.txt");

#[derive(Parser, Debug)]
#[clap(author, version, about)]
pub struct Args {
@@ -201,28 +203,30 @@ async fn main() -> Result<(), anyhow::Error> {
        }
    };

    let static_file = |file: String| {
    let static_file = |file: Option<String>, fallback_data: Option<&'static [u8]>| {
        move || async move {
            let mime_type = mime_guess::from_path(&file)
            let mime_type = mime_guess::from_path(&file.clone().unwrap_or_default())
                .first()
                .map_or("text/plain".to_string(), |mime| {
                    mime.essence_str().to_string()
                });

            let content = std::fs::read(&PathBuf::from(&file)).unwrap_or_else(|_| {
                if file.is_empty() {
                    warn!("Requested filename is blank.");
                }
                error!("Unable to load file: {}", file);
                Vec::new()
            });
            let mut content = Vec::new();
            match file.clone() {
                Some(path) => content = read_file_contents(path.clone(), fallback_data).await,
                None => (),
            };

            //TODO: create a 410 (gone) response here when content is empty or file is invalid.
            let mut resp = Response::new(Body::from(content));
            resp.headers_mut().insert(
                http::header::CONTENT_TYPE,
                HeaderValue::from_str(&mime_type).unwrap_or_else(|_| {
                    warn!("Invalid MIME type: {} for file: {}", mime_type, file);
                    warn!(
                        "Invalid MIME type: {} for file: {}",
                        mime_type,
                        file.unwrap_or_default()
                    );
                    HeaderValue::from_static("text/plain")
                }),
            );
@@ -257,7 +261,14 @@ async fn main() -> Result<(), anyhow::Error> {
        .route("/about", get(methods::about::handle))
        .route(
            &format!("/{}", get_logo_filename(config.logo.clone())),
            get(static_file(config.logo.clone())),
            get(static_file(Some(config.logo.clone()), None)),
        )
        .route(
            "/robots.txt",
            get(static_file(
                Some(config.robots_txt.clone().unwrap_or_default()),
                Some(GLOBAL_ROBOTS_TXT),
            )),
        )
        .fallback(methods::repo::service)
        .layer(TimeoutLayer::new(args.request_timeout.into()))
@@ -434,3 +445,28 @@ pub fn get_logo_filename(filename: String) -> String {
        .to_string();
    format!("logo.{}", extension)
}

async fn read_file_contents(file: String, fallback_data: Option<&'static [u8]>) -> Vec<u8> {
    let content = tokio::fs::read(&PathBuf::from(&file))
        .await
        .unwrap_or_else(|_| {
            let data = match (file.is_empty(), fallback_data) {
                (true, Some(fallback)) => {
                    // file is empty, there is fallback data
                    fallback.into()
                }
                (true, None) => {
                    // file is empty, there is no fallback data
                    warn!("Fallback data missing.");
                    Vec::new()
                }
                (false, _) => {
                    // return the data
                    file.into()
                }
            };

            data
        });
    content
}
diff --git a/statics/robots.txt b/statics/robots.txt
new file mode 100644
index 0000000..c2a49f4
--- /dev/null
+++ b/statics/robots.txt
@@ -0,0 +1,2 @@
User-agent: *
Allow: /