Add markdown parsing to README with syntax highlighting
Diff
Cargo.lock | 315 ++++++++++++++++++++++++++++++++++++++++++++++-
Cargo.toml | 1 +-
src/git.rs | 66 +++++++---
src/git_cgi.rs | 1 +-
src/main.rs | 1 +-
src/methods/filters.rs | 8 +-
src/methods/repo.rs | 4 +-
src/syntax_highlight.rs | 39 ++++++-
templates/repo/about.html | 15 +-
9 files changed, 428 insertions(+), 22 deletions(-)
@@ -227,7 +227,7 @@ dependencies = [
"semver",
"serde",
"serde_yaml",
"syntect",
"syntect 5.0.0",
"thiserror",
"unicode-width",
"walkdir",
@@ -243,12 +243,48 @@ dependencies = [
]
[[package]]
name = "bit-set"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e11e16035ea35e4e5997b393eacbf6f63983188f7a2ad25bfb13465f5ad59de"
dependencies = [
"bit-vec",
]
[[package]]
name = "bit-vec"
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb"
[[package]]
name = "bitflags"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "block-buffer"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b"
dependencies = [
"block-padding",
"byte-tools",
"byteorder",
"generic-array",
]
[[package]]
name = "block-padding"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5"
dependencies = [
"byte-tools",
]
[[package]]
name = "bstr"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -264,6 +300,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37ccbd214614c6783386c1af30caf03192f17891059cecc394b4fb119e363de3"
[[package]]
name = "byte-tools"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7"
[[package]]
name = "bytecount"
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -366,6 +408,21 @@ dependencies = [
[[package]]
name = "clap"
version = "2.34.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"
dependencies = [
"ansi_term",
"atty",
"bitflags",
"strsim 0.8.0",
"textwrap 0.11.0",
"unicode-width",
"vec_map",
]
[[package]]
name = "clap"
version = "3.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "190814073e85d238f31ff738fcb0bf6910cedeb73376c87cd69291028966fd83"
@@ -375,9 +432,9 @@ dependencies = [
"clap_lex",
"indexmap",
"once_cell",
"strsim",
"strsim 0.10.0",
"termcolor",
"textwrap",
"textwrap 0.15.0",
]
[[package]]
@@ -411,6 +468,26 @@ dependencies = [
]
[[package]]
name = "comrak"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "15bf1e432b302dc6236dd0db580d182ce520bb24af82d6462e2d7a5e0a31c50d"
dependencies = [
"clap 2.34.0",
"entities",
"lazy_static",
"memchr",
"pest",
"pest_derive",
"regex",
"shell-words",
"syntect 4.6.0",
"typed-arena",
"unicode_categories",
"xdg",
]
[[package]]
name = "concurrent-queue"
version = "1.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -513,6 +590,35 @@ dependencies = [
]
[[package]]
name = "digest"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5"
dependencies = [
"generic-array",
]
[[package]]
name = "dirs"
version = "4.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059"
dependencies = [
"dirs-sys",
]
[[package]]
name = "dirs-sys"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6"
dependencies = [
"libc",
"redox_users",
"winapi",
]
[[package]]
name = "encode_unicode"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -583,6 +689,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a246d82be1c9d791c5dfde9a2bd045fc3cbba3fa2b11ad558f27d01712f00569"
[[package]]
name = "entities"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5320ae4c3782150d900b79807611a59a99fc9a1d61d686faafc24b93fc8d7ca"
[[package]]
name = "error-chain"
version = "0.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -598,6 +710,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77f3309417938f28bf8228fcff79a4a37103981e3e186d2ccd19c74b38f4eb71"
[[package]]
name = "fake-simd"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed"
[[package]]
name = "fancy-regex"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d6b8560a05112eb52f04b00e5d3790c0dd75d9d980eb8a122fb23b92a623ccf"
dependencies = [
"bit-set",
"regex",
]
[[package]]
name = "fastrand"
version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -762,6 +890,15 @@ dependencies = [
]
[[package]]
name = "generic-array"
version = "0.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ffdf9f34f1447443d37393cc6c2b8313aebddcd96906caf34e54c68d8e57d7bd"
dependencies = [
"typenum",
]
[[package]]
name = "getrandom"
version = "0.2.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -973,6 +1110,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "lazycell"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
[[package]]
name = "libc"
version = "0.2.126"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1062,6 +1205,12 @@ dependencies = [
]
[[package]]
name = "maplit"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d"
[[package]]
name = "matches"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1287,6 +1436,12 @@ dependencies = [
]
[[package]]
name = "opaque-debug"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c"
[[package]]
name = "openssl-probe"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1387,6 +1542,49 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
[[package]]
name = "pest"
version = "2.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53"
dependencies = [
"ucd-trie",
]
[[package]]
name = "pest_derive"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "833d1ae558dc601e9a60366421196a8d94bc0ac980476d0b67e1d0988d72b2d0"
dependencies = [
"pest",
"pest_generator",
]
[[package]]
name = "pest_generator"
version = "2.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "99b8db626e31e5b81787b9783425769681b347011cc59471e33ea46d2ea0cf55"
dependencies = [
"pest",
"pest_meta",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "pest_meta"
version = "2.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "54be6e404f5317079812fc8f9f5279de376d8856929e21c184ecf6bbd692a11d"
dependencies = [
"maplit",
"pest",
"sha-1",
]
[[package]]
name = "phf"
version = "0.7.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1668,6 +1866,17 @@ dependencies = [
]
[[package]]
name = "redox_users"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"
dependencies = [
"getrandom",
"redox_syscall",
"thiserror",
]
[[package]]
name = "regex"
version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1712,7 +1921,8 @@ dependencies = [
"bat",
"bincode",
"bytes",
"clap",
"clap 3.2.8",
"comrak",
"futures",
"git2",
"hex",
@@ -1726,7 +1936,7 @@ dependencies = [
"rsass",
"serde",
"sled",
"syntect",
"syntect 5.0.0",
"time 0.3.11",
"timeago",
"tokio",
@@ -1858,6 +2068,18 @@ dependencies = [
]
[[package]]
name = "sha-1"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7d94d0bede923b3cea61f3f1ff57ff8cdfd77b400fb8f9998949e0cf04163df"
dependencies = [
"block-buffer",
"digest",
"fake-simd",
"opaque-debug",
]
[[package]]
name = "sharded-slab"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1867,6 +2089,12 @@ dependencies = [
]
[[package]]
name = "shell-words"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde"
[[package]]
name = "signal-hook-registry"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1949,6 +2177,12 @@ checksum = "8207e78455ffdf55661170876f88daf85356e4edd54e0a3dbc79586ca1e50cbe"
[[package]]
name = "strsim"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
[[package]]
name = "strsim"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
@@ -1984,6 +2218,29 @@ dependencies = [
[[package]]
name = "syntect"
version = "4.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b20815bbe80ee0be06e6957450a841185fcf690fe0178f14d77a05ce2caa031"
dependencies = [
"bincode",
"bitflags",
"fancy-regex",
"flate2",
"fnv",
"lazy_static",
"lazycell",
"onig",
"plist",
"regex-syntax",
"serde",
"serde_derive",
"serde_json",
"walkdir",
"yaml-rust",
]
[[package]]
name = "syntect"
version = "5.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c6c454c27d9d7d9a84c7803aaa3c50cd088d2906fe3c6e42da3209aa623576a8"
@@ -2046,6 +2303,15 @@ dependencies = [
[[package]]
name = "textwrap"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
dependencies = [
"unicode-width",
]
[[package]]
name = "textwrap"
version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb"
@@ -2285,6 +2551,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642"
[[package]]
name = "typed-arena"
version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9b2228007eba4120145f785df0f6c92ea538f5a3635a612ecf4e334c8c1446d"
[[package]]
name = "typenum"
version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987"
[[package]]
name = "ucd-trie"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89570599c4fe5585de2b388aab47e99f7fa4e9238a1399f707a02e356058141c"
[[package]]
name = "unicase"
version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2327,6 +2611,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04"
[[package]]
name = "unicode_categories"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"
[[package]]
name = "unix_mode"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2366,6 +2656,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
[[package]]
name = "vec_map"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
[[package]]
name = "version_check"
version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2558,6 +2854,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680"
[[package]]
name = "xdg"
version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c4583db5cbd4c4c0303df2d15af80f0539db703fa1c68802d4cbbd2dd0f88f6"
dependencies = [
"dirs",
]
[[package]]
name = "xml-rs"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -11,6 +11,7 @@ axum = "0.5"
bat = { version = "0.21", default-features = false, features = ["build-assets"] }
bytes = "1.1"
bincode = "1.3"
comrak = "0.14"
clap = { version = "3.2", features = ["cargo"] }
futures = "0.3"
git2 = "0.14"
@@ -1,6 +1,10 @@
use std::ffi::OsStr;
use std::path::Path;
use std::{borrow::Cow, fmt::Write, path::PathBuf, sync::Arc, time::Duration};
use crate::syntax_highlight::ComrakSyntectAdapter;
use bytes::{Bytes, BytesMut};
use comrak::{ComrakOptions, ComrakPlugins};
use git2::{
BranchType, DiffFormat, DiffLineType, DiffOptions, DiffStatsFormat, ObjectType, Oid, Signature,
};
@@ -14,7 +18,7 @@ use tracing::instrument;
pub struct Git {
commits: Cache<Oid, Arc<Commit>>,
readme_cache: Cache<PathBuf, Option<Arc<str>>>,
readme_cache: Cache<PathBuf, Option<(ReadmeFormat, Arc<str>)>>,
refs: Cache<PathBuf, Arc<Refs>>,
syntax_set: SyntaxSet,
}
@@ -230,7 +234,7 @@ impl OpenRepository {
}
#[instrument(skip(self))]
pub async fn readme(self: Arc<Self>) -> Option<Arc<str>> {
pub async fn readme(self: Arc<Self>) -> Option<(ReadmeFormat, Arc<str>)> {
const README_FILES: &[&str] = &["README.md", "README", "README.txt"];
let git = self.git.clone();
@@ -244,19 +248,38 @@ impl OpenRepository {
let commit = head.peel_to_commit().unwrap();
let tree = commit.tree().unwrap();
let blob = README_FILES
.iter()
.map(|file| tree.get_name(file))
.next()
.flatten()?
.to_object(&repo)
.unwrap()
.into_blob()
.unwrap();
for name in README_FILES {
let tree_entry = if let Some(file) = tree.get_name(name) {
file
} else {
continue;
};
let blob = if let Some(blob) = tree_entry
.to_object(&repo)
.ok()
.and_then(|v| v.into_blob().ok())
{
blob
} else {
continue;
};
let content = if let Ok(content) = std::str::from_utf8(blob.content()) {
content
} else {
continue;
};
if Path::new(name).extension().and_then(OsStr::to_str) == Some("md") {
let value = parse_and_transform_markdown(content, &self.git.syntax_set);
return Some((ReadmeFormat::Markdown, Arc::from(value)));
}
Some(Arc::from(
String::from_utf8(blob.content().to_vec()).unwrap(),
))
return Some((ReadmeFormat::Plaintext, Arc::from(content)));
}
None
})
.await
.unwrap()
@@ -313,6 +336,21 @@ impl OpenRepository {
}
}
fn parse_and_transform_markdown(s: &str, syntax_set: &SyntaxSet) -> String {
let mut plugins = ComrakPlugins::default();
let highlighter = ComrakSyntectAdapter { syntax_set };
plugins.render.codefence_syntax_highlighter = Some(&highlighter);
comrak::markdown_to_html_with_plugins(s, &ComrakOptions::default(), &plugins)
}
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
pub enum ReadmeFormat {
Markdown,
Plaintext,
}
pub enum PathDestination {
Tree(Vec<TreeItem>),
File(FileWithContent),
@@ -4,6 +4,7 @@ use axum::http::HeaderValue;
use axum::response::Response;
use std::str::FromStr;
pub fn cgi_to_response(buffer: &[u8]) -> Response {
let mut headers = [httparse::EMPTY_HEADER; 10];
let (body_offset, headers) = httparse::parse_headers(buffer, &mut headers)
@@ -21,6 +21,7 @@ mod git;
mod git_cgi;
mod layers;
mod methods;
mod syntax_highlight;
const CRATE_VERSION: &str = clap::crate_version!();
@@ -16,3 +16,11 @@ pub fn hex(s: &[u8]) -> Result<String, askama::Error> {
pub fn md5(s: &str) -> Result<String, askama::Error> {
Ok(hex::encode(&md5::compute(s).0))
}
#[allow(dead_code)]
pub fn md(md: &str) -> Result<String, askama::Error> {
Ok(comrak::markdown_to_html(
md,
&comrak::ComrakOptions::default(),
))
}
@@ -25,7 +25,7 @@ use tower::{util::BoxCloneService, Service};
use yoke::Yoke;
use super::filters;
use crate::git::{DetailedTag, FileWithContent, PathDestination, Refs, TreeItem};
use crate::git::{DetailedTag, FileWithContent, PathDestination, ReadmeFormat, Refs, TreeItem};
use crate::{git::Commit, into_response, layers::UnwrapInfallible, Git};
#[derive(Clone)]
@@ -295,7 +295,7 @@ pub async fn handle_refs(
#[template(path = "repo/about.html")]
pub struct AboutView {
repo: Repository,
readme: Option<Arc<str>>,
readme: Option<(ReadmeFormat, Arc<str>)>,
}
pub async fn handle_about(
@@ -0,0 +1,39 @@
use comrak::adapters::SyntaxHighlighterAdapter;
use std::collections::HashMap;
use syntect::html::{ClassStyle, ClassedHTMLGenerator};
use syntect::parsing::SyntaxSet;
use syntect::util::LinesWithEndings;
pub struct ComrakSyntectAdapter<'a> {
pub(crate) syntax_set: &'a SyntaxSet,
}
impl SyntaxHighlighterAdapter for ComrakSyntectAdapter<'_> {
fn highlight(&self, lang: Option<&str>, code: &str) -> String {
let syntax = lang
.and_then(|v| self.syntax_set.find_syntax_by_token(v))
.unwrap_or_else(|| self.syntax_set.find_syntax_plain_text());
let mut html_generator =
ClassedHTMLGenerator::new_with_class_style(syntax, self.syntax_set, ClassStyle::Spaced);
for line in LinesWithEndings::from(code) {
html_generator
.parse_html_for_line_which_includes_newline(line)
.unwrap();
}
format!(
"<code>{}</code>",
html_generator.finalize().replace('\n', "</code>\n<code>")
)
}
fn build_pre_tag(&self, _attributes: &HashMap<String, String>) -> String {
r#"<pre>"#.to_string()
}
fn build_code_tag(&self, _attributes: &HashMap<String, String>) -> String {
String::with_capacity(0)
}
}
@@ -1,10 +1,23 @@
{% extends "repo/base.html" %}
{% block head %}
{% if let Some(readme) = readme %}
{% if readme.0 == crate::git::ReadmeFormat::Markdown %}
<link rel="stylesheet" type="text/css" href="/highlight.css" />
{% endif %}
{% endif %}
{% endblock %}
{% block about_nav_class %}active{% endblock %}
{% block content %}
{% if let Some(readme) = readme %}
<pre>{{ readme }}</pre>
{% match readme.0 %}
{% when crate::git::ReadmeFormat::Markdown %}
{{ readme.1|safe }}
{% when crate::git::ReadmeFormat::Plaintext %}
<pre>{{ readme.1 }}</pre>
{% endmatch %}
{% else %}
No README in repository HEAD.
{% endif %}
