From a41e966d637b74dec806da7193a54996f0760e56 Mon Sep 17 00:00:00 2001 From: Jordan Doyle Date: Sun, 17 Jul 2022 20:39:33 +0100 Subject: [PATCH] Load reflog from sled --- src/database/indexer.rs | 65 +++++++++++++++++++++++++++++++++++++++++++++++++++++++---------- src/database/schema/commit.rs | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++-------- src/database/schema/prefixes.rs | 10 ++++++---- src/database/schema/repository.rs | 11 +++++------ src/git.rs | 43 ------------------------------------------- src/methods/filters.rs | 4 ++++ src/methods/repo.rs | 23 ++++++++++++++++------- templates/repo/log.html | 10 +++++----- 8 files changed, 142 insertions(+), 83 deletions(-) diff --git a/src/database/indexer.rs b/src/database/indexer.rs index 56bf671..109f197 100644 --- a/src/database/indexer.rs +++ b/src/database/indexer.rs @@ -1,31 +1,76 @@ +use git2::Sort; use std::path::{Path, PathBuf}; use time::OffsetDateTime; +use crate::database::schema::commit::{Author, Commit}; use crate::database::schema::repository::{Repository, RepositoryId}; pub fn run_indexer(db: &sled::Db) { let scan_path = Path::new("/Users/jordan/Code/test-git"); update_repository_metadata(scan_path, &db); - for (relative_path, _repository) in Repository::fetch_all(&db) { + for (relative_path, db_repository) in Repository::fetch_all(&db) { let git_repository = git2::Repository::open(scan_path.join(relative_path)).unwrap(); for reference in git_repository.references().unwrap() { - let _reference = if let Some(reference) = reference.as_ref().ok().and_then(|v| v.name()) + let reference = if let Some(reference) = reference.as_ref().ok().and_then(|v| v.name()) { reference } else { continue; }; - // let mut revwalk = git_repository.revwalk().unwrap(); - // revwalk.set_sorting(Sort::REVERSE).unwrap(); - // revwalk.push_ref(reference).unwrap(); - // - // for rev in revwalk { - // let rev = rev.unwrap(); - // let commit = git_repository.find_commit(rev).unwrap(); - // } + let commit_tree = db_repository.commit_tree(db, reference); + + // TODO: only scan revs from the last time we looked + let mut revwalk = git_repository.revwalk().unwrap(); + revwalk.set_sorting(Sort::REVERSE).unwrap(); + revwalk.push_ref(reference).unwrap(); + + let mut i = 0; + + for rev in revwalk { + let rev = rev.unwrap(); + let commit = git_repository.find_commit(rev).unwrap(); + + let author = commit.author(); + let committer = commit.committer(); + + // TODO: all these unwrap_or_defaults need to properly handle non-utf8 data + let author = Author { + name: author.name().map(ToString::to_string).unwrap_or_default(), + email: author.email().map(ToString::to_string).unwrap_or_default(), + // TODO: this needs to deal with offset + time: OffsetDateTime::from_unix_timestamp(author.when().seconds()).unwrap(), + }; + let committer = Author { + name: committer + .name() + .map(ToString::to_string) + .unwrap_or_default(), + email: committer + .email() + .map(ToString::to_string) + .unwrap_or_default(), + // TODO: this needs to deal with offset + time: OffsetDateTime::from_unix_timestamp(committer.when().seconds()).unwrap(), + }; + + let db_commit = Commit { + summary: commit + .summary() + .map(ToString::to_string) + .unwrap_or_default(), + message: commit.body().map(ToString::to_string).unwrap_or_default(), + committer, + author, + hash: commit.id().as_bytes().to_vec(), + }; + + i += 1; + + db_commit.insert(&commit_tree, i); + } } } } diff --git a/src/database/schema/commit.rs b/src/database/schema/commit.rs index eca799a..db513b4 100644 --- a/src/database/schema/commit.rs +++ b/src/database/schema/commit.rs @@ -1,20 +1,63 @@ use serde::{Deserialize, Serialize}; +use std::ops::Deref; +use time::OffsetDateTime; #[derive(Serialize, Deserialize, Debug)] pub struct Commit { - age: String, - message: String, - author: String, + pub summary: String, + pub message: String, + pub author: Author, + pub committer: Author, + pub hash: Vec, } -impl Commit {} +#[derive(Serialize, Deserialize, Debug)] +pub struct Author { + pub name: String, + pub email: String, + pub time: OffsetDateTime, +} -pub struct CommitVault { - _tree: sled::Tree, +impl Commit { + pub fn insert(&self, database: &CommitTree, id: usize) { + database + .insert(id.to_be_bytes(), bincode::serialize(self).unwrap()) + .unwrap(); + } } -impl CommitVault { +pub struct CommitTree(sled::Tree); + +impl Deref for CommitTree { + type Target = sled::Tree; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl CommitTree { pub(super) fn new(tree: sled::Tree) -> Self { - Self { _tree: tree } + Self(tree) + } + + pub fn fetch_latest(&self, amount: usize, offset: usize) -> Vec { + let (latest_key, _) = self.last().unwrap().unwrap(); + let mut latest_key_bytes = [0; std::mem::size_of::()]; + latest_key_bytes.copy_from_slice(&latest_key); + + let end = usize::from_be_bytes(latest_key_bytes).saturating_sub(offset); + let start = end.saturating_sub(amount); + + self.range(start.to_be_bytes()..end.to_be_bytes()) + .rev() + .map(|res| { + let (_, value) = res?; + let details = bincode::deserialize(&value).unwrap(); + + Ok(details) + }) + .collect::, sled::Error>>() + .unwrap() } } diff --git a/src/database/schema/prefixes.rs b/src/database/schema/prefixes.rs index e088bba..4473e4a 100644 --- a/src/database/schema/prefixes.rs +++ b/src/database/schema/prefixes.rs @@ -20,15 +20,17 @@ impl TreePrefix { prefixed } - pub fn commit_id>(repository: RepositoryId, commit: T) -> Vec { - let commit = commit.as_ref(); + pub fn commit_id>(repository: RepositoryId, reference: T) -> Vec { + let reference = reference.as_ref(); let mut prefixed = Vec::with_capacity( - commit.len() + std::mem::size_of::() + std::mem::size_of::(), + reference.len() + + std::mem::size_of::() + + std::mem::size_of::(), ); prefixed.push(TreePrefix::Commit as u8); prefixed.extend_from_slice(&repository.to_ne_bytes()); - prefixed.extend_from_slice(&commit); + prefixed.extend_from_slice(&reference); prefixed } diff --git a/src/database/schema/repository.rs b/src/database/schema/repository.rs index da272e6..7a063b1 100644 --- a/src/database/schema/repository.rs +++ b/src/database/schema/repository.rs @@ -1,4 +1,4 @@ -use crate::database::schema::commit::CommitVault; +use crate::database::schema::commit::CommitTree; use crate::database::schema::prefixes::TreePrefix; use serde::{Deserialize, Serialize}; use std::collections::HashMap; @@ -27,6 +27,7 @@ impl Repository { .scan_prefix([TreePrefix::Repository as u8]) .filter_map(Result::ok) .map(|(k, v)| { + // strip the prefix we've just scanned for let key = String::from_utf8_lossy(&k[1..]).to_string(); let value = bincode::deserialize(&v).unwrap(); @@ -53,14 +54,12 @@ impl Repository { .unwrap() } - #[allow(dead_code)] - pub fn commit_vault(&self, database: &sled::Db, commit: &str) -> CommitVault { - let commit = hex::decode(commit).unwrap(); + pub fn commit_tree(&self, database: &sled::Db, reference: &str) -> CommitTree { let tree = database - .open_tree(TreePrefix::commit_id(self.id, commit)) + .open_tree(TreePrefix::commit_id(self.id, reference)) .unwrap(); - CommitVault::new(tree) + CommitTree::new(tree) } } diff --git a/src/git.rs b/src/git.rs index 11e3811..dd41468 100644 --- a/src/git.rs +++ b/src/git.rs @@ -299,49 +299,6 @@ impl OpenRepository { }) .await } - - #[instrument(skip(self))] - pub async fn commits( - self: Arc, - branch: Option<&str>, - offset: usize, - ) -> (Vec, Option) { - const LIMIT: usize = 200; - - let ref_name = branch.map(|branch| format!("refs/heads/{}", branch)); - - tokio::task::spawn_blocking(move || { - let repo = self.repo.lock(); - let mut revs = repo.revwalk().unwrap(); - - if let Some(ref_name) = ref_name.as_deref() { - revs.push_ref(ref_name).unwrap(); - } else { - revs.push_head().unwrap(); - } - - let mut commits: Vec = revs - .skip(offset) - .take(LIMIT + 1) - .map(|rev| { - let rev = rev.unwrap(); - repo.find_commit(rev).unwrap().into() - }) - .collect(); - - // TODO: avoid having to take + 1 and popping the last commit off - let next_offset = if commits.len() > LIMIT { - commits.truncate(LIMIT); - Some(offset + LIMIT) - } else { - None - }; - - (commits, next_offset) - }) - .await - .unwrap() - } } pub enum PathDestination { diff --git a/src/methods/filters.rs b/src/methods/filters.rs index 2cb33e2..3287bf4 100644 --- a/src/methods/filters.rs +++ b/src/methods/filters.rs @@ -5,3 +5,7 @@ pub fn timeago(s: time::OffsetDateTime) -> Result { pub fn file_perms(s: &i32) -> Result { Ok(unix_mode::to_string(s.unsigned_abs())) } + +pub fn hex(s: &[u8]) -> Result { + Ok(hex::encode(s)) +} diff --git a/src/methods/repo.rs b/src/methods/repo.rs index 0cf36ec..465decb 100644 --- a/src/methods/repo.rs +++ b/src/methods/repo.rs @@ -158,21 +158,30 @@ pub struct LogQuery { #[template(path = "repo/log.html")] pub struct LogView { repo: Repository, - commits: Vec, + commits: Vec, next_offset: Option, branch: Option, } pub async fn handle_log( Extension(repo): Extension, - Extension(RepositoryPath(repository_path)): Extension, - Extension(git): Extension>, + Extension(db): Extension, Query(query): Query, ) -> Response { - let open_repo = git.repo(repository_path).await; - let (commits, next_offset) = open_repo - .commits(query.branch.as_deref(), query.offset.unwrap_or(0)) - .await; + let offset = query.offset.unwrap_or(0); + + let reference = format!("refs/heads/{}", query.branch.as_deref().unwrap_or("master")); + let repository = + crate::database::schema::repository::Repository::open(&db, &*repo).unwrap(); + let commit_tree = repository.commit_tree(&db, &reference); + let mut commits = commit_tree.fetch_latest(101, offset); + + let next_offset = if commits.len() == 101 { + commits.pop(); + Some(offset + 100) + } else { + None + }; into_response(&LogView { repo, diff --git a/templates/repo/log.html b/templates/repo/log.html index f73e143..4239127 100644 --- a/templates/repo/log.html +++ b/templates/repo/log.html @@ -16,14 +16,14 @@ {% for commit in commits %} -