From f7b2da2b5aa076855fa78ef37cc63377856f842b Mon Sep 17 00:00:00 2001 From: Valentin Brandl Date: Fri, 19 Apr 2019 22:11:03 +0200 Subject: [PATCH 1/5] Add serde and internal error --- src/error.rs | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/error.rs b/src/error.rs index df5bf64..b639c22 100644 --- a/src/error.rs +++ b/src/error.rs @@ -2,19 +2,23 @@ use actix_web::{HttpResponse, ResponseError}; #[derive(Debug)] pub(crate) enum Error { - Git(git2::Error), - Io(std::io::Error), Badge(String), + Git(git2::Error), + Internal, + Io(std::io::Error), ParseColor, + Serial(serde_json::Error), } impl std::fmt::Display for Error { fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result { match self { - Error::Git(e) => write!(fmt, "Git({})", e), - Error::Io(e) => write!(fmt, "Io({})", e), Error::Badge(s) => write!(fmt, "Badge({})", s), + Error::Git(e) => write!(fmt, "Git({})", e), + Error::Internal => write!(fmt, "Internal Error"), + Error::Io(e) => write!(fmt, "Io({})", e), Error::ParseColor => write!(fmt, "Parse error"), + Error::Serial(e) => write!(fmt, "Serial({})", e), } } } @@ -44,3 +48,9 @@ impl From for Error { Error::Io(err) } } + +impl From for Error { + fn from(err: serde_json::Error) -> Self { + Error::Serial(err) + } +} From b31f66d54bd83522169ef8123327fa06d9a64b7f Mon Sep 17 00:00:00 2001 From: Valentin Brandl Date: Fri, 19 Apr 2019 22:12:41 +0200 Subject: [PATCH 2/5] Add serde_json to serialize and deserialize the cache --- Cargo.lock | 1 + Cargo.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 1e5a177..3e6a2bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -684,6 +684,7 @@ dependencies = [ "pretty_env_logger 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)", "structopt 0.2.15 (registry+https://github.com/rust-lang/crates.io-index)", ] diff --git a/Cargo.toml b/Cargo.toml index 83a6e03..33df8bb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,4 +14,5 @@ openssl-probe = "0.1.2" pretty_env_logger = "0.3.0" serde = "1.0.90" serde_derive = "1.0.90" +serde_json = "1.0.39" structopt = "0.2.15" From 0deec63c2fc16044d389827d5017421a9195bdce Mon Sep 17 00:00:00 2001 From: Valentin Brandl Date: Fri, 19 Apr 2019 22:12:57 +0200 Subject: [PATCH 3/5] Ignore default cache directory --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ad235c2..53e1610 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /target **/*.rs.bk repos +cache From 9284bceda69170c5d1c1a98963c631eb7f35b892 Mon Sep 17 00:00:00 2001 From: Valentin Brandl Date: Fri, 19 Apr 2019 22:51:58 +0200 Subject: [PATCH 4/5] Implement caching --- src/cache.rs | 63 +++++++++++++++++++++++++++++++++++++ src/main.rs | 89 ++++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 128 insertions(+), 24 deletions(-) create mode 100644 src/cache.rs diff --git a/src/cache.rs b/src/cache.rs new file mode 100644 index 0000000..5ae46d3 --- /dev/null +++ b/src/cache.rs @@ -0,0 +1,63 @@ +use crate::Error; +use std::{ + fs::{create_dir_all, File, OpenOptions}, + io::BufReader, + path::Path, +}; + +/// Enum to indicate the state of the cache +pub(crate) enum CacheState { + /// Current head and cached head are the same + Current(u64), + /// Cached head is older than current head + Old(Cache), + /// No cache was found + No, +} + +impl CacheState { + pub(crate) fn read_from_file(path: impl AsRef, head: &str) -> Result { + if path.as_ref().exists() { + let cache: Cache = serde_json::from_reader(BufReader::new(File::open(path)?))?; + if cache.head == head { + Ok(CacheState::Current(cache.count)) + } else { + Ok(CacheState::Old(cache)) + } + } else { + Ok(CacheState::No) + } + } + + pub(crate) fn calculate_new_cache(self, count: u64, head: String) -> Cache { + match self { + CacheState::Old(mut cache) => { + cache.head = head; + cache.count += count; + cache + } + CacheState::No | CacheState::Current(_) => Cache { head, count }, + } + } +} + +#[derive(Serialize, Deserialize)] +pub(crate) struct Cache { + pub head: String, + pub count: u64, +} + +impl Cache { + pub(crate) fn write_to_file(&self, path: impl AsRef) -> Result<(), Error> { + create_dir_all(path.as_ref().parent().ok_or(Error::Internal)?)?; + serde_json::to_writer( + OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(path)?, + self, + )?; + Ok(()) + } +} diff --git a/src/main.rs b/src/main.rs index 444af0f..8b24781 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,12 +1,15 @@ #[macro_use] extern crate actix_web; +extern crate serde_json; #[macro_use] extern crate serde_derive; +mod cache; mod color; mod error; use crate::{ + cache::CacheState, color::{ColorKind, ToCode}, error::Error, }; @@ -32,7 +35,10 @@ use std::{ }; use structopt::StructOpt; -type State = Arc; +struct State { + repos: String, + cache: String, +} const INDEX: &str = include_str!("../static/index.html"); const CSS: &str = include_str!("../static/tacit-css.min.css"); @@ -47,6 +53,14 @@ struct Opt { )] /// Path to store cloned repositories outdir: PathBuf, + #[structopt( + short = "c", + long = "cachedir", + parse(from_os_str), + default_value = "./cache" + )] + /// Path to store cache + cachedir: PathBuf, #[structopt(short = "p", long = "port", default_value = "8080")] /// Port to listen on port: u16, @@ -67,25 +81,46 @@ fn pull(path: impl AsRef) -> Result<(), Error> { Ok(()) } -fn hoc(repo: &str) -> Result { +fn hoc(repo: &str, repo_dir: &str, cache_dir: &str) -> Result { + let repo_dir = format!("{}/{}", repo_dir, repo); + let cache_dir = format!("{}/{}.json", cache_dir, repo); + let cache_dir = Path::new(&cache_dir); + let head = format!( + "{}", + Repository::open_bare(&repo_dir)? + .head()? + .target() + .ok_or(Error::Internal)? + ); + let mut arg = vec![ + "log".to_string(), + "--pretty=tformat:".to_string(), + "--numstat".to_string(), + "--ignore-space-change".to_string(), + "--ignore-all-space".to_string(), + "--ignore-submodules".to_string(), + "--no-color".to_string(), + "--find-copies-harder".to_string(), + "-M".to_string(), + "--diff-filter=ACDM".to_string(), + ]; + let cache = CacheState::read_from_file(&cache_dir, &head)?; + match &cache { + CacheState::Current(res) => return Ok(*res), + CacheState::Old(cache) => { + arg.push(format!("{}..HEAD", cache.head)); + } + CacheState::No => {} + }; + arg.push("--".to_string()); + arg.push(".".to_string()); let output = Command::new("git") - .arg("log") - .arg("--pretty=tformat:") - .arg("--numstat") - .arg("--ignore-space-change") - .arg("--ignore-all-space") - .arg("--ignore-submodules") - .arg("--no-color") - .arg("--find-copies-harder") - .arg("-M") - .arg("--diff-filter=ACDM") - .arg("--") - .arg(".") - .current_dir(repo) + .args(&arg) + .current_dir(&repo_dir) .output()? .stdout; let output = String::from_utf8_lossy(&output); - let res: u64 = output + let count: u64 = output .lines() .map(|s| { s.split_whitespace() @@ -96,17 +131,20 @@ fn hoc(repo: &str) -> Result { }) .sum(); - Ok(res) + let cache = cache.calculate_new_cache(count, head); + cache.write_to_file(cache_dir)?; + + Ok(cache.count) } fn calculate_hoc( service: &str, - state: web::Data, + state: web::Data>, data: web::Path<(String, String)>, color: web::Query, ) -> Result { let service_path = format!("{}/{}/{}", service, data.0, data.1); - let path = format!("{}/{}", *state, service_path); + let path = format!("{}/{}", state.repos, service_path); let file = Path::new(&path); if !file.exists() { create_dir_all(file)?; @@ -115,7 +153,7 @@ fn calculate_hoc( repo.remote_set_url("origin", &format!("https://{}", service_path))?; } pull(&path)?; - let hoc = hoc(&path)?; + let hoc = hoc(&service_path, &state.repos, &state.cache)?; let color = color .into_inner() .color @@ -146,7 +184,7 @@ fn calculate_hoc( } fn github( - state: web::Data, + state: web::Data>, data: web::Path<(String, String)>, color: web::Query, ) -> Result { @@ -154,7 +192,7 @@ fn github( } fn gitlab( - state: web::Data, + state: web::Data>, data: web::Path<(String, String)>, color: web::Query, ) -> Result { @@ -162,7 +200,7 @@ fn gitlab( } fn bitbucket( - state: web::Data, + state: web::Data>, data: web::Path<(String, String)>, color: web::Query, ) -> Result { @@ -229,7 +267,10 @@ fn main() -> std::io::Result<()> { openssl_probe::init_ssl_cert_env_vars(); let opt = Opt::from_args(); let interface = format!("{}:{}", opt.host, opt.port); - let state = Arc::new(opt.outdir.display().to_string()); + let state = Arc::new(State { + repos: opt.outdir.display().to_string(), + cache: opt.cachedir.display().to_string(), + }); HttpServer::new(move || { App::new() .data(state.clone()) From 9129d6d1713761487dfd2400d1e87be10e4b900f Mon Sep 17 00:00:00 2001 From: Valentin Brandl Date: Fri, 19 Apr 2019 22:57:32 +0200 Subject: [PATCH 5/5] Document the caching mechanism --- doc/caching.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 doc/caching.md diff --git a/doc/caching.md b/doc/caching.md new file mode 100644 index 0000000..4808e49 --- /dev/null +++ b/doc/caching.md @@ -0,0 +1,6 @@ +# Caching + +To prevent calculating the whole stats each time, the `HEAD` and HoC is cached, once it was calculated. If a cached +version is found, current `HEAD` and cached `HEAD` are compared, if they are the same, the cached value is returned, +else only the HoC between the cached `HEAD` and the current `HEAD` is calculated, added to the cached score and the +cache gets updated.