Merge branch 'feature/caching'

Closes #4
This commit is contained in:
Valentin Brandl 2019-04-19 22:57:39 +02:00
commit 906511f7fa
No known key found for this signature in database
GPG Key ID: 30D341DD34118D7D
7 changed files with 151 additions and 28 deletions

1
.gitignore vendored
View File

@ -1,3 +1,4 @@
/target /target
**/*.rs.bk **/*.rs.bk
repos repos
cache

1
Cargo.lock generated
View File

@ -684,6 +684,7 @@ dependencies = [
"pretty_env_logger 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "pretty_env_logger 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_derive 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)",
"structopt 0.2.15 (registry+https://github.com/rust-lang/crates.io-index)", "structopt 0.2.15 (registry+https://github.com/rust-lang/crates.io-index)",
] ]

View File

@ -14,4 +14,5 @@ openssl-probe = "0.1.2"
pretty_env_logger = "0.3.0" pretty_env_logger = "0.3.0"
serde = "1.0.90" serde = "1.0.90"
serde_derive = "1.0.90" serde_derive = "1.0.90"
serde_json = "1.0.39"
structopt = "0.2.15" structopt = "0.2.15"

6
doc/caching.md Normal file
View File

@ -0,0 +1,6 @@
# Caching
To prevent calculating the whole stats each time, the `HEAD` and HoC is cached, once it was calculated. If a cached
version is found, current `HEAD` and cached `HEAD` are compared, if they are the same, the cached value is returned,
else only the HoC between the cached `HEAD` and the current `HEAD` is calculated, added to the cached score and the
cache gets updated.

63
src/cache.rs Normal file
View File

@ -0,0 +1,63 @@
use crate::Error;
use std::{
fs::{create_dir_all, File, OpenOptions},
io::BufReader,
path::Path,
};
/// Enum to indicate the state of the cache
pub(crate) enum CacheState {
/// Current head and cached head are the same
Current(u64),
/// Cached head is older than current head
Old(Cache),
/// No cache was found
No,
}
impl CacheState {
pub(crate) fn read_from_file(path: impl AsRef<Path>, head: &str) -> Result<CacheState, Error> {
if path.as_ref().exists() {
let cache: Cache = serde_json::from_reader(BufReader::new(File::open(path)?))?;
if cache.head == head {
Ok(CacheState::Current(cache.count))
} else {
Ok(CacheState::Old(cache))
}
} else {
Ok(CacheState::No)
}
}
pub(crate) fn calculate_new_cache(self, count: u64, head: String) -> Cache {
match self {
CacheState::Old(mut cache) => {
cache.head = head;
cache.count += count;
cache
}
CacheState::No | CacheState::Current(_) => Cache { head, count },
}
}
}
#[derive(Serialize, Deserialize)]
pub(crate) struct Cache {
pub head: String,
pub count: u64,
}
impl Cache {
pub(crate) fn write_to_file(&self, path: impl AsRef<Path>) -> Result<(), Error> {
create_dir_all(path.as_ref().parent().ok_or(Error::Internal)?)?;
serde_json::to_writer(
OpenOptions::new()
.write(true)
.create(true)
.truncate(true)
.open(path)?,
self,
)?;
Ok(())
}
}

View File

@ -2,19 +2,23 @@ use actix_web::{HttpResponse, ResponseError};
#[derive(Debug)] #[derive(Debug)]
pub(crate) enum Error { pub(crate) enum Error {
Git(git2::Error),
Io(std::io::Error),
Badge(String), Badge(String),
Git(git2::Error),
Internal,
Io(std::io::Error),
ParseColor, ParseColor,
Serial(serde_json::Error),
} }
impl std::fmt::Display for Error { impl std::fmt::Display for Error {
fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result { fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
match self { match self {
Error::Git(e) => write!(fmt, "Git({})", e),
Error::Io(e) => write!(fmt, "Io({})", e),
Error::Badge(s) => write!(fmt, "Badge({})", s), Error::Badge(s) => write!(fmt, "Badge({})", s),
Error::Git(e) => write!(fmt, "Git({})", e),
Error::Internal => write!(fmt, "Internal Error"),
Error::Io(e) => write!(fmt, "Io({})", e),
Error::ParseColor => write!(fmt, "Parse error"), Error::ParseColor => write!(fmt, "Parse error"),
Error::Serial(e) => write!(fmt, "Serial({})", e),
} }
} }
} }
@ -44,3 +48,9 @@ impl From<std::io::Error> for Error {
Error::Io(err) Error::Io(err)
} }
} }
impl From<serde_json::Error> for Error {
fn from(err: serde_json::Error) -> Self {
Error::Serial(err)
}
}

View File

@ -1,12 +1,15 @@
#[macro_use] #[macro_use]
extern crate actix_web; extern crate actix_web;
extern crate serde_json;
#[macro_use] #[macro_use]
extern crate serde_derive; extern crate serde_derive;
mod cache;
mod color; mod color;
mod error; mod error;
use crate::{ use crate::{
cache::CacheState,
color::{ColorKind, ToCode}, color::{ColorKind, ToCode},
error::Error, error::Error,
}; };
@ -32,7 +35,10 @@ use std::{
}; };
use structopt::StructOpt; use structopt::StructOpt;
type State = Arc<String>; struct State {
repos: String,
cache: String,
}
const INDEX: &str = include_str!("../static/index.html"); const INDEX: &str = include_str!("../static/index.html");
const CSS: &str = include_str!("../static/tacit-css.min.css"); const CSS: &str = include_str!("../static/tacit-css.min.css");
@ -47,6 +53,14 @@ struct Opt {
)] )]
/// Path to store cloned repositories /// Path to store cloned repositories
outdir: PathBuf, outdir: PathBuf,
#[structopt(
short = "c",
long = "cachedir",
parse(from_os_str),
default_value = "./cache"
)]
/// Path to store cache
cachedir: PathBuf,
#[structopt(short = "p", long = "port", default_value = "8080")] #[structopt(short = "p", long = "port", default_value = "8080")]
/// Port to listen on /// Port to listen on
port: u16, port: u16,
@ -67,25 +81,46 @@ fn pull(path: impl AsRef<Path>) -> Result<(), Error> {
Ok(()) Ok(())
} }
fn hoc(repo: &str) -> Result<u64, Error> { fn hoc(repo: &str, repo_dir: &str, cache_dir: &str) -> Result<u64, Error> {
let repo_dir = format!("{}/{}", repo_dir, repo);
let cache_dir = format!("{}/{}.json", cache_dir, repo);
let cache_dir = Path::new(&cache_dir);
let head = format!(
"{}",
Repository::open_bare(&repo_dir)?
.head()?
.target()
.ok_or(Error::Internal)?
);
let mut arg = vec![
"log".to_string(),
"--pretty=tformat:".to_string(),
"--numstat".to_string(),
"--ignore-space-change".to_string(),
"--ignore-all-space".to_string(),
"--ignore-submodules".to_string(),
"--no-color".to_string(),
"--find-copies-harder".to_string(),
"-M".to_string(),
"--diff-filter=ACDM".to_string(),
];
let cache = CacheState::read_from_file(&cache_dir, &head)?;
match &cache {
CacheState::Current(res) => return Ok(*res),
CacheState::Old(cache) => {
arg.push(format!("{}..HEAD", cache.head));
}
CacheState::No => {}
};
arg.push("--".to_string());
arg.push(".".to_string());
let output = Command::new("git") let output = Command::new("git")
.arg("log") .args(&arg)
.arg("--pretty=tformat:") .current_dir(&repo_dir)
.arg("--numstat")
.arg("--ignore-space-change")
.arg("--ignore-all-space")
.arg("--ignore-submodules")
.arg("--no-color")
.arg("--find-copies-harder")
.arg("-M")
.arg("--diff-filter=ACDM")
.arg("--")
.arg(".")
.current_dir(repo)
.output()? .output()?
.stdout; .stdout;
let output = String::from_utf8_lossy(&output); let output = String::from_utf8_lossy(&output);
let res: u64 = output let count: u64 = output
.lines() .lines()
.map(|s| { .map(|s| {
s.split_whitespace() s.split_whitespace()
@ -96,17 +131,20 @@ fn hoc(repo: &str) -> Result<u64, Error> {
}) })
.sum(); .sum();
Ok(res) let cache = cache.calculate_new_cache(count, head);
cache.write_to_file(cache_dir)?;
Ok(cache.count)
} }
fn calculate_hoc( fn calculate_hoc(
service: &str, service: &str,
state: web::Data<State>, state: web::Data<Arc<State>>,
data: web::Path<(String, String)>, data: web::Path<(String, String)>,
color: web::Query<BadgeQuery>, color: web::Query<BadgeQuery>,
) -> Result<HttpResponse, Error> { ) -> Result<HttpResponse, Error> {
let service_path = format!("{}/{}/{}", service, data.0, data.1); let service_path = format!("{}/{}/{}", service, data.0, data.1);
let path = format!("{}/{}", *state, service_path); let path = format!("{}/{}", state.repos, service_path);
let file = Path::new(&path); let file = Path::new(&path);
if !file.exists() { if !file.exists() {
create_dir_all(file)?; create_dir_all(file)?;
@ -115,7 +153,7 @@ fn calculate_hoc(
repo.remote_set_url("origin", &format!("https://{}", service_path))?; repo.remote_set_url("origin", &format!("https://{}", service_path))?;
} }
pull(&path)?; pull(&path)?;
let hoc = hoc(&path)?; let hoc = hoc(&service_path, &state.repos, &state.cache)?;
let color = color let color = color
.into_inner() .into_inner()
.color .color
@ -146,7 +184,7 @@ fn calculate_hoc(
} }
fn github( fn github(
state: web::Data<State>, state: web::Data<Arc<State>>,
data: web::Path<(String, String)>, data: web::Path<(String, String)>,
color: web::Query<BadgeQuery>, color: web::Query<BadgeQuery>,
) -> Result<HttpResponse, Error> { ) -> Result<HttpResponse, Error> {
@ -154,7 +192,7 @@ fn github(
} }
fn gitlab( fn gitlab(
state: web::Data<State>, state: web::Data<Arc<State>>,
data: web::Path<(String, String)>, data: web::Path<(String, String)>,
color: web::Query<BadgeQuery>, color: web::Query<BadgeQuery>,
) -> Result<HttpResponse, Error> { ) -> Result<HttpResponse, Error> {
@ -162,7 +200,7 @@ fn gitlab(
} }
fn bitbucket( fn bitbucket(
state: web::Data<State>, state: web::Data<Arc<State>>,
data: web::Path<(String, String)>, data: web::Path<(String, String)>,
color: web::Query<BadgeQuery>, color: web::Query<BadgeQuery>,
) -> Result<HttpResponse, Error> { ) -> Result<HttpResponse, Error> {
@ -229,7 +267,10 @@ fn main() -> std::io::Result<()> {
openssl_probe::init_ssl_cert_env_vars(); openssl_probe::init_ssl_cert_env_vars();
let opt = Opt::from_args(); let opt = Opt::from_args();
let interface = format!("{}:{}", opt.host, opt.port); let interface = format!("{}:{}", opt.host, opt.port);
let state = Arc::new(opt.outdir.display().to_string()); let state = Arc::new(State {
repos: opt.outdir.display().to_string(),
cache: opt.cachedir.display().to_string(),
});
HttpServer::new(move || { HttpServer::new(move || {
App::new() App::new()
.data(state.clone()) .data(state.clone())