vbrandl/hoc
1
0
Fork 0
Code Issues 1 Pull Requests Actions Projects Releases Wiki Activity

Restructure for testability and implement first tests

Browse Source
This commit is contained in:
Valentin Brandl 2021-03-22 12:46:10 +01:00
parent 1311e724ce
commit 3486d44bc5
13 changed files with 782 additions and 560 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
Cargo.lock generated
View File

@ -817,6 +817,16 @@ dependencies = [
"version_check",
]
[[package]]
name = "gethostname"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e692e296bfac1d2533ef168d0b60ff5897b8b70a4009276834014dd8924cc028"
dependencies = [
"libc",
"winapi 0.3.9",
]
[[package]]
name = "getrandom"
version = "0.1.16"
@ -937,9 +947,12 @@ dependencies = [
"serde_derive",
"serde_json",
"tempfile",
"tokio",
"tracing",
"tracing-actix-web",
"tracing-bunyan-formatter",
"tracing-futures",
"tracing-log",
"tracing-subscriber",
"vergen",
]
@ -2339,6 +2352,23 @@ dependencies = [
"syn",
]
[[package]]
name = "tracing-bunyan-formatter"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06718867c20ea03700d41a9413610cccf5d772caea792f34cc73cdd43f0e14a6"
dependencies = [
"chrono",
"gethostname",
"log",
"serde 1.0.123",
"serde_json",
"tracing",
"tracing-core",
"tracing-log",
"tracing-subscriber",
]
[[package]]
name = "tracing-core"
version = "0.1.17"
@ -2381,9 +2411,9 @@ dependencies = [
[[package]]
name = "tracing-subscriber"
version = "0.2.16"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ab8966ac3ca27126141f7999361cc97dd6fb4b71da04c02044fa9045d98bb96"
checksum = "705096c6f83bf68ea5d357a6aa01829ddbdac531b357b45abeca842938085baa"
dependencies = [
"ansi_term",
"chrono",

13
Cargo.toml
View File

@ -5,6 +5,13 @@ authors = ["Valentin Brandl <vbrandl@riseup.net>"]
edition = "2018"
build = "build.rs"
[lib]
path = "src/lib.rs"
[[bin]]
path = "src/main.rs"
name = "hoc"
[dependencies]
actix-rt = "1.1.1"
actix-web = "3.3.2"
@ -23,12 +30,16 @@ serde_derive = "1.0.123"
serde_json = "1.0.63"
tracing = "0.1.25"
tracing-actix-web = "0.2.1"
tracing-bunyan-formatter = "0.1.7"
tracing-futures = "0.2.5"
tracing-subscriber = "0.2.16"
tracing-log = "0.1.2"
tracing-subscriber = { version = "0.2.17", features = ["registry", "env-filter"] }
[build-dependencies]
ructe = "0.13.0"
vergen = { version = "4.1.0", default-features = false, features = ["git"] }
[dev-dependencies]
ructe = "0.13.0"
tempfile = "3.2.0"
tokio = "0.2.25"

13
src/config.rs
View File

@ -17,16 +17,13 @@ pub struct Settings {
pub workers: usize,
}
pub(crate) fn init() {
dotenv::dotenv().ok();
std::env::set_var("RUST_LOG", "actix_web=info,hoc=info");
openssl_probe::init_ssl_cert_env_vars();
tracing_subscriber::fmt().init();
}
impl Settings {
#[deprecated]
pub fn new() -> Result<Self, ConfigError> {
Self::load()
}
pub fn load() -> Result<Self, ConfigError> {
let mut config = Config::new();
config
.merge(File::with_name("hoc.toml").required(false))?

26
src/error.rs
View File

@ -1,9 +1,6 @@
use crate::{
statics::{REPO_COUNT, VERSION_INFO},
templates,
};
use actix_web::{HttpResponse, ResponseError};
use std::{fmt, sync::atomic::Ordering};
use crate::{statics::VERSION_INFO, templates};
use actix_web::{http::StatusCode, HttpResponse, ResponseError};
use std::fmt;
pub(crate) type Result<T> = std::result::Result<T, Error>;
@ -35,21 +32,22 @@ impl fmt::Display for Error {
}
impl ResponseError for Error {
fn status_code(&self) -> StatusCode {
match self {
Error::BranchNotFound => StatusCode::NOT_FOUND,
_ => StatusCode::INTERNAL_SERVER_ERROR,
}
}
fn error_response(&self) -> HttpResponse {
let mut buf = Vec::new();
match self {
Error::BranchNotFound => {
templates::p404_no_master(
&mut buf,
VERSION_INFO,
REPO_COUNT.load(Ordering::Relaxed),
)
.unwrap();
templates::p404_no_master(&mut buf, VERSION_INFO, 0).unwrap();
HttpResponse::NotFound().content_type("text/html").body(buf)
}
_ => {
templates::p500(&mut buf, VERSION_INFO, REPO_COUNT.load(Ordering::Relaxed))
.unwrap();
templates::p500(&mut buf, VERSION_INFO, 0).unwrap();
HttpResponse::InternalServerError()
.content_type("text/html")
.body(buf)

536
src/lib.rs Normal file
View File

@ -0,0 +1,536 @@
#![type_length_limit = "2257138"]
#[macro_use]
extern crate actix_web;
#[macro_use]
extern crate lazy_static;
#[macro_use]
extern crate serde_derive;
#[macro_use]
extern crate tracing;
mod cache;
pub mod config;
mod count;
mod error;
mod service;
mod statics;
pub mod telemetry;
mod template;
#[cfg(test)]
mod tests;
use crate::{
cache::CacheState,
config::Settings,
error::{Error, Result},
service::{Bitbucket, FormService, GitHub, Gitlab, Service},
statics::{CLIENT, CSS, FAVICON, VERSION_INFO},
template::RepoInfo,
};
use actix_web::{
dev::Server,
http::header::{CacheControl, CacheDirective, Expires, LOCATION},
middleware::{self, normalize::TrailingSlash},
web, App, HttpResponse, HttpServer, Responder,
};
use badge::{Badge, BadgeOptions};
use git2::{BranchType, Repository};
use number_prefix::NumberPrefix;
use std::{
borrow::Cow,
fs::create_dir_all,
io,
net::TcpListener,
path::Path,
process::Command,
sync::atomic::AtomicUsize,
sync::atomic::Ordering,
time::{Duration, SystemTime},
};
use tracing::Instrument;
include!(concat!(env!("OUT_DIR"), "/templates.rs"));
#[derive(Deserialize, Serialize)]
struct GeneratorForm<'a> {
service: FormService,
user: Cow<'a, str>,
repo: Cow<'a, str>,
}
#[derive(Debug)]
pub(crate) struct State {
settings: Settings,
// repos: String,
// cache: String,
// repos: settings.repodir.display().to_string(),
// cache: settings.cachedir.display().to_string(),
}
impl State {
fn repos(&self) -> String {
self.settings.repodir.display().to_string()
}
fn cache(&self) -> String {
self.settings.cachedir.display().to_string()
}
}
#[derive(Serialize)]
struct JsonResponse<'a> {
head: &'a str,
branch: &'a str,
count: u64,
commits: u64,
}
#[derive(Deserialize, Debug)]
struct BranchQuery {
branch: Option<String>,
}
fn pull(path: impl AsRef<Path>) -> Result<()> {
let repo = Repository::open_bare(path)?;
let mut origin = repo.find_remote("origin")?;
origin.fetch(&["refs/heads/*:refs/heads/*"], None, None)?;
Ok(())
}
fn hoc(repo: &str, repo_dir: &str, cache_dir: &str, branch: &str) -> Result<(u64, String, u64)> {
let repo_dir = format!("{}/{}", repo_dir, repo);
let cache_dir = format!("{}/{}.json", cache_dir, repo);
let cache_dir = Path::new(&cache_dir);
let repo = Repository::open_bare(&repo_dir)?;
// TODO: do better...
let head = repo
.find_branch(branch, BranchType::Local)
.map_err(|_| Error::BranchNotFound)?
.into_reference();
let head = format!("{}", head.target().ok_or(Error::BranchNotFound)?);
let mut arg_commit_count = vec!["rev-list".to_string(), "--count".to_string()];
let mut arg = vec![
"log".to_string(),
"--pretty=tformat:".to_string(),
"--numstat".to_string(),
"--ignore-space-change".to_string(),
"--ignore-all-space".to_string(),
"--ignore-submodules".to_string(),
"--no-color".to_string(),
"--find-copies-harder".to_string(),
"-M".to_string(),
"--diff-filter=ACDM".to_string(),
];
let cache = CacheState::read_from_file(&cache_dir, branch, &head)?;
match &cache {
CacheState::Current { count, commits, .. } => {
info!("Using cache");
return Ok((*count, head, *commits));
}
CacheState::Old { head, .. } => {
info!("Updating cache");
arg.push(format!("{}..{}", head, branch));
arg_commit_count.push(format!("{}..{}", head, branch));
}
CacheState::No | CacheState::NoneForBranch(..) => {
info!("Creating cache");
arg.push(branch.to_string());
arg_commit_count.push(branch.to_string());
}
};
arg.push("--".to_string());
arg.push(".".to_string());
let output = Command::new("git")
.args(&arg)
.current_dir(&repo_dir)
.output()?
.stdout;
let output = String::from_utf8_lossy(&output);
let output_commits = Command::new("git")
.args(&arg_commit_count)
.current_dir(&repo_dir)
.output()?
.stdout;
let output_commits = String::from_utf8_lossy(&output_commits);
let commits: u64 = output_commits.trim().parse()?;
let count: u64 = output
.lines()
.map(|s| {
s.split_whitespace()
.take(2)
.map(str::parse::<u64>)
.filter_map(std::result::Result::ok)
.sum::<u64>()
})
.sum();
let cache = cache.calculate_new_cache(count, commits, (&head).into(), branch);
cache.write_to_file(cache_dir)?;
Ok((count, head, commits))
}
async fn remote_exists(url: &str) -> Result<bool> {
let resp = CLIENT.head(url).send().await?;
Ok(resp.status() == reqwest::StatusCode::OK)
}
enum HocResult {
Hoc {
hoc: u64,
commits: u64,
hoc_pretty: String,
head: String,
url: String,
repo: String,
service_path: String,
},
NotFound,
}
async fn delete_repo_and_cache<T>(
state: web::Data<State>,
repo_count: web::Data<AtomicUsize>,
data: web::Path<(String, String)>,
) -> Result<impl Responder>
where
T: Service,
{
let data = data.into_inner();
let span = info_span!(
"deleting repository and cache",
service = T::domain(),
user = data.0.as_str(),
repo = data.1.as_str()
);
let future = async {
let repo = format!(
"{}/{}/{}",
T::domain(),
data.0.to_lowercase(),
data.1.to_lowercase()
);
info!("Deleting cache and repository");
let cache_dir = format!("{}/{}.json", &state.cache(), repo);
let repo_dir = format!("{}/{}", &state.repos(), repo);
std::fs::remove_file(&cache_dir).or_else(|e| {
if e.kind() == io::ErrorKind::NotFound {
Ok(())
} else {
Err(e)
}
})?;
std::fs::remove_dir_all(&repo_dir).or_else(|e| {
if e.kind() == io::ErrorKind::NotFound {
Ok(())
} else {
Err(e)
}
})?;
repo_count.fetch_sub(1, Ordering::Relaxed);
Ok(HttpResponse::TemporaryRedirect()
.header(
LOCATION,
format!("/{}/{}/{}/view", T::url_path(), data.0, data.1),
)
.finish())
};
future.instrument(span).await
}
async fn handle_hoc_request<T, F>(
state: web::Data<State>,
repo_count: web::Data<AtomicUsize>,
data: web::Path<(String, String)>,
branch: &str,
mapper: F,
) -> Result<HttpResponse>
where
T: Service,
F: FnOnce(HocResult) -> Result<HttpResponse>,
{
let data = data.into_inner();
let span = info_span!(
"handling hoc calculation",
service = T::domain(),
user = data.0.as_str(),
repo = data.1.as_str(),
branch
);
let future = async {
let repo = format!("{}/{}", data.0.to_lowercase(), data.1.to_lowercase());
let service_path = format!("{}/{}", T::url_path(), repo);
let service_url = format!("{}/{}", T::domain(), repo);
let path = format!("{}/{}", state.repos(), service_url);
let url = format!("https://{}", service_url);
let remote_exists = remote_exists(&url).await?;
let file = Path::new(&path);
if !file.exists() {
if !remote_exists {
warn!("Repository does not exist");
return mapper(HocResult::NotFound);
}
info!("Cloning for the first time");
create_dir_all(file)?;
let repo = Repository::init_bare(file)?;
repo.remote_add_fetch("origin", "refs/heads/*:refs/heads/*")?;
repo.remote_set_url("origin", &url)?;
repo_count.fetch_add(1, Ordering::Relaxed);
}
pull(&path)?;
let (hoc, head, commits) = hoc(&service_url, &state.repos(), &state.cache(), branch)?;
let hoc_pretty = match NumberPrefix::decimal(hoc as f64) {
NumberPrefix::Standalone(hoc) => hoc.to_string(),
NumberPrefix::Prefixed(prefix, hoc) => format!("{:.1}{}", hoc, prefix),
};
let res = HocResult::Hoc {
hoc,
commits,
hoc_pretty,
head,
url,
repo,
service_path,
};
mapper(res)
};
future.instrument(span).await
}
pub(crate) async fn json_hoc<T: Service>(
state: web::Data<State>,
repo_count: web::Data<AtomicUsize>,
data: web::Path<(String, String)>,
branch: web::Query<BranchQuery>,
) -> Result<HttpResponse> {
let branch = branch.branch.as_deref().unwrap_or("master");
let rc_clone = repo_count.clone();
let mapper = move |r| match r {
HocResult::NotFound => p404(rc_clone),
HocResult::Hoc {
hoc, head, commits, ..
} => Ok(HttpResponse::Ok().json(JsonResponse {
branch,
head: &head,
count: hoc,
commits,
})),
};
handle_hoc_request::<T, _>(state, repo_count, data, branch, mapper).await
}
fn no_cache_response(body: Vec<u8>) -> HttpResponse {
let expiration = SystemTime::now() + Duration::from_secs(30);
HttpResponse::Ok()
.content_type("image/svg+xml")
.set(Expires(expiration.into()))
.set(CacheControl(vec![
CacheDirective::MaxAge(0u32),
CacheDirective::MustRevalidate,
CacheDirective::NoCache,
CacheDirective::NoStore,
]))
.body(body)
}
pub(crate) async fn calculate_hoc<T: Service>(
state: web::Data<State>,
repo_count: web::Data<AtomicUsize>,
data: web::Path<(String, String)>,
branch: web::Query<BranchQuery>,
) -> HttpResponse {
let rc_clone = repo_count.clone();
let mapper = move |r| match r {
HocResult::NotFound => p404(rc_clone),
HocResult::Hoc { hoc_pretty, .. } => {
let badge_opt = BadgeOptions {
subject: "Hits-of-Code".to_string(),
color: "#007ec6".to_string(),
status: hoc_pretty,
};
let badge = Badge::new(badge_opt)?;
// TODO: remove clone
let body = badge.to_svg().as_bytes().to_vec();
Ok(no_cache_response(body))
}
};
let branch = branch.branch.as_deref().unwrap_or("master");
let error_badge = |_| {
let error_badge = Badge::new(BadgeOptions {
subject: "Hits-of-Code".to_string(),
color: "#ff0000".to_string(),
status: "error".to_string(),
})
.unwrap();
let body = error_badge.to_svg().as_bytes().to_vec();
no_cache_response(body)
};
handle_hoc_request::<T, _>(state, repo_count, data, branch, mapper)
.await
.unwrap_or_else(error_badge)
}
async fn overview<T: Service>(
state: web::Data<State>,
repo_count: web::Data<AtomicUsize>,
data: web::Path<(String, String)>,
branch: web::Query<BranchQuery>,
) -> Result<HttpResponse> {
let branch = branch.branch.as_deref().unwrap_or("master");
let base_url = state.settings.base_url.clone();
let rc_clone = repo_count.clone();
let mapper = move |r| match r {
HocResult::NotFound => p404(rc_clone),
HocResult::Hoc {
hoc,
commits,
hoc_pretty,
url,
head,
repo,
service_path,
} => {
let mut buf = Vec::new();
let repo_info = RepoInfo {
commit_url: &T::commit_url(&repo, &head),
commits,
base_url: &base_url,
head: &head,
hoc,
hoc_pretty: &hoc_pretty,
path: &service_path,
url: &url,
branch,
};
templates::overview(
&mut buf,
VERSION_INFO,
rc_clone.load(Ordering::Relaxed),
repo_info,
)?;
Ok(HttpResponse::Ok().content_type("text/html").body(buf))
}
};
handle_hoc_request::<T, _>(state, repo_count, data, branch, mapper).await
}
#[get("/health_check")]
async fn health_check() -> HttpResponse {
HttpResponse::Ok().finish()
}
#[get("/")]
async fn index(
state: web::Data<State>,
repo_count: web::Data<AtomicUsize>,
) -> Result<HttpResponse> {
let mut buf = Vec::new();
templates::index(
&mut buf,
VERSION_INFO,
repo_count.load(Ordering::Relaxed),
&state.settings.base_url,
)?;
Ok(HttpResponse::Ok().content_type("text/html").body(buf))
}
#[post("/generate")]
async fn generate(
params: web::Form<GeneratorForm<'_>>,
state: web::Data<State>,
repo_count: web::Data<AtomicUsize>,
) -> Result<HttpResponse> {
let repo = format!("{}/{}", params.user, params.repo);
let mut buf = Vec::new();
templates::generate(
&mut buf,
VERSION_INFO,
repo_count.load(Ordering::Relaxed),
&state.settings.base_url,
params.service.url(),
params.service.service(),
&repo,
)?;
Ok(HttpResponse::Ok().content_type("text/html").body(buf))
}
fn p404(repo_count: web::Data<AtomicUsize>) -> Result<HttpResponse> {
let mut buf = Vec::new();
templates::p404(&mut buf, VERSION_INFO, repo_count.load(Ordering::Relaxed))?;
Ok(HttpResponse::NotFound().content_type("text/html").body(buf))
}
async fn async_p404(repo_count: web::Data<AtomicUsize>) -> Result<HttpResponse> {
p404(repo_count)
}
fn css() -> HttpResponse {
HttpResponse::Ok().content_type("text/css").body(CSS)
}
fn favicon32() -> HttpResponse {
HttpResponse::Ok().content_type("image/png").body(FAVICON)
}
async fn start_server(listener: TcpListener, settings: Settings) -> std::io::Result<Server> {
let workers = settings.workers;
let repo_count =
// TODO: errorhandling
web::Data::new(AtomicUsize::new(count::count_repositories(&settings.repodir).unwrap()));
let state = web::Data::new(State {
settings
// repos: settings.repodir.display().to_string(),
// cache: settings.cachedir.display().to_string(),
});
Ok(HttpServer::new(move || {
App::new()
.app_data(state.clone())
.app_data(repo_count.clone())
.wrap(tracing_actix_web::TracingLogger)
.wrap(middleware::NormalizePath::new(TrailingSlash::Trim))
.service(index)
.service(health_check)
.service(web::resource("/tacit-css.min.css").route(web::get().to(css)))
.service(web::resource("/favicon.ico").route(web::get().to(favicon32)))
.service(generate)
.service(web::resource("/github/{user}/{repo}").to(calculate_hoc::<GitHub>))
.service(web::resource("/gitlab/{user}/{repo}").to(calculate_hoc::<Gitlab>))
.service(web::resource("/bitbucket/{user}/{repo}").to(calculate_hoc::<Bitbucket>))
.service(
web::resource("/github/{user}/{repo}/delete")
.route(web::post().to(delete_repo_and_cache::<GitHub>)),
)
.service(
web::resource("/gitlab/{user}/{repo}/delete")
.route(web::post().to(delete_repo_and_cache::<Gitlab>)),
)
.service(
web::resource("/bitbucket/{user}/{repo}/delete")
.route(web::post().to(delete_repo_and_cache::<Bitbucket>)),
)
.service(web::resource("/github/{user}/{repo}/json").to(json_hoc::<GitHub>))
.service(web::resource("/gitlab/{user}/{repo}/json").to(json_hoc::<Gitlab>))
.service(web::resource("/bitbucket/{user}/{repo}/json").to(json_hoc::<Bitbucket>))
.service(web::resource("/view/github/{user}/{repo}").to(overview::<GitHub>))
.service(web::resource("/view/gitlab/{user}/{repo}").to(overview::<Gitlab>))
.service(web::resource("/view/bitbucket/{user}/{repo}").to(overview::<Bitbucket>))
.service(web::resource("/github/{user}/{repo}/view").to(overview::<GitHub>))
.service(web::resource("/gitlab/{user}/{repo}/view").to(overview::<Gitlab>))
.service(web::resource("/bitbucket/{user}/{repo}/view").to(overview::<Bitbucket>))
.default_service(web::resource("").route(web::get().to(async_p404)))
})
.workers(workers)
.listen(listener)?
.run())
}
pub async fn run(listener: TcpListener, settings: Settings) -> std::io::Result<Server> {
let span = info_span!("hoc", version = env!("CARGO_PKG_VERSION"));
let _ = span.enter();
start_server(listener, settings).instrument(span).await
}

505
src/main.rs
View File

@ -1,495 +1,26 @@
#![type_length_limit = "2257138"]
use hoc::{config::Settings, telemetry};
#[macro_use]
extern crate actix_web;
#[macro_use]
extern crate lazy_static;
#[macro_use]
extern crate serde_derive;
#[macro_use]
extern crate tracing;
use std::net::TcpListener;
mod cache;
mod config;
mod count;
mod error;
mod service;
mod statics;
mod template;
fn init() {
dotenv::dotenv().ok();
openssl_probe::init_ssl_cert_env_vars();
#[cfg(test)]
mod tests;
use crate::{
cache::CacheState,
error::{Error, Result},
service::{Bitbucket, FormService, GitHub, Gitlab, Service},
statics::{CLIENT, CSS, FAVICON, OPT, REPO_COUNT, VERSION_INFO},
template::RepoInfo,
};
use actix_web::{
http::header::{CacheControl, CacheDirective, Expires, LOCATION},
middleware::{self, normalize::TrailingSlash},
web, App, HttpResponse, HttpServer, Responder,
};
use badge::{Badge, BadgeOptions};
use git2::{BranchType, Repository};
use number_prefix::NumberPrefix;
use std::{
borrow::Cow,
fs::create_dir_all,
io,
path::Path,
process::Command,
sync::atomic::Ordering,
sync::Arc,
time::{Duration, SystemTime},
};
use tracing::Instrument;
include!(concat!(env!("OUT_DIR"), "/templates.rs"));
#[derive(Deserialize, Serialize)]
struct GeneratorForm<'a> {
service: FormService,
user: Cow<'a, str>,
repo: Cow<'a, str>,
}
#[derive(Debug)]
pub(crate) struct State {
repos: String,
cache: String,
}
#[derive(Serialize)]
struct JsonResponse<'a> {
head: &'a str,
branch: &'a str,
count: u64,
commits: u64,
}
#[derive(Deserialize, Debug)]
struct BranchQuery {
branch: Option<String>,
}
fn pull(path: impl AsRef<Path>) -> Result<()> {
let repo = Repository::open_bare(path)?;
let mut origin = repo.find_remote("origin")?;
origin.fetch(&["refs/heads/*:refs/heads/*"], None, None)?;
Ok(())
}
fn hoc(repo: &str, repo_dir: &str, cache_dir: &str, branch: &str) -> Result<(u64, String, u64)> {
let repo_dir = format!("{}/{}", repo_dir, repo);
let cache_dir = format!("{}/{}.json", cache_dir, repo);
let cache_dir = Path::new(&cache_dir);
let repo = Repository::open_bare(&repo_dir)?;
// TODO: do better...
let head = repo
.find_branch(branch, BranchType::Local)
.map_err(|_| Error::BranchNotFound)?
.into_reference();
let head = format!("{}", head.target().ok_or(Error::BranchNotFound)?);
let mut arg_commit_count = vec!["rev-list".to_string(), "--count".to_string()];
let mut arg = vec![
"log".to_string(),
"--pretty=tformat:".to_string(),
"--numstat".to_string(),
"--ignore-space-change".to_string(),
"--ignore-all-space".to_string(),
"--ignore-submodules".to_string(),
"--no-color".to_string(),
"--find-copies-harder".to_string(),
"-M".to_string(),
"--diff-filter=ACDM".to_string(),
];
let cache = CacheState::read_from_file(&cache_dir, branch, &head)?;
match &cache {
CacheState::Current { count, commits, .. } => {
info!("Using cache");
return Ok((*count, head, *commits));
}
CacheState::Old { head, .. } => {
info!("Updating cache");
arg.push(format!("{}..{}", head, branch));
arg_commit_count.push(format!("{}..{}", head, branch));
}
CacheState::No | CacheState::NoneForBranch(..) => {
info!("Creating cache");
arg.push(branch.to_string());
arg_commit_count.push(branch.to_string());
}
};
arg.push("--".to_string());
arg.push(".".to_string());
let output = Command::new("git")
.args(&arg)
.current_dir(&repo_dir)
.output()?
.stdout;
let output = String::from_utf8_lossy(&output);
let output_commits = Command::new("git")
.args(&arg_commit_count)
.current_dir(&repo_dir)
.output()?
.stdout;
let output_commits = String::from_utf8_lossy(&output_commits);
let commits: u64 = output_commits.trim().parse()?;
let count: u64 = output
.lines()
.map(|s| {
s.split_whitespace()
.take(2)
.map(str::parse::<u64>)
.filter_map(std::result::Result::ok)
.sum::<u64>()
})
.sum();
let cache = cache.calculate_new_cache(count, commits, (&head).into(), branch);
cache.write_to_file(cache_dir)?;
Ok((count, head, commits))
}
async fn remote_exists(url: &str) -> Result<bool> {
let resp = CLIENT.head(url).send().await?;
Ok(resp.status() == reqwest::StatusCode::OK)
}
enum HocResult {
Hoc {
hoc: u64,
commits: u64,
hoc_pretty: String,
head: String,
url: String,
repo: String,
service_path: String,
},
NotFound,
}
async fn delete_repo_and_cache<T>(
state: web::Data<Arc<State>>,
data: web::Path<(String, String)>,
) -> Result<impl Responder>
where
T: Service,
{
let data = data.into_inner();
let span = info_span!(
"deleting repository and cache",
service = T::domain(),
user = data.0.as_str(),
repo = data.1.as_str()
);
let future = async {
let repo = format!(
"{}/{}/{}",
T::domain(),
data.0.to_lowercase(),
data.1.to_lowercase()
);
info!("Deleting cache and repository");
let cache_dir = format!("{}/{}.json", &state.cache, repo);
let repo_dir = format!("{}/{}", &state.repos, repo);
std::fs::remove_file(&cache_dir).or_else(|e| {
if e.kind() == io::ErrorKind::NotFound {
Ok(())
} else {
Err(e)
}
})?;
std::fs::remove_dir_all(&repo_dir).or_else(|e| {
if e.kind() == io::ErrorKind::NotFound {
Ok(())
} else {
Err(e)
}
})?;
REPO_COUNT.fetch_sub(1, Ordering::Relaxed);
Ok(HttpResponse::TemporaryRedirect()
.header(
LOCATION,
format!("/{}/{}/{}/view", T::url_path(), data.0, data.1),
)
.finish())
};
future.instrument(span).await
}
async fn handle_hoc_request<T, F>(
state: web::Data<Arc<State>>,
data: web::Path<(String, String)>,
branch: &str,
mapper: F,
) -> Result<HttpResponse>
where
T: Service,
F: Fn(HocResult) -> Result<HttpResponse>,
{
let data = data.into_inner();
let span = info_span!(
"handling hoc calculation",
service = T::domain(),
user = data.0.as_str(),
repo = data.1.as_str(),
branch
);
let future = async {
let repo = format!("{}/{}", data.0.to_lowercase(), data.1.to_lowercase());
let service_path = format!("{}/{}", T::url_path(), repo);
let service_url = format!("{}/{}", T::domain(), repo);
let path = format!("{}/{}", state.repos, service_url);
let url = format!("https://{}", service_url);
let remote_exists = remote_exists(&url).await?;
let file = Path::new(&path);
if !file.exists() {
if !remote_exists {
warn!("Repository does not exist");
return mapper(HocResult::NotFound);
}
info!("Cloning for the first time");
create_dir_all(file)?;
let repo = Repository::init_bare(file)?;
repo.remote_add_fetch("origin", "refs/heads/*:refs/heads/*")?;
repo.remote_set_url("origin", &url)?;
REPO_COUNT.fetch_add(1, Ordering::Relaxed);
}
pull(&path)?;
let (hoc, head, commits) = hoc(&service_url, &state.repos, &state.cache, branch)?;
let hoc_pretty = match NumberPrefix::decimal(hoc as f64) {
NumberPrefix::Standalone(hoc) => hoc.to_string(),
NumberPrefix::Prefixed(prefix, hoc) => format!("{:.1}{}", hoc, prefix),
};
let res = HocResult::Hoc {
hoc,
commits,
hoc_pretty,
head,
url,
repo,
service_path,
};
mapper(res)
};
future.instrument(span).await
}
pub(crate) async fn json_hoc<T: Service>(
state: web::Data<Arc<State>>,
data: web::Path<(String, String)>,
branch: web::Query<BranchQuery>,
) -> Result<HttpResponse> {
let branch = branch.branch.as_deref().unwrap_or("master");
let mapper = |r| match r {
HocResult::NotFound => p404(),
HocResult::Hoc {
hoc, head, commits, ..
} => Ok(HttpResponse::Ok().json(JsonResponse {
branch,
head: &head,
count: hoc,
commits,
})),
};
handle_hoc_request::<T, _>(state, data, branch, mapper).await
}
fn no_cache_response(body: Vec<u8>) -> HttpResponse {
let expiration = SystemTime::now() + Duration::from_secs(30);
HttpResponse::Ok()
.content_type("image/svg+xml")
.set(Expires(expiration.into()))
.set(CacheControl(vec![
CacheDirective::MaxAge(0u32),
CacheDirective::MustRevalidate,
CacheDirective::NoCache,
CacheDirective::NoStore,
]))
.body(body)
}
pub(crate) async fn calculate_hoc<T: Service>(
state: web::Data<Arc<State>>,
data: web::Path<(String, String)>,
branch: web::Query<BranchQuery>,
) -> HttpResponse {
let mapper = move |r| match r {
HocResult::NotFound => p404(),
HocResult::Hoc { hoc_pretty, .. } => {
let badge_opt = BadgeOptions {
subject: "Hits-of-Code".to_string(),
color: "#007ec6".to_string(),
status: hoc_pretty,
};
let badge = Badge::new(badge_opt)?;
// TODO: remove clone
let body = badge.to_svg().as_bytes().to_vec();
Ok(no_cache_response(body))
}
};
let branch = branch.branch.as_deref().unwrap_or("master");
let error_badge = |_| {
let error_badge = Badge::new(BadgeOptions {
subject: "Hits-of-Code".to_string(),
color: "#ff0000".to_string(),
status: "error".to_string(),
})
.unwrap();
let body = error_badge.to_svg().as_bytes().to_vec();
no_cache_response(body)
};
handle_hoc_request::<T, _>(state, data, branch, mapper)
.await
.unwrap_or_else(error_badge)
}
async fn overview<T: Service>(
state: web::Data<Arc<State>>,
data: web::Path<(String, String)>,
branch: web::Query<BranchQuery>,
) -> Result<HttpResponse> {
let branch = branch.branch.as_deref().unwrap_or("master");
let mapper = |r| match r {
HocResult::NotFound => p404(),
HocResult::Hoc {
hoc,
commits,
hoc_pretty,
url,
head,
repo,
service_path,
} => {
let mut buf = Vec::new();
let repo_info = RepoInfo {
commit_url: &T::commit_url(&repo, &head),
commits,
base_url: &OPT.base_url,
head: &head,
hoc,
hoc_pretty: &hoc_pretty,
path: &service_path,
url: &url,
branch,
};
templates::overview(
&mut buf,
VERSION_INFO,
REPO_COUNT.load(Ordering::Relaxed),
repo_info,
)?;
Ok(HttpResponse::Ok().content_type("text/html").body(buf))
}
};
handle_hoc_request::<T, _>(state, data, branch, mapper).await
}
#[get("/")]
async fn index() -> Result<HttpResponse> {
let mut buf = Vec::new();
templates::index(
&mut buf,
VERSION_INFO,
REPO_COUNT.load(Ordering::Relaxed),
&OPT.base_url,
)?;
Ok(HttpResponse::Ok().content_type("text/html").body(buf))
}
#[post("/generate")]
async fn generate(params: web::Form<GeneratorForm<'_>>) -> Result<HttpResponse> {
let repo = format!("{}/{}", params.user, params.repo);
let mut buf = Vec::new();
templates::generate(
&mut buf,
VERSION_INFO,
REPO_COUNT.load(Ordering::Relaxed),
&OPT.base_url,
params.service.url(),
params.service.service(),
&repo,
)?;
Ok(HttpResponse::Ok().content_type("text/html").body(buf))
}
fn p404() -> Result<HttpResponse> {
let mut buf = Vec::new();
templates::p404(&mut buf, VERSION_INFO, REPO_COUNT.load(Ordering::Relaxed))?;
Ok(HttpResponse::NotFound().content_type("text/html").body(buf))
}
async fn async_p404() -> Result<HttpResponse> {
p404()
}
fn css() -> HttpResponse {
HttpResponse::Ok().content_type("text/css").body(CSS)
}
fn favicon32() -> HttpResponse {
HttpResponse::Ok().content_type("image/png").body(FAVICON)
}
async fn start_server() -> std::io::Result<()> {
let interface = format!("{}:{}", OPT.host, OPT.port);
let state = Arc::new(State {
repos: OPT.repodir.display().to_string(),
cache: OPT.cachedir.display().to_string(),
});
HttpServer::new(move || {
App::new()
.data(state.clone())
.wrap(tracing_actix_web::TracingLogger)
.wrap(middleware::NormalizePath::new(TrailingSlash::Trim))
.service(index)
.service(web::resource("/tacit-css.min.css").route(web::get().to(css)))
.service(web::resource("/favicon.ico").route(web::get().to(favicon32)))
.service(generate)
.service(web::resource("/github/{user}/{repo}").to(calculate_hoc::<GitHub>))
.service(web::resource("/gitlab/{user}/{repo}").to(calculate_hoc::<Gitlab>))
.service(web::resource("/bitbucket/{user}/{repo}").to(calculate_hoc::<Bitbucket>))
.service(
web::resource("/github/{user}/{repo}/delete")
.route(web::post().to(delete_repo_and_cache::<GitHub>)),
)
.service(
web::resource("/gitlab/{user}/{repo}/delete")
.route(web::post().to(delete_repo_and_cache::<Gitlab>)),
)
.service(
web::resource("/bitbucket/{user}/{repo}/delete")
.route(web::post().to(delete_repo_and_cache::<Bitbucket>)),
)
.service(web::resource("/github/{user}/{repo}/json").to(json_hoc::<GitHub>))
.service(web::resource("/gitlab/{user}/{repo}/json").to(json_hoc::<Gitlab>))
.service(web::resource("/bitbucket/{user}/{repo}/json").to(json_hoc::<Bitbucket>))
.service(web::resource("/view/github/{user}/{repo}").to(overview::<GitHub>))
.service(web::resource("/view/gitlab/{user}/{repo}").to(overview::<Gitlab>))
.service(web::resource("/view/bitbucket/{user}/{repo}").to(overview::<Bitbucket>))
.service(web::resource("/github/{user}/{repo}/view").to(overview::<GitHub>))
.service(web::resource("/gitlab/{user}/{repo}/view").to(overview::<Gitlab>))
.service(web::resource("/bitbucket/{user}/{repo}/view").to(overview::<Bitbucket>))
.default_service(web::resource("").route(web::get().to(async_p404)))
})
.workers(OPT.workers)
.bind(interface)?
.run()
.await
telemetry::init_subscriber(telemetry::get_subscriber("hoc", "info"))
}
#[actix_rt::main]
async fn main() -> std::io::Result<()> {
config::init();
let span = info_span!("hoc", version = env!("CARGO_PKG_VERSION"));
let _ = span.enter();
start_server().instrument(span).await
init();
// TODO: error handling
let settings = Settings::load().expect("Cannot load config");
let address = format!("{}:{}", settings.host, settings.port);
// TODO: error handling
let listener = TcpListener::bind(address)?;
hoc::run(listener, settings)
.await
.expect("Server error")
.await
}

6
src/statics.rs
View File

@ -1,6 +1,3 @@
use crate::{config::Settings, count::count_repositories};
use std::sync::atomic::AtomicUsize;
pub struct VersionInfo<'a> {
pub commit: &'a str,
pub version: &'a str,
@ -15,7 +12,4 @@ pub(crate) const FAVICON: &[u8] = include_bytes!("../static/favicon32.png");
lazy_static! {
pub(crate) static ref CLIENT: reqwest::Client = reqwest::Client::new();
pub(crate) static ref OPT: Settings = Settings::new().unwrap();
pub(crate) static ref REPO_COUNT: AtomicUsize =
AtomicUsize::new(count_repositories(&OPT.repodir).unwrap());
}

21
src/telemetry.rs Normal file
View File

@ -0,0 +1,21 @@
use tracing::{subscriber::set_global_default, Subscriber};
use tracing_bunyan_formatter::{BunyanFormattingLayer, JsonStorageLayer};
use tracing_log::LogTracer;
use tracing_subscriber::{layer::SubscriberExt, EnvFilter, Registry};
pub fn get_subscriber(name: &str, env_filter: &str) -> impl Subscriber + Send + Sync {
let env_filter =
EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(env_filter));
let formatting_layer = BunyanFormattingLayer::new(name.to_string(), std::io::stdout);
Registry::default()
.with(env_filter)
.with(JsonStorageLayer)
.with(formatting_layer)
}
pub fn init_subscriber(subscriber: impl Subscriber + Send + Sync) {
LogTracer::init().expect("Failed to set logger");
set_global_default(subscriber).expect("Failed to set tracing subscriber");
}

84
src/tests.rs
View File

@ -16,53 +16,53 @@ macro_rules! test_app {
};
}
macro_rules! test_service {
($name: ident, $path: tt, $what: ident) => {
async fn $name<T: 'static + Service>(req_path: &str) {
let repo_dir = dbg!(tempdir().unwrap());
let cache_dir = dbg!(tempdir().unwrap());
let repos = format!("{}/", repo_dir.path().display());
let cache = format!("{}/", cache_dir.path().display());
let state = dbg!(State { repos, cache });
// macro_rules! test_service {
// ($name: ident, $path: tt, $what: ident) => {
// async fn $name<T: 'static + Service>(req_path: &str) {
// let repo_dir = dbg!(tempdir().unwrap());
// let cache_dir = dbg!(tempdir().unwrap());
// let repos = format!("{}/", repo_dir.path().display());
// let cache = format!("{}/", cache_dir.path().display());
// let state = dbg!(State { repos, cache });
let mut app = test_app!(state, web::resource($path).to($what::<T>));
// let mut app = test_app!(state, web::resource($path).to($what::<T>));
let req = dbg!(test::TestRequest::with_uri(req_path).to_request());
let resp = dbg!(test::call_service(&mut app, req).await);
// let req = dbg!(test::TestRequest::with_uri(req_path).to_request());
// let resp = dbg!(test::call_service(&mut app, req).await);
assert_eq!(resp.status(), http::StatusCode::OK);
}
};
}
// assert_eq!(resp.status(), http::StatusCode::OK);
// }
// };
// }
#[actix_rt::test]
async fn test_index() {
std::env::set_var("HOC_BASE_URL", "http://0.0.0.0:8080");
let mut app = test_app!(index);
let req = dbg!(test::TestRequest::with_uri("/").to_request());
let resp = dbg!(test::call_service(&mut app, req).await);
assert_eq!(resp.status(), http::StatusCode::OK);
}
// TODO: fix this test
// #[actix_rt::test]
async fn test_json() {
test_service!(test_json_service, "/service/{user}/{repo}/json", json_hoc);
// async fn test_index() {
// std::env::set_var("HOC_BASE_URL", "http://0.0.0.0:8080");
test_json_service::<Gitlab>("/service/vbrandl/hoc/json").await;
test_json_service::<GitHub>("/service/vbrandl/hoc/json").await;
test_json_service::<Bitbucket>("/service/vbrandl/hoc/json").await;
}
// let mut app = test_app!(index);
// TODO: fix this test
// #[actix_rt::test]
async fn test_badge() {
test_service!(test_badge_service, "/service/{user}/{repo}", calculate_hoc);
// let req = dbg!(test::TestRequest::with_uri("/").to_request());
// let resp = dbg!(test::call_service(&mut app, req).await);
test_badge_service::<Gitlab>("/service/vbrandl/hoc").await;
test_badge_service::<GitHub>("/service/vbrandl/hoc").await;
test_badge_service::<Bitbucket>("/service/vbrandl/hoc").await;
}
// assert_eq!(resp.status(), http::StatusCode::OK);
// }
// // TODO: fix this test
// // #[actix_rt::test]
// async fn test_json() {
// test_service!(test_json_service, "/service/{user}/{repo}/json", json_hoc);
// test_json_service::<Gitlab>("/service/vbrandl/hoc/json").await;
// test_json_service::<GitHub>("/service/vbrandl/hoc/json").await;
// test_json_service::<Bitbucket>("/service/vbrandl/hoc/json").await;
// }
// // TODO: fix this test
// // #[actix_rt::test]
// async fn test_badge() {
// test_service!(test_badge_service, "/service/{user}/{repo}", calculate_hoc);
// test_badge_service::<Gitlab>("/service/vbrandl/hoc").await;
// test_badge_service::<GitHub>("/service/vbrandl/hoc").await;
// test_badge_service::<Bitbucket>("/service/vbrandl/hoc").await;
// }

18
tests/health_check.rs Normal file
View File

@ -0,0 +1,18 @@
mod util;
use actix_web::client;
#[actix_rt::test]
async fn health_check_works() {
let test_app = util::spawn_app().await;
let client = client::Client::default();
let response = client
.get(&format!("{}/health_check", test_app.address))
.send()
.await
.expect("Failed to execute request");
assert!(response.status().is_success());
}

18
tests/index.rs Normal file
View File

@ -0,0 +1,18 @@
mod util;
use actix_web::client;
#[actix_rt::test]
async fn index_returns_success() {
let test_app = util::spawn_app().await;
let client = client::Client::default();
let response = client
.get(&format!("{}/", test_app.address))
.send()
.await
.expect("Failed to execute request");
assert!(response.status().is_success());
}

18
tests/json.rs Normal file
View File

@ -0,0 +1,18 @@
mod util;
use actix_web::client;
#[actix_rt::test]
async fn json_returns_success() {
let test_app = util::spawn_app().await;
let client = client::Client::default();
let response = client
.get(&format!("{}/github/vbrandl/hoc/json", test_app.address))
.send()
.await
.expect("Failed to execute request");
assert!(response.status().is_success());
}

50
tests/util/mod.rs Normal file
View File

@ -0,0 +1,50 @@
use hoc::{config::Settings, telemetry};
use std::net::TcpListener;
use tempfile::{tempdir, TempDir};
lazy_static::lazy_static! {
static ref TRACING: () = {
let filter = if std::env::var("TEST_LOG").is_ok() { "debug" } else { "" };
let subscriber = telemetry::get_subscriber("test", filter);
telemetry::init_subscriber(subscriber);
};
}
pub struct TestApp {
pub address: String,
repo_dir: TempDir,
cache_dir: TempDir,
}
pub async fn spawn_app() -> TestApp {
lazy_static::initialize(&TRACING);
let listener = TcpListener::bind("127.0.0.1:0").expect("Failed to bind random port");
let port = listener.local_addr().unwrap().port();
let address = format!("http://127.0.0.1:{}", port);
let repo_dir = tempdir().expect("Cannot create repo_dir");
let cache_dir = tempdir().expect("Cannot create cache_dir");
let mut settings = Settings::load().expect("Failed to read configuration.");
settings.repodir = repo_dir.path().to_path_buf();
settings.cachedir = cache_dir.path().to_path_buf();
// configuration.database.database_name = Uuid::new_v4().to_string();
// let connection_pool = configure_database(&configuration.database).await;
let server = hoc::run(listener, settings)
.await
.expect("Failed to bind address");
let _ = tokio::spawn(server);
TestApp {
address,
repo_dir,
cache_dir,
}
}