From 3486d44bc5cbce9233ddee31730f210fde6837d9 Mon Sep 17 00:00:00 2001 From: Valentin Brandl Date: Mon, 22 Mar 2021 12:46:10 +0100 Subject: [PATCH] Restructure for testability and implement first tests --- Cargo.lock | 34 ++- Cargo.toml | 13 +- src/config.rs | 13 +- src/error.rs | 26 +- src/lib.rs | 536 ++++++++++++++++++++++++++++++++++++++++++ src/main.rs | 505 ++------------------------------------- src/statics.rs | 6 - src/telemetry.rs | 21 ++ src/tests.rs | 84 +++---- tests/health_check.rs | 18 ++ tests/index.rs | 18 ++ tests/json.rs | 18 ++ tests/util/mod.rs | 50 ++++ 13 files changed, 782 insertions(+), 560 deletions(-) create mode 100644 src/lib.rs create mode 100644 src/telemetry.rs create mode 100644 tests/health_check.rs create mode 100644 tests/index.rs create mode 100644 tests/json.rs create mode 100644 tests/util/mod.rs diff --git a/Cargo.lock b/Cargo.lock index f332231..2568a78 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -817,6 +817,16 @@ dependencies = [ "version_check", ] +[[package]] +name = "gethostname" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e692e296bfac1d2533ef168d0b60ff5897b8b70a4009276834014dd8924cc028" +dependencies = [ + "libc", + "winapi 0.3.9", +] + [[package]] name = "getrandom" version = "0.1.16" @@ -937,9 +947,12 @@ dependencies = [ "serde_derive", "serde_json", "tempfile", + "tokio", "tracing", "tracing-actix-web", + "tracing-bunyan-formatter", "tracing-futures", + "tracing-log", "tracing-subscriber", "vergen", ] @@ -2339,6 +2352,23 @@ dependencies = [ "syn", ] +[[package]] +name = "tracing-bunyan-formatter" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06718867c20ea03700d41a9413610cccf5d772caea792f34cc73cdd43f0e14a6" +dependencies = [ + "chrono", + "gethostname", + "log", + "serde 1.0.123", + "serde_json", + "tracing", + "tracing-core", + "tracing-log", + "tracing-subscriber", +] + [[package]] name = "tracing-core" version = "0.1.17" @@ -2381,9 +2411,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ab8966ac3ca27126141f7999361cc97dd6fb4b71da04c02044fa9045d98bb96" +checksum = "705096c6f83bf68ea5d357a6aa01829ddbdac531b357b45abeca842938085baa" dependencies = [ "ansi_term", "chrono", diff --git a/Cargo.toml b/Cargo.toml index 91bff33..790dc27 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,13 @@ authors = ["Valentin Brandl "] edition = "2018" build = "build.rs" +[lib] +path = "src/lib.rs" + +[[bin]] +path = "src/main.rs" +name = "hoc" + [dependencies] actix-rt = "1.1.1" actix-web = "3.3.2" @@ -23,12 +30,16 @@ serde_derive = "1.0.123" serde_json = "1.0.63" tracing = "0.1.25" tracing-actix-web = "0.2.1" +tracing-bunyan-formatter = "0.1.7" tracing-futures = "0.2.5" -tracing-subscriber = "0.2.16" +tracing-log = "0.1.2" +tracing-subscriber = { version = "0.2.17", features = ["registry", "env-filter"] } [build-dependencies] ructe = "0.13.0" vergen = { version = "4.1.0", default-features = false, features = ["git"] } [dev-dependencies] +ructe = "0.13.0" tempfile = "3.2.0" +tokio = "0.2.25" diff --git a/src/config.rs b/src/config.rs index feab98b..1aa0671 100644 --- a/src/config.rs +++ b/src/config.rs @@ -17,16 +17,13 @@ pub struct Settings { pub workers: usize, } -pub(crate) fn init() { - dotenv::dotenv().ok(); - std::env::set_var("RUST_LOG", "actix_web=info,hoc=info"); - openssl_probe::init_ssl_cert_env_vars(); - - tracing_subscriber::fmt().init(); -} - impl Settings { + #[deprecated] pub fn new() -> Result { + Self::load() + } + + pub fn load() -> Result { let mut config = Config::new(); config .merge(File::with_name("hoc.toml").required(false))? diff --git a/src/error.rs b/src/error.rs index afe0a63..42b7eaf 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,9 +1,6 @@ -use crate::{ - statics::{REPO_COUNT, VERSION_INFO}, - templates, -}; -use actix_web::{HttpResponse, ResponseError}; -use std::{fmt, sync::atomic::Ordering}; +use crate::{statics::VERSION_INFO, templates}; +use actix_web::{http::StatusCode, HttpResponse, ResponseError}; +use std::fmt; pub(crate) type Result = std::result::Result; @@ -35,21 +32,22 @@ impl fmt::Display for Error { } impl ResponseError for Error { + fn status_code(&self) -> StatusCode { + match self { + Error::BranchNotFound => StatusCode::NOT_FOUND, + _ => StatusCode::INTERNAL_SERVER_ERROR, + } + } + fn error_response(&self) -> HttpResponse { let mut buf = Vec::new(); match self { Error::BranchNotFound => { - templates::p404_no_master( - &mut buf, - VERSION_INFO, - REPO_COUNT.load(Ordering::Relaxed), - ) - .unwrap(); + templates::p404_no_master(&mut buf, VERSION_INFO, 0).unwrap(); HttpResponse::NotFound().content_type("text/html").body(buf) } _ => { - templates::p500(&mut buf, VERSION_INFO, REPO_COUNT.load(Ordering::Relaxed)) - .unwrap(); + templates::p500(&mut buf, VERSION_INFO, 0).unwrap(); HttpResponse::InternalServerError() .content_type("text/html") .body(buf) diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..82a5a28 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,536 @@ +#![type_length_limit = "2257138"] + +#[macro_use] +extern crate actix_web; +#[macro_use] +extern crate lazy_static; +#[macro_use] +extern crate serde_derive; +#[macro_use] +extern crate tracing; + +mod cache; +pub mod config; +mod count; +mod error; +mod service; +mod statics; +pub mod telemetry; +mod template; + +#[cfg(test)] +mod tests; + +use crate::{ + cache::CacheState, + config::Settings, + error::{Error, Result}, + service::{Bitbucket, FormService, GitHub, Gitlab, Service}, + statics::{CLIENT, CSS, FAVICON, VERSION_INFO}, + template::RepoInfo, +}; +use actix_web::{ + dev::Server, + http::header::{CacheControl, CacheDirective, Expires, LOCATION}, + middleware::{self, normalize::TrailingSlash}, + web, App, HttpResponse, HttpServer, Responder, +}; +use badge::{Badge, BadgeOptions}; +use git2::{BranchType, Repository}; +use number_prefix::NumberPrefix; +use std::{ + borrow::Cow, + fs::create_dir_all, + io, + net::TcpListener, + path::Path, + process::Command, + sync::atomic::AtomicUsize, + sync::atomic::Ordering, + time::{Duration, SystemTime}, +}; +use tracing::Instrument; + +include!(concat!(env!("OUT_DIR"), "/templates.rs")); + +#[derive(Deserialize, Serialize)] +struct GeneratorForm<'a> { + service: FormService, + user: Cow<'a, str>, + repo: Cow<'a, str>, +} + +#[derive(Debug)] +pub(crate) struct State { + settings: Settings, + // repos: String, + // cache: String, + // repos: settings.repodir.display().to_string(), + // cache: settings.cachedir.display().to_string(), +} + +impl State { + fn repos(&self) -> String { + self.settings.repodir.display().to_string() + } + + fn cache(&self) -> String { + self.settings.cachedir.display().to_string() + } +} + +#[derive(Serialize)] +struct JsonResponse<'a> { + head: &'a str, + branch: &'a str, + count: u64, + commits: u64, +} + +#[derive(Deserialize, Debug)] +struct BranchQuery { + branch: Option, +} + +fn pull(path: impl AsRef) -> Result<()> { + let repo = Repository::open_bare(path)?; + let mut origin = repo.find_remote("origin")?; + origin.fetch(&["refs/heads/*:refs/heads/*"], None, None)?; + Ok(()) +} + +fn hoc(repo: &str, repo_dir: &str, cache_dir: &str, branch: &str) -> Result<(u64, String, u64)> { + let repo_dir = format!("{}/{}", repo_dir, repo); + let cache_dir = format!("{}/{}.json", cache_dir, repo); + let cache_dir = Path::new(&cache_dir); + let repo = Repository::open_bare(&repo_dir)?; + // TODO: do better... + let head = repo + .find_branch(branch, BranchType::Local) + .map_err(|_| Error::BranchNotFound)? + .into_reference(); + let head = format!("{}", head.target().ok_or(Error::BranchNotFound)?); + let mut arg_commit_count = vec!["rev-list".to_string(), "--count".to_string()]; + let mut arg = vec![ + "log".to_string(), + "--pretty=tformat:".to_string(), + "--numstat".to_string(), + "--ignore-space-change".to_string(), + "--ignore-all-space".to_string(), + "--ignore-submodules".to_string(), + "--no-color".to_string(), + "--find-copies-harder".to_string(), + "-M".to_string(), + "--diff-filter=ACDM".to_string(), + ]; + let cache = CacheState::read_from_file(&cache_dir, branch, &head)?; + match &cache { + CacheState::Current { count, commits, .. } => { + info!("Using cache"); + return Ok((*count, head, *commits)); + } + CacheState::Old { head, .. } => { + info!("Updating cache"); + arg.push(format!("{}..{}", head, branch)); + arg_commit_count.push(format!("{}..{}", head, branch)); + } + CacheState::No | CacheState::NoneForBranch(..) => { + info!("Creating cache"); + arg.push(branch.to_string()); + arg_commit_count.push(branch.to_string()); + } + }; + arg.push("--".to_string()); + arg.push(".".to_string()); + let output = Command::new("git") + .args(&arg) + .current_dir(&repo_dir) + .output()? + .stdout; + let output = String::from_utf8_lossy(&output); + let output_commits = Command::new("git") + .args(&arg_commit_count) + .current_dir(&repo_dir) + .output()? + .stdout; + let output_commits = String::from_utf8_lossy(&output_commits); + let commits: u64 = output_commits.trim().parse()?; + let count: u64 = output + .lines() + .map(|s| { + s.split_whitespace() + .take(2) + .map(str::parse::) + .filter_map(std::result::Result::ok) + .sum::() + }) + .sum(); + + let cache = cache.calculate_new_cache(count, commits, (&head).into(), branch); + cache.write_to_file(cache_dir)?; + + Ok((count, head, commits)) +} + +async fn remote_exists(url: &str) -> Result { + let resp = CLIENT.head(url).send().await?; + Ok(resp.status() == reqwest::StatusCode::OK) +} + +enum HocResult { + Hoc { + hoc: u64, + commits: u64, + hoc_pretty: String, + head: String, + url: String, + repo: String, + service_path: String, + }, + NotFound, +} + +async fn delete_repo_and_cache( + state: web::Data, + repo_count: web::Data, + data: web::Path<(String, String)>, +) -> Result +where + T: Service, +{ + let data = data.into_inner(); + let span = info_span!( + "deleting repository and cache", + service = T::domain(), + user = data.0.as_str(), + repo = data.1.as_str() + ); + let future = async { + let repo = format!( + "{}/{}/{}", + T::domain(), + data.0.to_lowercase(), + data.1.to_lowercase() + ); + info!("Deleting cache and repository"); + let cache_dir = format!("{}/{}.json", &state.cache(), repo); + let repo_dir = format!("{}/{}", &state.repos(), repo); + std::fs::remove_file(&cache_dir).or_else(|e| { + if e.kind() == io::ErrorKind::NotFound { + Ok(()) + } else { + Err(e) + } + })?; + std::fs::remove_dir_all(&repo_dir).or_else(|e| { + if e.kind() == io::ErrorKind::NotFound { + Ok(()) + } else { + Err(e) + } + })?; + repo_count.fetch_sub(1, Ordering::Relaxed); + Ok(HttpResponse::TemporaryRedirect() + .header( + LOCATION, + format!("/{}/{}/{}/view", T::url_path(), data.0, data.1), + ) + .finish()) + }; + future.instrument(span).await +} + +async fn handle_hoc_request( + state: web::Data, + repo_count: web::Data, + data: web::Path<(String, String)>, + branch: &str, + mapper: F, +) -> Result +where + T: Service, + F: FnOnce(HocResult) -> Result, +{ + let data = data.into_inner(); + let span = info_span!( + "handling hoc calculation", + service = T::domain(), + user = data.0.as_str(), + repo = data.1.as_str(), + branch + ); + let future = async { + let repo = format!("{}/{}", data.0.to_lowercase(), data.1.to_lowercase()); + let service_path = format!("{}/{}", T::url_path(), repo); + let service_url = format!("{}/{}", T::domain(), repo); + let path = format!("{}/{}", state.repos(), service_url); + let url = format!("https://{}", service_url); + let remote_exists = remote_exists(&url).await?; + let file = Path::new(&path); + if !file.exists() { + if !remote_exists { + warn!("Repository does not exist"); + return mapper(HocResult::NotFound); + } + info!("Cloning for the first time"); + create_dir_all(file)?; + let repo = Repository::init_bare(file)?; + repo.remote_add_fetch("origin", "refs/heads/*:refs/heads/*")?; + repo.remote_set_url("origin", &url)?; + repo_count.fetch_add(1, Ordering::Relaxed); + } + pull(&path)?; + let (hoc, head, commits) = hoc(&service_url, &state.repos(), &state.cache(), branch)?; + let hoc_pretty = match NumberPrefix::decimal(hoc as f64) { + NumberPrefix::Standalone(hoc) => hoc.to_string(), + NumberPrefix::Prefixed(prefix, hoc) => format!("{:.1}{}", hoc, prefix), + }; + let res = HocResult::Hoc { + hoc, + commits, + hoc_pretty, + head, + url, + repo, + service_path, + }; + mapper(res) + }; + future.instrument(span).await +} + +pub(crate) async fn json_hoc( + state: web::Data, + repo_count: web::Data, + data: web::Path<(String, String)>, + branch: web::Query, +) -> Result { + let branch = branch.branch.as_deref().unwrap_or("master"); + let rc_clone = repo_count.clone(); + let mapper = move |r| match r { + HocResult::NotFound => p404(rc_clone), + HocResult::Hoc { + hoc, head, commits, .. + } => Ok(HttpResponse::Ok().json(JsonResponse { + branch, + head: &head, + count: hoc, + commits, + })), + }; + handle_hoc_request::(state, repo_count, data, branch, mapper).await +} + +fn no_cache_response(body: Vec) -> HttpResponse { + let expiration = SystemTime::now() + Duration::from_secs(30); + HttpResponse::Ok() + .content_type("image/svg+xml") + .set(Expires(expiration.into())) + .set(CacheControl(vec![ + CacheDirective::MaxAge(0u32), + CacheDirective::MustRevalidate, + CacheDirective::NoCache, + CacheDirective::NoStore, + ])) + .body(body) +} + +pub(crate) async fn calculate_hoc( + state: web::Data, + repo_count: web::Data, + data: web::Path<(String, String)>, + branch: web::Query, +) -> HttpResponse { + let rc_clone = repo_count.clone(); + let mapper = move |r| match r { + HocResult::NotFound => p404(rc_clone), + HocResult::Hoc { hoc_pretty, .. } => { + let badge_opt = BadgeOptions { + subject: "Hits-of-Code".to_string(), + color: "#007ec6".to_string(), + status: hoc_pretty, + }; + let badge = Badge::new(badge_opt)?; + // TODO: remove clone + let body = badge.to_svg().as_bytes().to_vec(); + + Ok(no_cache_response(body)) + } + }; + let branch = branch.branch.as_deref().unwrap_or("master"); + let error_badge = |_| { + let error_badge = Badge::new(BadgeOptions { + subject: "Hits-of-Code".to_string(), + color: "#ff0000".to_string(), + status: "error".to_string(), + }) + .unwrap(); + let body = error_badge.to_svg().as_bytes().to_vec(); + no_cache_response(body) + }; + handle_hoc_request::(state, repo_count, data, branch, mapper) + .await + .unwrap_or_else(error_badge) +} + +async fn overview( + state: web::Data, + repo_count: web::Data, + data: web::Path<(String, String)>, + branch: web::Query, +) -> Result { + let branch = branch.branch.as_deref().unwrap_or("master"); + let base_url = state.settings.base_url.clone(); + let rc_clone = repo_count.clone(); + let mapper = move |r| match r { + HocResult::NotFound => p404(rc_clone), + HocResult::Hoc { + hoc, + commits, + hoc_pretty, + url, + head, + repo, + service_path, + } => { + let mut buf = Vec::new(); + let repo_info = RepoInfo { + commit_url: &T::commit_url(&repo, &head), + commits, + base_url: &base_url, + head: &head, + hoc, + hoc_pretty: &hoc_pretty, + path: &service_path, + url: &url, + branch, + }; + templates::overview( + &mut buf, + VERSION_INFO, + rc_clone.load(Ordering::Relaxed), + repo_info, + )?; + + Ok(HttpResponse::Ok().content_type("text/html").body(buf)) + } + }; + handle_hoc_request::(state, repo_count, data, branch, mapper).await +} + +#[get("/health_check")] +async fn health_check() -> HttpResponse { + HttpResponse::Ok().finish() +} + +#[get("/")] +async fn index( + state: web::Data, + repo_count: web::Data, +) -> Result { + let mut buf = Vec::new(); + templates::index( + &mut buf, + VERSION_INFO, + repo_count.load(Ordering::Relaxed), + &state.settings.base_url, + )?; + Ok(HttpResponse::Ok().content_type("text/html").body(buf)) +} + +#[post("/generate")] +async fn generate( + params: web::Form>, + state: web::Data, + repo_count: web::Data, +) -> Result { + let repo = format!("{}/{}", params.user, params.repo); + let mut buf = Vec::new(); + templates::generate( + &mut buf, + VERSION_INFO, + repo_count.load(Ordering::Relaxed), + &state.settings.base_url, + params.service.url(), + params.service.service(), + &repo, + )?; + + Ok(HttpResponse::Ok().content_type("text/html").body(buf)) +} + +fn p404(repo_count: web::Data) -> Result { + let mut buf = Vec::new(); + templates::p404(&mut buf, VERSION_INFO, repo_count.load(Ordering::Relaxed))?; + Ok(HttpResponse::NotFound().content_type("text/html").body(buf)) +} + +async fn async_p404(repo_count: web::Data) -> Result { + p404(repo_count) +} + +fn css() -> HttpResponse { + HttpResponse::Ok().content_type("text/css").body(CSS) +} + +fn favicon32() -> HttpResponse { + HttpResponse::Ok().content_type("image/png").body(FAVICON) +} + +async fn start_server(listener: TcpListener, settings: Settings) -> std::io::Result { + let workers = settings.workers; + let repo_count = + // TODO: errorhandling + web::Data::new(AtomicUsize::new(count::count_repositories(&settings.repodir).unwrap())); + let state = web::Data::new(State { + settings + // repos: settings.repodir.display().to_string(), + // cache: settings.cachedir.display().to_string(), + }); + Ok(HttpServer::new(move || { + App::new() + .app_data(state.clone()) + .app_data(repo_count.clone()) + .wrap(tracing_actix_web::TracingLogger) + .wrap(middleware::NormalizePath::new(TrailingSlash::Trim)) + .service(index) + .service(health_check) + .service(web::resource("/tacit-css.min.css").route(web::get().to(css))) + .service(web::resource("/favicon.ico").route(web::get().to(favicon32))) + .service(generate) + .service(web::resource("/github/{user}/{repo}").to(calculate_hoc::)) + .service(web::resource("/gitlab/{user}/{repo}").to(calculate_hoc::)) + .service(web::resource("/bitbucket/{user}/{repo}").to(calculate_hoc::)) + .service( + web::resource("/github/{user}/{repo}/delete") + .route(web::post().to(delete_repo_and_cache::)), + ) + .service( + web::resource("/gitlab/{user}/{repo}/delete") + .route(web::post().to(delete_repo_and_cache::)), + ) + .service( + web::resource("/bitbucket/{user}/{repo}/delete") + .route(web::post().to(delete_repo_and_cache::)), + ) + .service(web::resource("/github/{user}/{repo}/json").to(json_hoc::)) + .service(web::resource("/gitlab/{user}/{repo}/json").to(json_hoc::)) + .service(web::resource("/bitbucket/{user}/{repo}/json").to(json_hoc::)) + .service(web::resource("/view/github/{user}/{repo}").to(overview::)) + .service(web::resource("/view/gitlab/{user}/{repo}").to(overview::)) + .service(web::resource("/view/bitbucket/{user}/{repo}").to(overview::)) + .service(web::resource("/github/{user}/{repo}/view").to(overview::)) + .service(web::resource("/gitlab/{user}/{repo}/view").to(overview::)) + .service(web::resource("/bitbucket/{user}/{repo}/view").to(overview::)) + .default_service(web::resource("").route(web::get().to(async_p404))) + }) + .workers(workers) + .listen(listener)? + .run()) +} + +pub async fn run(listener: TcpListener, settings: Settings) -> std::io::Result { + let span = info_span!("hoc", version = env!("CARGO_PKG_VERSION")); + let _ = span.enter(); + start_server(listener, settings).instrument(span).await +} diff --git a/src/main.rs b/src/main.rs index 78d0b0c..945b6ef 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,495 +1,26 @@ -#![type_length_limit = "2257138"] +use hoc::{config::Settings, telemetry}; -#[macro_use] -extern crate actix_web; -#[macro_use] -extern crate lazy_static; -#[macro_use] -extern crate serde_derive; -#[macro_use] -extern crate tracing; +use std::net::TcpListener; -mod cache; -mod config; -mod count; -mod error; -mod service; -mod statics; -mod template; +fn init() { + dotenv::dotenv().ok(); + openssl_probe::init_ssl_cert_env_vars(); -#[cfg(test)] -mod tests; - -use crate::{ - cache::CacheState, - error::{Error, Result}, - service::{Bitbucket, FormService, GitHub, Gitlab, Service}, - statics::{CLIENT, CSS, FAVICON, OPT, REPO_COUNT, VERSION_INFO}, - template::RepoInfo, -}; -use actix_web::{ - http::header::{CacheControl, CacheDirective, Expires, LOCATION}, - middleware::{self, normalize::TrailingSlash}, - web, App, HttpResponse, HttpServer, Responder, -}; -use badge::{Badge, BadgeOptions}; -use git2::{BranchType, Repository}; -use number_prefix::NumberPrefix; -use std::{ - borrow::Cow, - fs::create_dir_all, - io, - path::Path, - process::Command, - sync::atomic::Ordering, - sync::Arc, - time::{Duration, SystemTime}, -}; -use tracing::Instrument; - -include!(concat!(env!("OUT_DIR"), "/templates.rs")); - -#[derive(Deserialize, Serialize)] -struct GeneratorForm<'a> { - service: FormService, - user: Cow<'a, str>, - repo: Cow<'a, str>, -} - -#[derive(Debug)] -pub(crate) struct State { - repos: String, - cache: String, -} - -#[derive(Serialize)] -struct JsonResponse<'a> { - head: &'a str, - branch: &'a str, - count: u64, - commits: u64, -} - -#[derive(Deserialize, Debug)] -struct BranchQuery { - branch: Option, -} - -fn pull(path: impl AsRef) -> Result<()> { - let repo = Repository::open_bare(path)?; - let mut origin = repo.find_remote("origin")?; - origin.fetch(&["refs/heads/*:refs/heads/*"], None, None)?; - Ok(()) -} - -fn hoc(repo: &str, repo_dir: &str, cache_dir: &str, branch: &str) -> Result<(u64, String, u64)> { - let repo_dir = format!("{}/{}", repo_dir, repo); - let cache_dir = format!("{}/{}.json", cache_dir, repo); - let cache_dir = Path::new(&cache_dir); - let repo = Repository::open_bare(&repo_dir)?; - // TODO: do better... - let head = repo - .find_branch(branch, BranchType::Local) - .map_err(|_| Error::BranchNotFound)? - .into_reference(); - let head = format!("{}", head.target().ok_or(Error::BranchNotFound)?); - let mut arg_commit_count = vec!["rev-list".to_string(), "--count".to_string()]; - let mut arg = vec![ - "log".to_string(), - "--pretty=tformat:".to_string(), - "--numstat".to_string(), - "--ignore-space-change".to_string(), - "--ignore-all-space".to_string(), - "--ignore-submodules".to_string(), - "--no-color".to_string(), - "--find-copies-harder".to_string(), - "-M".to_string(), - "--diff-filter=ACDM".to_string(), - ]; - let cache = CacheState::read_from_file(&cache_dir, branch, &head)?; - match &cache { - CacheState::Current { count, commits, .. } => { - info!("Using cache"); - return Ok((*count, head, *commits)); - } - CacheState::Old { head, .. } => { - info!("Updating cache"); - arg.push(format!("{}..{}", head, branch)); - arg_commit_count.push(format!("{}..{}", head, branch)); - } - CacheState::No | CacheState::NoneForBranch(..) => { - info!("Creating cache"); - arg.push(branch.to_string()); - arg_commit_count.push(branch.to_string()); - } - }; - arg.push("--".to_string()); - arg.push(".".to_string()); - let output = Command::new("git") - .args(&arg) - .current_dir(&repo_dir) - .output()? - .stdout; - let output = String::from_utf8_lossy(&output); - let output_commits = Command::new("git") - .args(&arg_commit_count) - .current_dir(&repo_dir) - .output()? - .stdout; - let output_commits = String::from_utf8_lossy(&output_commits); - let commits: u64 = output_commits.trim().parse()?; - let count: u64 = output - .lines() - .map(|s| { - s.split_whitespace() - .take(2) - .map(str::parse::) - .filter_map(std::result::Result::ok) - .sum::() - }) - .sum(); - - let cache = cache.calculate_new_cache(count, commits, (&head).into(), branch); - cache.write_to_file(cache_dir)?; - - Ok((count, head, commits)) -} - -async fn remote_exists(url: &str) -> Result { - let resp = CLIENT.head(url).send().await?; - Ok(resp.status() == reqwest::StatusCode::OK) -} - -enum HocResult { - Hoc { - hoc: u64, - commits: u64, - hoc_pretty: String, - head: String, - url: String, - repo: String, - service_path: String, - }, - NotFound, -} - -async fn delete_repo_and_cache( - state: web::Data>, - data: web::Path<(String, String)>, -) -> Result -where - T: Service, -{ - let data = data.into_inner(); - let span = info_span!( - "deleting repository and cache", - service = T::domain(), - user = data.0.as_str(), - repo = data.1.as_str() - ); - let future = async { - let repo = format!( - "{}/{}/{}", - T::domain(), - data.0.to_lowercase(), - data.1.to_lowercase() - ); - info!("Deleting cache and repository"); - let cache_dir = format!("{}/{}.json", &state.cache, repo); - let repo_dir = format!("{}/{}", &state.repos, repo); - std::fs::remove_file(&cache_dir).or_else(|e| { - if e.kind() == io::ErrorKind::NotFound { - Ok(()) - } else { - Err(e) - } - })?; - std::fs::remove_dir_all(&repo_dir).or_else(|e| { - if e.kind() == io::ErrorKind::NotFound { - Ok(()) - } else { - Err(e) - } - })?; - REPO_COUNT.fetch_sub(1, Ordering::Relaxed); - Ok(HttpResponse::TemporaryRedirect() - .header( - LOCATION, - format!("/{}/{}/{}/view", T::url_path(), data.0, data.1), - ) - .finish()) - }; - future.instrument(span).await -} - -async fn handle_hoc_request( - state: web::Data>, - data: web::Path<(String, String)>, - branch: &str, - mapper: F, -) -> Result -where - T: Service, - F: Fn(HocResult) -> Result, -{ - let data = data.into_inner(); - let span = info_span!( - "handling hoc calculation", - service = T::domain(), - user = data.0.as_str(), - repo = data.1.as_str(), - branch - ); - let future = async { - let repo = format!("{}/{}", data.0.to_lowercase(), data.1.to_lowercase()); - let service_path = format!("{}/{}", T::url_path(), repo); - let service_url = format!("{}/{}", T::domain(), repo); - let path = format!("{}/{}", state.repos, service_url); - let url = format!("https://{}", service_url); - let remote_exists = remote_exists(&url).await?; - let file = Path::new(&path); - if !file.exists() { - if !remote_exists { - warn!("Repository does not exist"); - return mapper(HocResult::NotFound); - } - info!("Cloning for the first time"); - create_dir_all(file)?; - let repo = Repository::init_bare(file)?; - repo.remote_add_fetch("origin", "refs/heads/*:refs/heads/*")?; - repo.remote_set_url("origin", &url)?; - REPO_COUNT.fetch_add(1, Ordering::Relaxed); - } - pull(&path)?; - let (hoc, head, commits) = hoc(&service_url, &state.repos, &state.cache, branch)?; - let hoc_pretty = match NumberPrefix::decimal(hoc as f64) { - NumberPrefix::Standalone(hoc) => hoc.to_string(), - NumberPrefix::Prefixed(prefix, hoc) => format!("{:.1}{}", hoc, prefix), - }; - let res = HocResult::Hoc { - hoc, - commits, - hoc_pretty, - head, - url, - repo, - service_path, - }; - mapper(res) - }; - future.instrument(span).await -} - -pub(crate) async fn json_hoc( - state: web::Data>, - data: web::Path<(String, String)>, - branch: web::Query, -) -> Result { - let branch = branch.branch.as_deref().unwrap_or("master"); - let mapper = |r| match r { - HocResult::NotFound => p404(), - HocResult::Hoc { - hoc, head, commits, .. - } => Ok(HttpResponse::Ok().json(JsonResponse { - branch, - head: &head, - count: hoc, - commits, - })), - }; - handle_hoc_request::(state, data, branch, mapper).await -} - -fn no_cache_response(body: Vec) -> HttpResponse { - let expiration = SystemTime::now() + Duration::from_secs(30); - HttpResponse::Ok() - .content_type("image/svg+xml") - .set(Expires(expiration.into())) - .set(CacheControl(vec![ - CacheDirective::MaxAge(0u32), - CacheDirective::MustRevalidate, - CacheDirective::NoCache, - CacheDirective::NoStore, - ])) - .body(body) -} - -pub(crate) async fn calculate_hoc( - state: web::Data>, - data: web::Path<(String, String)>, - branch: web::Query, -) -> HttpResponse { - let mapper = move |r| match r { - HocResult::NotFound => p404(), - HocResult::Hoc { hoc_pretty, .. } => { - let badge_opt = BadgeOptions { - subject: "Hits-of-Code".to_string(), - color: "#007ec6".to_string(), - status: hoc_pretty, - }; - let badge = Badge::new(badge_opt)?; - // TODO: remove clone - let body = badge.to_svg().as_bytes().to_vec(); - - Ok(no_cache_response(body)) - } - }; - let branch = branch.branch.as_deref().unwrap_or("master"); - let error_badge = |_| { - let error_badge = Badge::new(BadgeOptions { - subject: "Hits-of-Code".to_string(), - color: "#ff0000".to_string(), - status: "error".to_string(), - }) - .unwrap(); - let body = error_badge.to_svg().as_bytes().to_vec(); - no_cache_response(body) - }; - handle_hoc_request::(state, data, branch, mapper) - .await - .unwrap_or_else(error_badge) -} - -async fn overview( - state: web::Data>, - data: web::Path<(String, String)>, - branch: web::Query, -) -> Result { - let branch = branch.branch.as_deref().unwrap_or("master"); - let mapper = |r| match r { - HocResult::NotFound => p404(), - HocResult::Hoc { - hoc, - commits, - hoc_pretty, - url, - head, - repo, - service_path, - } => { - let mut buf = Vec::new(); - let repo_info = RepoInfo { - commit_url: &T::commit_url(&repo, &head), - commits, - base_url: &OPT.base_url, - head: &head, - hoc, - hoc_pretty: &hoc_pretty, - path: &service_path, - url: &url, - branch, - }; - templates::overview( - &mut buf, - VERSION_INFO, - REPO_COUNT.load(Ordering::Relaxed), - repo_info, - )?; - - Ok(HttpResponse::Ok().content_type("text/html").body(buf)) - } - }; - handle_hoc_request::(state, data, branch, mapper).await -} - -#[get("/")] -async fn index() -> Result { - let mut buf = Vec::new(); - templates::index( - &mut buf, - VERSION_INFO, - REPO_COUNT.load(Ordering::Relaxed), - &OPT.base_url, - )?; - Ok(HttpResponse::Ok().content_type("text/html").body(buf)) -} - -#[post("/generate")] -async fn generate(params: web::Form>) -> Result { - let repo = format!("{}/{}", params.user, params.repo); - let mut buf = Vec::new(); - templates::generate( - &mut buf, - VERSION_INFO, - REPO_COUNT.load(Ordering::Relaxed), - &OPT.base_url, - params.service.url(), - params.service.service(), - &repo, - )?; - - Ok(HttpResponse::Ok().content_type("text/html").body(buf)) -} - -fn p404() -> Result { - let mut buf = Vec::new(); - templates::p404(&mut buf, VERSION_INFO, REPO_COUNT.load(Ordering::Relaxed))?; - Ok(HttpResponse::NotFound().content_type("text/html").body(buf)) -} - -async fn async_p404() -> Result { - p404() -} - -fn css() -> HttpResponse { - HttpResponse::Ok().content_type("text/css").body(CSS) -} - -fn favicon32() -> HttpResponse { - HttpResponse::Ok().content_type("image/png").body(FAVICON) -} - -async fn start_server() -> std::io::Result<()> { - let interface = format!("{}:{}", OPT.host, OPT.port); - let state = Arc::new(State { - repos: OPT.repodir.display().to_string(), - cache: OPT.cachedir.display().to_string(), - }); - HttpServer::new(move || { - App::new() - .data(state.clone()) - .wrap(tracing_actix_web::TracingLogger) - .wrap(middleware::NormalizePath::new(TrailingSlash::Trim)) - .service(index) - .service(web::resource("/tacit-css.min.css").route(web::get().to(css))) - .service(web::resource("/favicon.ico").route(web::get().to(favicon32))) - .service(generate) - .service(web::resource("/github/{user}/{repo}").to(calculate_hoc::)) - .service(web::resource("/gitlab/{user}/{repo}").to(calculate_hoc::)) - .service(web::resource("/bitbucket/{user}/{repo}").to(calculate_hoc::)) - .service( - web::resource("/github/{user}/{repo}/delete") - .route(web::post().to(delete_repo_and_cache::)), - ) - .service( - web::resource("/gitlab/{user}/{repo}/delete") - .route(web::post().to(delete_repo_and_cache::)), - ) - .service( - web::resource("/bitbucket/{user}/{repo}/delete") - .route(web::post().to(delete_repo_and_cache::)), - ) - .service(web::resource("/github/{user}/{repo}/json").to(json_hoc::)) - .service(web::resource("/gitlab/{user}/{repo}/json").to(json_hoc::)) - .service(web::resource("/bitbucket/{user}/{repo}/json").to(json_hoc::)) - .service(web::resource("/view/github/{user}/{repo}").to(overview::)) - .service(web::resource("/view/gitlab/{user}/{repo}").to(overview::)) - .service(web::resource("/view/bitbucket/{user}/{repo}").to(overview::)) - .service(web::resource("/github/{user}/{repo}/view").to(overview::)) - .service(web::resource("/gitlab/{user}/{repo}/view").to(overview::)) - .service(web::resource("/bitbucket/{user}/{repo}/view").to(overview::)) - .default_service(web::resource("").route(web::get().to(async_p404))) - }) - .workers(OPT.workers) - .bind(interface)? - .run() - .await + telemetry::init_subscriber(telemetry::get_subscriber("hoc", "info")) } #[actix_rt::main] async fn main() -> std::io::Result<()> { - config::init(); - let span = info_span!("hoc", version = env!("CARGO_PKG_VERSION")); - let _ = span.enter(); - start_server().instrument(span).await + init(); + + // TODO: error handling + let settings = Settings::load().expect("Cannot load config"); + + let address = format!("{}:{}", settings.host, settings.port); + // TODO: error handling + let listener = TcpListener::bind(address)?; + hoc::run(listener, settings) + .await + .expect("Server error") + .await } diff --git a/src/statics.rs b/src/statics.rs index 0f6f3ed..8aa5d3c 100644 --- a/src/statics.rs +++ b/src/statics.rs @@ -1,6 +1,3 @@ -use crate::{config::Settings, count::count_repositories}; -use std::sync::atomic::AtomicUsize; - pub struct VersionInfo<'a> { pub commit: &'a str, pub version: &'a str, @@ -15,7 +12,4 @@ pub(crate) const FAVICON: &[u8] = include_bytes!("../static/favicon32.png"); lazy_static! { pub(crate) static ref CLIENT: reqwest::Client = reqwest::Client::new(); - pub(crate) static ref OPT: Settings = Settings::new().unwrap(); - pub(crate) static ref REPO_COUNT: AtomicUsize = - AtomicUsize::new(count_repositories(&OPT.repodir).unwrap()); } diff --git a/src/telemetry.rs b/src/telemetry.rs new file mode 100644 index 0000000..e19fd1a --- /dev/null +++ b/src/telemetry.rs @@ -0,0 +1,21 @@ +use tracing::{subscriber::set_global_default, Subscriber}; +use tracing_bunyan_formatter::{BunyanFormattingLayer, JsonStorageLayer}; +use tracing_log::LogTracer; +use tracing_subscriber::{layer::SubscriberExt, EnvFilter, Registry}; + +pub fn get_subscriber(name: &str, env_filter: &str) -> impl Subscriber + Send + Sync { + let env_filter = + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(env_filter)); + + let formatting_layer = BunyanFormattingLayer::new(name.to_string(), std::io::stdout); + + Registry::default() + .with(env_filter) + .with(JsonStorageLayer) + .with(formatting_layer) +} + +pub fn init_subscriber(subscriber: impl Subscriber + Send + Sync) { + LogTracer::init().expect("Failed to set logger"); + set_global_default(subscriber).expect("Failed to set tracing subscriber"); +} diff --git a/src/tests.rs b/src/tests.rs index 15642b5..c4d2a47 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -16,53 +16,53 @@ macro_rules! test_app { }; } -macro_rules! test_service { - ($name: ident, $path: tt, $what: ident) => { - async fn $name(req_path: &str) { - let repo_dir = dbg!(tempdir().unwrap()); - let cache_dir = dbg!(tempdir().unwrap()); - let repos = format!("{}/", repo_dir.path().display()); - let cache = format!("{}/", cache_dir.path().display()); - let state = dbg!(State { repos, cache }); +// macro_rules! test_service { +// ($name: ident, $path: tt, $what: ident) => { +// async fn $name(req_path: &str) { +// let repo_dir = dbg!(tempdir().unwrap()); +// let cache_dir = dbg!(tempdir().unwrap()); +// let repos = format!("{}/", repo_dir.path().display()); +// let cache = format!("{}/", cache_dir.path().display()); +// let state = dbg!(State { repos, cache }); - let mut app = test_app!(state, web::resource($path).to($what::)); +// let mut app = test_app!(state, web::resource($path).to($what::)); - let req = dbg!(test::TestRequest::with_uri(req_path).to_request()); - let resp = dbg!(test::call_service(&mut app, req).await); +// let req = dbg!(test::TestRequest::with_uri(req_path).to_request()); +// let resp = dbg!(test::call_service(&mut app, req).await); - assert_eq!(resp.status(), http::StatusCode::OK); - } - }; -} +// assert_eq!(resp.status(), http::StatusCode::OK); +// } +// }; +// } -#[actix_rt::test] -async fn test_index() { - std::env::set_var("HOC_BASE_URL", "http://0.0.0.0:8080"); - - let mut app = test_app!(index); - - let req = dbg!(test::TestRequest::with_uri("/").to_request()); - let resp = dbg!(test::call_service(&mut app, req).await); - - assert_eq!(resp.status(), http::StatusCode::OK); -} - -// TODO: fix this test // #[actix_rt::test] -async fn test_json() { - test_service!(test_json_service, "/service/{user}/{repo}/json", json_hoc); +// async fn test_index() { +// std::env::set_var("HOC_BASE_URL", "http://0.0.0.0:8080"); - test_json_service::("/service/vbrandl/hoc/json").await; - test_json_service::("/service/vbrandl/hoc/json").await; - test_json_service::("/service/vbrandl/hoc/json").await; -} +// let mut app = test_app!(index); -// TODO: fix this test -// #[actix_rt::test] -async fn test_badge() { - test_service!(test_badge_service, "/service/{user}/{repo}", calculate_hoc); +// let req = dbg!(test::TestRequest::with_uri("/").to_request()); +// let resp = dbg!(test::call_service(&mut app, req).await); - test_badge_service::("/service/vbrandl/hoc").await; - test_badge_service::("/service/vbrandl/hoc").await; - test_badge_service::("/service/vbrandl/hoc").await; -} +// assert_eq!(resp.status(), http::StatusCode::OK); +// } + +// // TODO: fix this test +// // #[actix_rt::test] +// async fn test_json() { +// test_service!(test_json_service, "/service/{user}/{repo}/json", json_hoc); + +// test_json_service::("/service/vbrandl/hoc/json").await; +// test_json_service::("/service/vbrandl/hoc/json").await; +// test_json_service::("/service/vbrandl/hoc/json").await; +// } + +// // TODO: fix this test +// // #[actix_rt::test] +// async fn test_badge() { +// test_service!(test_badge_service, "/service/{user}/{repo}", calculate_hoc); + +// test_badge_service::("/service/vbrandl/hoc").await; +// test_badge_service::("/service/vbrandl/hoc").await; +// test_badge_service::("/service/vbrandl/hoc").await; +// } diff --git a/tests/health_check.rs b/tests/health_check.rs new file mode 100644 index 0000000..8985d86 --- /dev/null +++ b/tests/health_check.rs @@ -0,0 +1,18 @@ +mod util; + +use actix_web::client; + +#[actix_rt::test] +async fn health_check_works() { + let test_app = util::spawn_app().await; + + let client = client::Client::default(); + + let response = client + .get(&format!("{}/health_check", test_app.address)) + .send() + .await + .expect("Failed to execute request"); + + assert!(response.status().is_success()); +} diff --git a/tests/index.rs b/tests/index.rs new file mode 100644 index 0000000..24604e9 --- /dev/null +++ b/tests/index.rs @@ -0,0 +1,18 @@ +mod util; + +use actix_web::client; + +#[actix_rt::test] +async fn index_returns_success() { + let test_app = util::spawn_app().await; + + let client = client::Client::default(); + + let response = client + .get(&format!("{}/", test_app.address)) + .send() + .await + .expect("Failed to execute request"); + + assert!(response.status().is_success()); +} diff --git a/tests/json.rs b/tests/json.rs new file mode 100644 index 0000000..cbce6d8 --- /dev/null +++ b/tests/json.rs @@ -0,0 +1,18 @@ +mod util; + +use actix_web::client; + +#[actix_rt::test] +async fn json_returns_success() { + let test_app = util::spawn_app().await; + + let client = client::Client::default(); + + let response = client + .get(&format!("{}/github/vbrandl/hoc/json", test_app.address)) + .send() + .await + .expect("Failed to execute request"); + + assert!(response.status().is_success()); +} diff --git a/tests/util/mod.rs b/tests/util/mod.rs new file mode 100644 index 0000000..a28e3a1 --- /dev/null +++ b/tests/util/mod.rs @@ -0,0 +1,50 @@ +use hoc::{config::Settings, telemetry}; + +use std::net::TcpListener; + +use tempfile::{tempdir, TempDir}; + +lazy_static::lazy_static! { + static ref TRACING: () = { + let filter = if std::env::var("TEST_LOG").is_ok() { "debug" } else { "" }; + let subscriber = telemetry::get_subscriber("test", filter); + telemetry::init_subscriber(subscriber); + }; +} + +pub struct TestApp { + pub address: String, + repo_dir: TempDir, + cache_dir: TempDir, +} + +pub async fn spawn_app() -> TestApp { + lazy_static::initialize(&TRACING); + + let listener = TcpListener::bind("127.0.0.1:0").expect("Failed to bind random port"); + + let port = listener.local_addr().unwrap().port(); + let address = format!("http://127.0.0.1:{}", port); + + let repo_dir = tempdir().expect("Cannot create repo_dir"); + let cache_dir = tempdir().expect("Cannot create cache_dir"); + + let mut settings = Settings::load().expect("Failed to read configuration."); + settings.repodir = repo_dir.path().to_path_buf(); + settings.cachedir = cache_dir.path().to_path_buf(); + // configuration.database.database_name = Uuid::new_v4().to_string(); + + // let connection_pool = configure_database(&configuration.database).await; + + let server = hoc::run(listener, settings) + .await + .expect("Failed to bind address"); + + let _ = tokio::spawn(server); + + TestApp { + address, + repo_dir, + cache_dir, + } +}