From 566f4c0e010785a350955f97a39152da3aeb12f5 Mon Sep 17 00:00:00 2001 From: Valentin Brandl Date: Wed, 7 Aug 2019 18:50:28 +0200 Subject: [PATCH 1/3] Add cache struct and implementation --- backend/src/cache.rs | 83 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 backend/src/cache.rs diff --git a/backend/src/cache.rs b/backend/src/cache.rs new file mode 100644 index 0000000..9ca559d --- /dev/null +++ b/backend/src/cache.rs @@ -0,0 +1,83 @@ +use std::{ + collections::HashMap, + hash::Hash, + time::{Duration, Instant}, +}; + +pub(crate) struct Cache { + cache: HashMap>, + duration: Duration, +} + +impl Cache +where + K: Eq + Hash, +{ + pub(crate) fn new() -> Self { + Self { + cache: HashMap::new(), + duration: Duration::from_secs(5 * 60), + } + } + + pub(crate) fn get(&self, key: &K) -> CacheResult<&V> { + if let Some(entry) = self.cache.get(key) { + if Self::is_valid(Instant::now(), entry) { + CacheResult::Cached(&entry.1) + } else { + CacheResult::Invalid + } + } else { + CacheResult::Empty + } + } + + pub(crate) fn invalidate(&mut self, key: &K) -> bool { + self.cache.remove(key).is_some() + } + + pub(crate) fn store(&mut self, key: K, value: V) -> Option { + self.cache + .insert(key, CacheEntry::new(value, self.duration)) + .map(|old| old.1) + } + + pub(crate) fn clear(&mut self) { + let now = Instant::now(); + self.cache.retain(|_, v| !Self::is_valid(now, v)); + } + + fn is_valid(when: Instant, entry: &CacheEntry) -> bool { + entry.0 >= when + } +} + +pub(crate) enum CacheResult { + Cached(T), + Invalid, + Empty, +} + +struct CacheEntry(Instant, T); + +impl CacheEntry { + fn new(value: T, duration: Duration) -> Self { + CacheEntry(Instant::now() + duration, value) + } +} + +#[derive(Eq, PartialEq, Hash, Debug)] +pub(crate) struct Key(Service, String, String, String); + +#[derive(Eq, PartialEq, Hash, Debug)] +pub(crate) enum Service { + GitHub, + GitLab, + Bitbucket, +} + +impl Key { + pub(crate) fn new(service: Service, user: String, repo: String, branch: String) -> Self { + Key(service, user, repo, branch) + } +} From b65df17a4e41b67898b4a8fc0adbb8d72ff7eaa2 Mon Sep 17 00:00:00 2001 From: Valentin Brandl Date: Wed, 7 Aug 2019 18:50:52 +0200 Subject: [PATCH 2/3] Helper methods --- backend/src/data.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/backend/src/data.rs b/backend/src/data.rs index d68166e..5439e68 100644 --- a/backend/src/data.rs +++ b/backend/src/data.rs @@ -1,3 +1,11 @@ +use crate::{ + cache::{Cache, Key}, + service::Service, +}; +use std::sync::{Arc, RwLock}; + +pub(crate) type State = Arc>>; + #[derive(Deserialize, Debug)] pub(crate) struct FilePath { pub(crate) user: String, @@ -6,3 +14,17 @@ pub(crate) struct FilePath { pub(crate) file: String, } +impl FilePath { + pub(crate) fn path(&self) -> String { + format!("{}/{}/{}/{}", self.user, self.repo, self.commit, self.file) + } + + pub(crate) fn to_key(&self) -> Key { + Key::new( + T::cache_service(), + self.user.clone(), + self.repo.clone(), + self.commit.clone(), + ) + } +} From 902bad4ca503abaa07e8e1232c99742c43e134d9 Mon Sep 17 00:00:00 2001 From: Valentin Brandl Date: Wed, 7 Aug 2019 18:51:28 +0200 Subject: [PATCH 3/3] Use redirect cache for tags and branches --- backend/src/main.rs | 104 ++++++++++++++++++++++++++++++++++------- backend/src/service.rs | 29 +++++++++++- 2 files changed, 115 insertions(+), 18 deletions(-) diff --git a/backend/src/main.rs b/backend/src/main.rs index ee6c530..36a7177 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -7,6 +7,9 @@ extern crate serde_derive; #[macro_use] extern crate structopt; +// TODO: cow instead of string + +mod cache; mod cdn; mod config; mod data; @@ -15,19 +18,21 @@ mod service; mod statics; use crate::{ + cache::{Cache, CacheResult}, cdn::Cloudflare, - data::FilePath, + data::{FilePath, State}, error::Result, service::{Bitbucket, GitLab, Github, Service}, statics::{FAVICON, OPT}, }; use actix_files; use actix_web::{ - http::header::{self, CacheControl, CacheDirective}, + http::header::{self, CacheControl, CacheDirective, LOCATION}, middleware, web, App, Error, HttpResponse, HttpServer, }; use awc::{http::StatusCode, Client}; use futures::Future; +use std::sync::{Arc, RwLock}; fn proxy_file( client: web::Data, @@ -62,26 +67,55 @@ fn proxy_file( fn redirect( client: web::Data, + cache: web::Data, data: web::Path, ) -> Box> { + let invalid = { + if let Ok(cache) = cache.read() { + let key = data.to_key::(); + match cache.get(&key) { + CacheResult::Cached(head) => { + let head = head.clone(); + return Box::new(futures::future::ok(()).map(move |_| { + HttpResponse::SeeOther() + .header( + LOCATION, + T::redirect_url(&data.user, &data.repo, &head, &data.file).as_str(), + ) + .finish() + })); + } + CacheResult::Invalid => true, + CacheResult::Empty => false, + } + } else { + false + } + }; + if invalid { + if let Ok(mut cache) = cache.write() { + cache.clear(); + } + } Box::new( client .get(&T::api_url(&data)) .header(header::USER_AGENT, statics::USER_AGENT.as_str()) .send() .from_err() - .and_then(move |response| T::request_head(response, data, client)), + .and_then(move |response| T::request_head(response, data, client, Arc::clone(&cache))), ) } fn handle_request( client: web::Data, + cache: web::Data, data: web::Path, ) -> Box> { if data.commit.len() == 40 { proxy_file::(client, data) } else { - redirect::(client, data) + redirect::(client, cache, data) } } @@ -124,19 +158,49 @@ fn favicon32() -> HttpResponse { .body(FAVICON) } -fn purge_cache( +fn purge_cache( client: web::Data, - file: web::Path, -) -> impl Future { - Cloudflare::purge_cache::(&client, &file) - .map(|success| HttpResponse::Ok().body(success.to_string())) + cache: web::Data, + data: web::Path, +) -> Box> { + if data.commit.len() == 40 { + Box::new( + Cloudflare::purge_cache::(&client, &data.path()) + .map(|success| HttpResponse::Ok().body(success.to_string())), + ) + } else { + let cache = cache.clone(); + Box::new(futures::future::ok(()).map(move |_| { + if let Ok(mut cache) = cache.write() { + let key = data.to_key::(); + cache.invalidate(&key); + HttpResponse::Ok().finish() + } else { + HttpResponse::InternalServerError().finish() + } + })) + } } -fn dbg( +fn dbg( client: web::Data, - file: web::Path, -) -> impl Future { - Cloudflare::dbg::(&client, &file) + cache: web::Data, + data: web::Path, +) -> Box> { + if data.commit.len() == 40 { + Box::new(Cloudflare::dbg::(&client, &data.path())) + } else { + let cache = cache.clone(); + Box::new(futures::future::ok(()).map(move |_| { + if let Ok(mut cache) = cache.write() { + let key = data.to_key::(); + cache.invalidate(&key); + HttpResponse::Ok().finish() + } else { + HttpResponse::InternalServerError().finish() + } + })) + } } fn main() -> Result<()> { @@ -144,9 +208,11 @@ fn main() -> Result<()> { pretty_env_logger::init(); openssl_probe::init_ssl_cert_env_vars(); + let state: State = Arc::new(RwLock::new(Cache::new())); Ok(HttpServer::new(move || { App::new() .data(Client::new()) + .data(state.clone()) .wrap(middleware::Logger::default()) .wrap(middleware::NormalizePath) .service(favicon32) @@ -154,13 +220,16 @@ fn main() -> Result<()> { "/github/{user}/{repo}/{commit}/{file:.*}", web::get().to_async(handle_request::), ) - .route("/github/{file:.*}", web::delete().to_async(dbg::)) + .route( + "/github/{user}/{repo}/{commit}/{file:.*}", + web::delete().to_async(dbg::), + ) .route( "/bitbucket/{user}/{repo}/{commit}/{file:.*}", web::get().to_async(handle_request::), ) .route( - "/bitbucket//{file:.*}", + "/bitbucket/{user}/{repo}/{commit}/{file:.*}", web::delete().to_async(dbg::), ) .route( @@ -171,7 +240,10 @@ fn main() -> Result<()> { "/gist/{user}/{repo}/{commit}/{file:.*}", web::get().to_async(serve_gist), ) - .route("/gitlab/{file:.*}", web::delete().to_async(dbg::)) + .route( + "/gitlab/{user}/{repo}/{commit}/{file:.*}", + web::delete().to_async(dbg::), + ) .service(actix_files::Files::new("/", "./public").index_file("index.html")) }) .workers(OPT.workers) diff --git a/backend/src/service.rs b/backend/src/service.rs index a35ed79..4f1e924 100644 --- a/backend/src/service.rs +++ b/backend/src/service.rs @@ -1,5 +1,6 @@ use crate::{ - data::FilePath, + cache, + data::{FilePath, State}, statics::{load_env_var, GITHUB_AUTH_QUERY, OPT}, }; use actix_web::{ @@ -62,11 +63,13 @@ impl ApiResponse for GitLabApiResponse { } } -pub(crate) trait Service { +pub(crate) trait Service: Sized { type Response: for<'de> serde::Deserialize<'de> + ApiResponse + 'static; fn raw_url(user: &str, repo: &str, commit: &str, file: &str) -> String; + fn cache_service() -> cache::Service; + fn api_url(path: &FilePath) -> String; fn path() -> &'static str; @@ -77,6 +80,7 @@ pub(crate) trait Service { mut response: ClientResponse, data: web::Path, _client: web::Data, + cache: State, ) -> Box> where S: 'static + Stream, @@ -86,6 +90,10 @@ pub(crate) trait Service { response .json::() .map(move |resp| { + if let Ok(mut cache) = cache.write() { + let key = data.to_key::(); + cache.store(key, resp.commit_ref().to_string()); + } HttpResponse::SeeOther() .header( LOCATION, @@ -126,6 +134,10 @@ impl Github { impl Service for Github { type Response = GitHubApiResponse; + fn cache_service() -> cache::Service { + cache::Service::GitHub + } + fn path() -> &'static str { "github" } @@ -157,6 +169,10 @@ pub(crate) struct Bitbucket; impl Service for Bitbucket { type Response = BitbucketApiResponse; + fn cache_service() -> cache::Service { + cache::Service::Bitbucket + } + fn path() -> &'static str { "bitbucket" } @@ -185,6 +201,10 @@ pub(crate) struct GitLab; impl Service for GitLab { type Response = GitLabApiResponse; + fn cache_service() -> cache::Service { + cache::Service::GitLab + } + fn path() -> &'static str { "gitlab" } @@ -209,6 +229,7 @@ impl Service for GitLab { mut response: ClientResponse, data: web::Path, client: web::Data, + cache: State, ) -> Box> where S: 'static + Stream, @@ -232,6 +253,10 @@ impl Service for GitLab { respo .json::() .map(move |resp| { + if let Ok(mut cache) = cache.write() { + let key = data.to_key::(); + cache.store(key, resp.commit_ref().to_string()); + } HttpResponse::SeeOther() .header( LOCATION,