From 902bad4ca503abaa07e8e1232c99742c43e134d9 Mon Sep 17 00:00:00 2001 From: Valentin Brandl Date: Wed, 7 Aug 2019 18:51:28 +0200 Subject: [PATCH] Use redirect cache for tags and branches --- backend/src/main.rs | 104 ++++++++++++++++++++++++++++++++++------- backend/src/service.rs | 29 +++++++++++- 2 files changed, 115 insertions(+), 18 deletions(-) diff --git a/backend/src/main.rs b/backend/src/main.rs index ee6c530..36a7177 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -7,6 +7,9 @@ extern crate serde_derive; #[macro_use] extern crate structopt; +// TODO: cow instead of string + +mod cache; mod cdn; mod config; mod data; @@ -15,19 +18,21 @@ mod service; mod statics; use crate::{ + cache::{Cache, CacheResult}, cdn::Cloudflare, - data::FilePath, + data::{FilePath, State}, error::Result, service::{Bitbucket, GitLab, Github, Service}, statics::{FAVICON, OPT}, }; use actix_files; use actix_web::{ - http::header::{self, CacheControl, CacheDirective}, + http::header::{self, CacheControl, CacheDirective, LOCATION}, middleware, web, App, Error, HttpResponse, HttpServer, }; use awc::{http::StatusCode, Client}; use futures::Future; +use std::sync::{Arc, RwLock}; fn proxy_file( client: web::Data, @@ -62,26 +67,55 @@ fn proxy_file( fn redirect( client: web::Data, + cache: web::Data, data: web::Path, ) -> Box> { + let invalid = { + if let Ok(cache) = cache.read() { + let key = data.to_key::(); + match cache.get(&key) { + CacheResult::Cached(head) => { + let head = head.clone(); + return Box::new(futures::future::ok(()).map(move |_| { + HttpResponse::SeeOther() + .header( + LOCATION, + T::redirect_url(&data.user, &data.repo, &head, &data.file).as_str(), + ) + .finish() + })); + } + CacheResult::Invalid => true, + CacheResult::Empty => false, + } + } else { + false + } + }; + if invalid { + if let Ok(mut cache) = cache.write() { + cache.clear(); + } + } Box::new( client .get(&T::api_url(&data)) .header(header::USER_AGENT, statics::USER_AGENT.as_str()) .send() .from_err() - .and_then(move |response| T::request_head(response, data, client)), + .and_then(move |response| T::request_head(response, data, client, Arc::clone(&cache))), ) } fn handle_request( client: web::Data, + cache: web::Data, data: web::Path, ) -> Box> { if data.commit.len() == 40 { proxy_file::(client, data) } else { - redirect::(client, data) + redirect::(client, cache, data) } } @@ -124,19 +158,49 @@ fn favicon32() -> HttpResponse { .body(FAVICON) } -fn purge_cache( +fn purge_cache( client: web::Data, - file: web::Path, -) -> impl Future { - Cloudflare::purge_cache::(&client, &file) - .map(|success| HttpResponse::Ok().body(success.to_string())) + cache: web::Data, + data: web::Path, +) -> Box> { + if data.commit.len() == 40 { + Box::new( + Cloudflare::purge_cache::(&client, &data.path()) + .map(|success| HttpResponse::Ok().body(success.to_string())), + ) + } else { + let cache = cache.clone(); + Box::new(futures::future::ok(()).map(move |_| { + if let Ok(mut cache) = cache.write() { + let key = data.to_key::(); + cache.invalidate(&key); + HttpResponse::Ok().finish() + } else { + HttpResponse::InternalServerError().finish() + } + })) + } } -fn dbg( +fn dbg( client: web::Data, - file: web::Path, -) -> impl Future { - Cloudflare::dbg::(&client, &file) + cache: web::Data, + data: web::Path, +) -> Box> { + if data.commit.len() == 40 { + Box::new(Cloudflare::dbg::(&client, &data.path())) + } else { + let cache = cache.clone(); + Box::new(futures::future::ok(()).map(move |_| { + if let Ok(mut cache) = cache.write() { + let key = data.to_key::(); + cache.invalidate(&key); + HttpResponse::Ok().finish() + } else { + HttpResponse::InternalServerError().finish() + } + })) + } } fn main() -> Result<()> { @@ -144,9 +208,11 @@ fn main() -> Result<()> { pretty_env_logger::init(); openssl_probe::init_ssl_cert_env_vars(); + let state: State = Arc::new(RwLock::new(Cache::new())); Ok(HttpServer::new(move || { App::new() .data(Client::new()) + .data(state.clone()) .wrap(middleware::Logger::default()) .wrap(middleware::NormalizePath) .service(favicon32) @@ -154,13 +220,16 @@ fn main() -> Result<()> { "/github/{user}/{repo}/{commit}/{file:.*}", web::get().to_async(handle_request::), ) - .route("/github/{file:.*}", web::delete().to_async(dbg::)) + .route( + "/github/{user}/{repo}/{commit}/{file:.*}", + web::delete().to_async(dbg::), + ) .route( "/bitbucket/{user}/{repo}/{commit}/{file:.*}", web::get().to_async(handle_request::), ) .route( - "/bitbucket//{file:.*}", + "/bitbucket/{user}/{repo}/{commit}/{file:.*}", web::delete().to_async(dbg::), ) .route( @@ -171,7 +240,10 @@ fn main() -> Result<()> { "/gist/{user}/{repo}/{commit}/{file:.*}", web::get().to_async(serve_gist), ) - .route("/gitlab/{file:.*}", web::delete().to_async(dbg::)) + .route( + "/gitlab/{user}/{repo}/{commit}/{file:.*}", + web::delete().to_async(dbg::), + ) .service(actix_files::Files::new("/", "./public").index_file("index.html")) }) .workers(OPT.workers) diff --git a/backend/src/service.rs b/backend/src/service.rs index a35ed79..4f1e924 100644 --- a/backend/src/service.rs +++ b/backend/src/service.rs @@ -1,5 +1,6 @@ use crate::{ - data::FilePath, + cache, + data::{FilePath, State}, statics::{load_env_var, GITHUB_AUTH_QUERY, OPT}, }; use actix_web::{ @@ -62,11 +63,13 @@ impl ApiResponse for GitLabApiResponse { } } -pub(crate) trait Service { +pub(crate) trait Service: Sized { type Response: for<'de> serde::Deserialize<'de> + ApiResponse + 'static; fn raw_url(user: &str, repo: &str, commit: &str, file: &str) -> String; + fn cache_service() -> cache::Service; + fn api_url(path: &FilePath) -> String; fn path() -> &'static str; @@ -77,6 +80,7 @@ pub(crate) trait Service { mut response: ClientResponse, data: web::Path, _client: web::Data, + cache: State, ) -> Box> where S: 'static + Stream, @@ -86,6 +90,10 @@ pub(crate) trait Service { response .json::() .map(move |resp| { + if let Ok(mut cache) = cache.write() { + let key = data.to_key::(); + cache.store(key, resp.commit_ref().to_string()); + } HttpResponse::SeeOther() .header( LOCATION, @@ -126,6 +134,10 @@ impl Github { impl Service for Github { type Response = GitHubApiResponse; + fn cache_service() -> cache::Service { + cache::Service::GitHub + } + fn path() -> &'static str { "github" } @@ -157,6 +169,10 @@ pub(crate) struct Bitbucket; impl Service for Bitbucket { type Response = BitbucketApiResponse; + fn cache_service() -> cache::Service { + cache::Service::Bitbucket + } + fn path() -> &'static str { "bitbucket" } @@ -185,6 +201,10 @@ pub(crate) struct GitLab; impl Service for GitLab { type Response = GitLabApiResponse; + fn cache_service() -> cache::Service { + cache::Service::GitLab + } + fn path() -> &'static str { "gitlab" } @@ -209,6 +229,7 @@ impl Service for GitLab { mut response: ClientResponse, data: web::Path, client: web::Data, + cache: State, ) -> Box> where S: 'static + Stream, @@ -232,6 +253,10 @@ impl Service for GitLab { respo .json::() .map(move |resp| { + if let Ok(mut cache) = cache.write() { + let key = data.to_key::(); + cache.store(key, resp.commit_ref().to_string()); + } HttpResponse::SeeOther() .header( LOCATION,