vbrandl/yagcdn
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'feature/redirect-cache'

Browse Source
This commit is contained in:
Valentin Brandl 2019-08-07 18:51:52 +02:00
commit 49d3055d76
No known key found for this signature in database
GPG Key ID: 30D341DD34118D7D
4 changed files with 220 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

83
backend/src/cache.rs Normal file
View File

@ -0,0 +1,83 @@
use std::{
collections::HashMap,
hash::Hash,
time::{Duration, Instant},
};
pub(crate) struct Cache<K, V> {
cache: HashMap<K, CacheEntry<V>>,
duration: Duration,
}
impl<K, V> Cache<K, V>
where
K: Eq + Hash,
{
pub(crate) fn new() -> Self {
Self {
cache: HashMap::new(),
duration: Duration::from_secs(5 * 60),
}
}
pub(crate) fn get(&self, key: &K) -> CacheResult<&V> {
if let Some(entry) = self.cache.get(key) {
if Self::is_valid(Instant::now(), entry) {
CacheResult::Cached(&entry.1)
} else {
CacheResult::Invalid
}
} else {
CacheResult::Empty
}
}
pub(crate) fn invalidate(&mut self, key: &K) -> bool {
self.cache.remove(key).is_some()
}
pub(crate) fn store(&mut self, key: K, value: V) -> Option<V> {
self.cache
.insert(key, CacheEntry::new(value, self.duration))
.map(|old| old.1)
}
pub(crate) fn clear(&mut self) {
let now = Instant::now();
self.cache.retain(|_, v| !Self::is_valid(now, v));
}
fn is_valid(when: Instant, entry: &CacheEntry<V>) -> bool {
entry.0 >= when
}
}
pub(crate) enum CacheResult<T> {
Cached(T),
Invalid,
Empty,
}
struct CacheEntry<T>(Instant, T);
impl<T> CacheEntry<T> {
fn new(value: T, duration: Duration) -> Self {
CacheEntry(Instant::now() + duration, value)
}
}
#[derive(Eq, PartialEq, Hash, Debug)]
pub(crate) struct Key(Service, String, String, String);
#[derive(Eq, PartialEq, Hash, Debug)]
pub(crate) enum Service {
GitHub,
GitLab,
Bitbucket,
}
impl Key {
pub(crate) fn new(service: Service, user: String, repo: String, branch: String) -> Self {
Key(service, user, repo, branch)
}
}

22
backend/src/data.rs
View File

@ -1,3 +1,11 @@
use crate::{
cache::{Cache, Key},
service::Service,
};
use std::sync::{Arc, RwLock};
pub(crate) type State = Arc<RwLock<Cache<Key, String>>>;
#[derive(Deserialize, Debug)]
pub(crate) struct FilePath {
pub(crate) user: String,
@ -6,3 +14,17 @@ pub(crate) struct FilePath {
pub(crate) file: String,
}
impl FilePath {
pub(crate) fn path(&self) -> String {
format!("{}/{}/{}/{}", self.user, self.repo, self.commit, self.file)
}
pub(crate) fn to_key<T: Service>(&self) -> Key {
Key::new(
T::cache_service(),
self.user.clone(),
self.repo.clone(),
self.commit.clone(),
)
}
}

104
backend/src/main.rs
View File

@ -7,6 +7,9 @@ extern crate serde_derive;
#[macro_use]
extern crate structopt;
// TODO: cow instead of string
mod cache;
mod cdn;
mod config;
mod data;
@ -15,19 +18,21 @@ mod service;
mod statics;
use crate::{
cache::{Cache, CacheResult},
cdn::Cloudflare,
data::FilePath,
data::{FilePath, State},
error::Result,
service::{Bitbucket, GitLab, Github, Service},
statics::{FAVICON, OPT},
};
use actix_files;
use actix_web::{
http::header::{self, CacheControl, CacheDirective},
http::header::{self, CacheControl, CacheDirective, LOCATION},
middleware, web, App, Error, HttpResponse, HttpServer,
};
use awc::{http::StatusCode, Client};
use futures::Future;
use std::sync::{Arc, RwLock};
fn proxy_file<T: Service>(
client: web::Data<Client>,
@ -62,26 +67,55 @@ fn proxy_file<T: Service>(
fn redirect<T: Service>(
client: web::Data<Client>,
cache: web::Data<State>,
data: web::Path<FilePath>,
) -> Box<dyn Future<Item = HttpResponse, Error = Error>> {
let invalid = {
if let Ok(cache) = cache.read() {
let key = data.to_key::<T>();
match cache.get(&key) {
CacheResult::Cached(head) => {
let head = head.clone();
return Box::new(futures::future::ok(()).map(move |_| {
HttpResponse::SeeOther()
.header(
LOCATION,
T::redirect_url(&data.user, &data.repo, &head, &data.file).as_str(),
)
.finish()
}));
}
CacheResult::Invalid => true,
CacheResult::Empty => false,
}
} else {
false
}
};
if invalid {
if let Ok(mut cache) = cache.write() {
cache.clear();
}
}
Box::new(
client
.get(&T::api_url(&data))
.header(header::USER_AGENT, statics::USER_AGENT.as_str())
.send()
.from_err()
.and_then(move |response| T::request_head(response, data, client)),
.and_then(move |response| T::request_head(response, data, client, Arc::clone(&cache))),
)
}
fn handle_request<T: Service>(
client: web::Data<Client>,
cache: web::Data<State>,
data: web::Path<FilePath>,
) -> Box<dyn Future<Item = HttpResponse, Error = Error>> {
if data.commit.len() == 40 {
proxy_file::<T>(client, data)
} else {
redirect::<T>(client, data)
redirect::<T>(client, cache, data)
}
}
@ -124,19 +158,49 @@ fn favicon32() -> HttpResponse {
.body(FAVICON)
}
fn purge_cache<T: Service>(
fn purge_cache<T: 'static + Service>(
client: web::Data<Client>,
file: web::Path<String>,
) -> impl Future<Item = HttpResponse, Error = Error> {
Cloudflare::purge_cache::<T>(&client, &file)
.map(|success| HttpResponse::Ok().body(success.to_string()))
cache: web::Data<State>,
data: web::Path<FilePath>,
) -> Box<dyn Future<Item = HttpResponse, Error = Error>> {
if data.commit.len() == 40 {
Box::new(
Cloudflare::purge_cache::<T>(&client, &data.path())
.map(|success| HttpResponse::Ok().body(success.to_string())),
)
} else {
let cache = cache.clone();
Box::new(futures::future::ok(()).map(move |_| {
if let Ok(mut cache) = cache.write() {
let key = data.to_key::<T>();
cache.invalidate(&key);
HttpResponse::Ok().finish()
} else {
HttpResponse::InternalServerError().finish()
}
}))
}
}
fn dbg<T: Service>(
fn dbg<T: 'static + Service>(
client: web::Data<Client>,
file: web::Path<String>,
) -> impl Future<Item = HttpResponse, Error = Error> {
Cloudflare::dbg::<T>(&client, &file)
cache: web::Data<State>,
data: web::Path<FilePath>,
) -> Box<dyn Future<Item = HttpResponse, Error = Error>> {
if data.commit.len() == 40 {
Box::new(Cloudflare::dbg::<T>(&client, &data.path()))
} else {
let cache = cache.clone();
Box::new(futures::future::ok(()).map(move |_| {
if let Ok(mut cache) = cache.write() {
let key = data.to_key::<T>();
cache.invalidate(&key);
HttpResponse::Ok().finish()
} else {
HttpResponse::InternalServerError().finish()
}
}))
}
}
fn main() -> Result<()> {
@ -144,9 +208,11 @@ fn main() -> Result<()> {
pretty_env_logger::init();
openssl_probe::init_ssl_cert_env_vars();
let state: State = Arc::new(RwLock::new(Cache::new()));
Ok(HttpServer::new(move || {
App::new()
.data(Client::new())
.data(state.clone())
.wrap(middleware::Logger::default())
.wrap(middleware::NormalizePath)
.service(favicon32)
@ -154,13 +220,16 @@ fn main() -> Result<()> {
"/github/{user}/{repo}/{commit}/{file:.*}",
web::get().to_async(handle_request::<Github>),
)
.route("/github/{file:.*}", web::delete().to_async(dbg::<Github>))
.route(
"/github/{user}/{repo}/{commit}/{file:.*}",
web::delete().to_async(dbg::<Github>),
)
.route(
"/bitbucket/{user}/{repo}/{commit}/{file:.*}",
web::get().to_async(handle_request::<Bitbucket>),
)
.route(
"/bitbucket//{file:.*}",
"/bitbucket/{user}/{repo}/{commit}/{file:.*}",
web::delete().to_async(dbg::<Bitbucket>),
)
.route(
@ -171,7 +240,10 @@ fn main() -> Result<()> {
"/gist/{user}/{repo}/{commit}/{file:.*}",
web::get().to_async(serve_gist),
)
.route("/gitlab/{file:.*}", web::delete().to_async(dbg::<GitLab>))
.route(
"/gitlab/{user}/{repo}/{commit}/{file:.*}",
web::delete().to_async(dbg::<GitLab>),
)
.service(actix_files::Files::new("/", "./public").index_file("index.html"))
})
.workers(OPT.workers)

29
backend/src/service.rs
View File

@ -1,5 +1,6 @@
use crate::{
data::FilePath,
cache,
data::{FilePath, State},
statics::{load_env_var, GITHUB_AUTH_QUERY, OPT},
};
use actix_web::{
@ -62,11 +63,13 @@ impl ApiResponse for GitLabApiResponse {
}
}
pub(crate) trait Service {
pub(crate) trait Service: Sized {
type Response: for<'de> serde::Deserialize<'de> + ApiResponse + 'static;
fn raw_url(user: &str, repo: &str, commit: &str, file: &str) -> String;
fn cache_service() -> cache::Service;
fn api_url(path: &FilePath) -> String;
fn path() -> &'static str;
@ -77,6 +80,7 @@ pub(crate) trait Service {
mut response: ClientResponse<S>,
data: web::Path<FilePath>,
_client: web::Data<Client>,
cache: State,
) -> Box<dyn Future<Item = HttpResponse, Error = Error>>
where
S: 'static + Stream<Item = Bytes, Error = PayloadError>,
@ -86,6 +90,10 @@ pub(crate) trait Service {
response
.json::<Self::Response>()
.map(move |resp| {
if let Ok(mut cache) = cache.write() {
let key = data.to_key::<Self>();
cache.store(key, resp.commit_ref().to_string());
}
HttpResponse::SeeOther()
.header(
LOCATION,
@ -126,6 +134,10 @@ impl Github {
impl Service for Github {
type Response = GitHubApiResponse;
fn cache_service() -> cache::Service {
cache::Service::GitHub
}
fn path() -> &'static str {
"github"
}
@ -157,6 +169,10 @@ pub(crate) struct Bitbucket;
impl Service for Bitbucket {
type Response = BitbucketApiResponse;
fn cache_service() -> cache::Service {
cache::Service::Bitbucket
}
fn path() -> &'static str {
"bitbucket"
}
@ -185,6 +201,10 @@ pub(crate) struct GitLab;
impl Service for GitLab {
type Response = GitLabApiResponse;
fn cache_service() -> cache::Service {
cache::Service::GitLab
}
fn path() -> &'static str {
"gitlab"
}
@ -209,6 +229,7 @@ impl Service for GitLab {
mut response: ClientResponse<S>,
data: web::Path<FilePath>,
client: web::Data<Client>,
cache: State,
) -> Box<dyn Future<Item = HttpResponse, Error = Error>>
where
S: 'static + Stream<Item = Bytes, Error = PayloadError>,
@ -232,6 +253,10 @@ impl Service for GitLab {
respo
.json::<Self::Response>()
.map(move |resp| {
if let Ok(mut cache) = cache.write() {
let key = data.to_key::<Self>();
cache.store(key, resp.commit_ref().to_string());
}
HttpResponse::SeeOther()
.header(
LOCATION,