From 4985c497dfa3c1975f91711ec6469d0db3166c82 Mon Sep 17 00:00:00 2001 From: Valentin Brandl Date: Wed, 7 Aug 2019 21:44:20 +0200 Subject: [PATCH 1/6] Extract cache into own crate --- backend/Cargo.lock | 5 +++ backend/Cargo.toml | 1 + backend/src/cache.rs | 90 ------------------------------------------ backend/src/data.rs | 29 +++++++++++--- backend/src/main.rs | 4 +- backend/src/service.rs | 17 ++++---- time-cache/.gitignore | 3 ++ time-cache/Cargo.lock | 6 +++ time-cache/Cargo.toml | 9 +++++ time-cache/src/lib.rs | 74 ++++++++++++++++++++++++++++++++++ 10 files changed, 132 insertions(+), 106 deletions(-) create mode 100644 time-cache/.gitignore create mode 100644 time-cache/Cargo.lock create mode 100644 time-cache/Cargo.toml create mode 100644 time-cache/src/lib.rs diff --git a/backend/Cargo.lock b/backend/Cargo.lock index f80bfeb..9b59fdd 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -659,6 +659,7 @@ dependencies = [ "serde_derive 1.0.97 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.40 (registry+https://github.com/rust-lang/crates.io-index)", "structopt 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", + "time-cache 0.1.0", ] [[package]] @@ -1602,6 +1603,10 @@ dependencies = [ "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "time-cache" +version = "0.1.0" + [[package]] name = "tokio-codec" version = "0.1.1" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index b294c0c..f4dd078 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -19,3 +19,4 @@ serde = { version = "1.0.97", features = ["rc"] } serde_derive = "1.0.97" serde_json = "1.0.40" structopt = "0.2.18" +time-cache = { path = "../time-cache" } diff --git a/backend/src/cache.rs b/backend/src/cache.rs index a190484..e69de29 100644 --- a/backend/src/cache.rs +++ b/backend/src/cache.rs @@ -1,90 +0,0 @@ -use crate::statics::REDIRECT_AGE; -use std::{ - collections::HashMap, - hash::Hash, - sync::Arc, - time::{Duration, Instant}, -}; - -pub(crate) struct Cache { - cache: HashMap>, - duration: Duration, -} - -impl Cache -where - K: Eq + Hash, -{ - pub(crate) fn new() -> Self { - Self { - cache: HashMap::new(), - duration: REDIRECT_AGE, - } - } - - pub(crate) fn get(&self, key: &K) -> CacheResult<&V> { - if let Some(entry) = self.cache.get(key) { - if Self::is_valid(Instant::now(), entry) { - CacheResult::Cached(&entry.1) - } else { - CacheResult::Invalid - } - } else { - CacheResult::Empty - } - } - - pub(crate) fn invalidate(&mut self, key: &K) -> bool { - self.cache.remove(key).is_some() - } - - pub(crate) fn store(&mut self, key: K, value: V) -> Option { - self.cache - .insert(key, CacheEntry::new(value, self.duration)) - .map(|old| old.1) - } - - pub(crate) fn clear(&mut self) { - let now = Instant::now(); - self.cache.retain(|_, v| !Self::is_valid(now, v)); - } - - fn is_valid(when: Instant, entry: &CacheEntry) -> bool { - entry.0 >= when - } -} - -pub(crate) enum CacheResult { - Cached(T), - Invalid, - Empty, -} - -struct CacheEntry(Instant, T); - -impl CacheEntry { - fn new(value: T, duration: Duration) -> Self { - CacheEntry(Instant::now() + duration, value) - } -} - -#[derive(Eq, PartialEq, Hash, Debug)] -pub(crate) struct Key(Service, Arc, Arc, Arc); - -#[derive(Eq, PartialEq, Hash, Debug)] -pub(crate) enum Service { - GitHub, - GitLab, - Bitbucket, -} - -impl Key { - pub(crate) fn new( - service: Service, - user: Arc, - repo: Arc, - branch: Arc, - ) -> Self { - Key(service, user, repo, branch) - } -} diff --git a/backend/src/data.rs b/backend/src/data.rs index 7f36ab0..4c8395d 100644 --- a/backend/src/data.rs +++ b/backend/src/data.rs @@ -1,8 +1,6 @@ -use crate::{ - cache::{Cache, Key}, - service::Service, -}; +use crate::service; use std::sync::{Arc, RwLock}; +use time_cache::Cache; pub(crate) type State = Arc>>; @@ -19,7 +17,7 @@ impl FilePath { format!("{}/{}/{}/{}", self.user, self.repo, self.commit, self.file) } - pub(crate) fn to_key(&self) -> Key { + pub(crate) fn to_key(&self) -> Key { Key::new( T::cache_service(), self.user.clone(), @@ -28,3 +26,24 @@ impl FilePath { ) } } + +#[derive(Eq, PartialEq, Hash, Debug)] +pub(crate) struct Key(Service, Arc, Arc, Arc); + +#[derive(Eq, PartialEq, Hash, Debug)] +pub(crate) enum Service { + GitHub, + GitLab, + Bitbucket, +} + +impl Key { + pub(crate) fn new( + service: Service, + user: Arc, + repo: Arc, + branch: Arc, + ) -> Self { + Key(service, user, repo, branch) + } +} diff --git a/backend/src/main.rs b/backend/src/main.rs index 9aceba8..85a7c15 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -16,7 +16,6 @@ mod service; mod statics; use crate::{ - cache::{Cache, CacheResult}, cdn::Cloudflare, data::{FilePath, State}, error::Result, @@ -31,6 +30,7 @@ use actix_web::{ use awc::{http::StatusCode, Client}; use futures::Future; use std::sync::{Arc, RwLock}; +use time_cache::{Cache, CacheResult}; fn proxy_file( client: web::Data, @@ -210,7 +210,7 @@ fn main() -> Result<()> { pretty_env_logger::init(); openssl_probe::init_ssl_cert_env_vars(); - let state: State = Arc::new(RwLock::new(Cache::new())); + let state: State = Arc::new(RwLock::new(Cache::new(REDIRECT_AGE))); Ok(HttpServer::new(move || { App::new() .data(Client::new()) diff --git a/backend/src/service.rs b/backend/src/service.rs index 9edf522..e8490d6 100644 --- a/backend/src/service.rs +++ b/backend/src/service.rs @@ -1,6 +1,5 @@ use crate::{ - cache, - data::{FilePath, State}, + data::{self, FilePath, State}, statics::{load_env_var, GITHUB_AUTH_QUERY, OPT, REDIRECT_AGE}, }; use actix_web::{ @@ -71,7 +70,7 @@ pub(crate) trait Service: Sized { fn raw_url(user: &str, repo: &str, commit: &str, file: &str) -> String; - fn cache_service() -> cache::Service; + fn cache_service() -> data::Service; fn api_url(path: &FilePath) -> String; @@ -141,8 +140,8 @@ impl Github { impl Service for Github { type Response = GitHubApiResponse; - fn cache_service() -> cache::Service { - cache::Service::GitHub + fn cache_service() -> data::Service { + data::Service::GitHub } fn path() -> &'static str { @@ -176,8 +175,8 @@ pub(crate) struct Bitbucket; impl Service for Bitbucket { type Response = BitbucketApiResponse; - fn cache_service() -> cache::Service { - cache::Service::Bitbucket + fn cache_service() -> data::Service { + data::Service::Bitbucket } fn path() -> &'static str { @@ -208,8 +207,8 @@ pub(crate) struct GitLab; impl Service for GitLab { type Response = GitLabApiResponse; - fn cache_service() -> cache::Service { - cache::Service::GitLab + fn cache_service() -> data::Service { + data::Service::GitLab } fn path() -> &'static str { diff --git a/time-cache/.gitignore b/time-cache/.gitignore new file mode 100644 index 0000000..6936990 --- /dev/null +++ b/time-cache/.gitignore @@ -0,0 +1,3 @@ +/target +**/*.rs.bk +Cargo.lock diff --git a/time-cache/Cargo.lock b/time-cache/Cargo.lock new file mode 100644 index 0000000..5d28d78 --- /dev/null +++ b/time-cache/Cargo.lock @@ -0,0 +1,6 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "time-cache" +version = "0.1.0" + diff --git a/time-cache/Cargo.toml b/time-cache/Cargo.toml new file mode 100644 index 0000000..f453678 --- /dev/null +++ b/time-cache/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "time-cache" +version = "0.1.0" +authors = ["Valentin Brandl "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/time-cache/src/lib.rs b/time-cache/src/lib.rs new file mode 100644 index 0000000..52bce78 --- /dev/null +++ b/time-cache/src/lib.rs @@ -0,0 +1,74 @@ +use std::{ + collections::HashMap, + hash::Hash, + time::{Duration, Instant}, +}; + +pub struct Cache { + cache: HashMap>, + duration: Duration, +} + +impl Cache +where + K: Eq + Hash, +{ + pub fn new(duration: Duration) -> Self { + Self { + cache: HashMap::new(), + duration, + } + } + + pub fn get(&self, key: &K) -> CacheResult<&V> { + if let Some(entry) = self.cache.get(key) { + if Self::is_valid(Instant::now(), entry) { + CacheResult::Cached(&entry.1) + } else { + CacheResult::Invalid + } + } else { + CacheResult::Empty + } + } + + pub fn invalidate(&mut self, key: &K) -> bool { + self.cache.remove(key).is_some() + } + + pub fn store(&mut self, key: K, value: V) -> Option { + self.cache + .insert(key, CacheEntry::new(value, self.duration)) + .map(|old| old.1) + } + + pub fn clear(&mut self) { + let now = Instant::now(); + self.cache.retain(|_, v| !Self::is_valid(now, v)); + } + + fn is_valid(when: Instant, entry: &CacheEntry) -> bool { + entry.0 >= when + } +} + +pub enum CacheResult { + Cached(T), + Invalid, + Empty, +} + +struct CacheEntry(Instant, T); + +impl CacheEntry { + fn new(value: T, duration: Duration) -> Self { + CacheEntry(Instant::now() + duration, value) + } +} +#[cfg(test)] +mod tests { + #[test] + fn it_works() { + assert_eq!(2 + 2, 4); + } +} From 28296a0f6e91bbac2fd1a66f876bf2f5a9426c5f Mon Sep 17 00:00:00 2001 From: Valentin Brandl Date: Wed, 7 Aug 2019 22:18:26 +0200 Subject: [PATCH 2/6] Fix logic bug --- time-cache/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/time-cache/src/lib.rs b/time-cache/src/lib.rs index 52bce78..14747c8 100644 --- a/time-cache/src/lib.rs +++ b/time-cache/src/lib.rs @@ -44,7 +44,7 @@ where pub fn clear(&mut self) { let now = Instant::now(); - self.cache.retain(|_, v| !Self::is_valid(now, v)); + self.cache.retain(|_, v| Self::is_valid(now, v)) } fn is_valid(when: Instant, entry: &CacheEntry) -> bool { From 1e806ef4f0863e22364e82387b47bdcec229c606 Mon Sep 17 00:00:00 2001 From: Valentin Brandl Date: Wed, 7 Aug 2019 22:18:52 +0200 Subject: [PATCH 3/6] Add docs --- time-cache/src/lib.rs | 83 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/time-cache/src/lib.rs b/time-cache/src/lib.rs index 14747c8..a325285 100644 --- a/time-cache/src/lib.rs +++ b/time-cache/src/lib.rs @@ -1,9 +1,13 @@ +//! Simple cache structure that stores values for a specified time. The cache itself is backed by +//! a HashMap. + use std::{ collections::HashMap, hash::Hash, time::{Duration, Instant}, }; +/// Time based cache, that stores values for a defined time. pub struct Cache { cache: HashMap>, duration: Duration, @@ -13,6 +17,16 @@ impl Cache where K: Eq + Hash, { + /// Creates a new cache. + /// + /// # Example + /// ``` + /// use std::time::Duration; + /// use time_cache::{Cache, CacheResult}; + /// + /// let cache: Cache = Cache::new(Duration::from_secs(0)); + /// assert_eq!(CacheResult::Empty, cache.get(&0)); + /// ``` pub fn new(duration: Duration) -> Self { Self { cache: HashMap::new(), @@ -20,6 +34,20 @@ where } } + /// Get an item from the cache. The item can be either valid, invalid or non existent. + /// + /// # Example + /// ``` + /// use std::time::Duration; + /// use time_cache::{Cache, CacheResult}; + /// + /// let key = 0; + /// let value = 1; + /// let mut cache: Cache = Cache::new(Duration::from_secs(0)); + /// assert_eq!(CacheResult::Empty, cache.get(&key)); + /// cache.store(key, value); + /// assert_eq!(CacheResult::Invalid, cache.get(&key)); + /// ``` pub fn get(&self, key: &K) -> CacheResult<&V> { if let Some(entry) = self.cache.get(key) { if Self::is_valid(Instant::now(), entry) { @@ -32,16 +60,66 @@ where } } + /// Removes an item from the cache. Returns `true` if the key was present. + /// + /// # Example + /// ``` + /// use std::time::Duration; + /// use time_cache::{Cache, CacheResult}; + /// + /// let key = 0; + /// let value = 1; + /// let mut cache: Cache = Cache::new(Duration::from_secs(0)); + /// assert!(!cache.invalidate(&key)); + /// cache.store(key, value); + /// assert!(cache.invalidate(&key)); + /// ``` pub fn invalidate(&mut self, key: &K) -> bool { self.cache.remove(key).is_some() } + /// Stores an item in the cache. + /// + /// # Example + /// ``` + /// use std::time::Duration; + /// use time_cache::{Cache, CacheResult}; + /// + /// let key = 0; + /// let value = 1; + /// let dur = Duration::from_millis(500); + /// let mut cache: Cache = Cache::new(dur); + /// + /// assert_eq!(CacheResult::Empty, cache.get(&key)); + /// cache.store(key, value); + /// assert_eq!(CacheResult::Cached(&value), cache.get(&key)); + /// std::thread::sleep(dur); + /// assert_eq!(CacheResult::Invalid, cache.get(&key)); + /// ``` pub fn store(&mut self, key: K, value: V) -> Option { self.cache .insert(key, CacheEntry::new(value, self.duration)) .map(|old| old.1) } + /// Removes all invalid items from the cache. + /// + /// # Example + /// ``` + /// use std::time::Duration; + /// use time_cache::{Cache, CacheResult}; + /// + /// let key = 0; + /// let value = 1; + /// let dur = Duration::from_secs(0); + /// let mut cache: Cache = Cache::new(dur); + /// + /// assert_eq!(CacheResult::Empty, cache.get(&key)); + /// cache.store(key, value); + /// assert_eq!(CacheResult::Invalid, cache.get(&key)); + /// cache.clear(); + /// assert_eq!(CacheResult::Empty, cache.get(&key)); + /// ``` pub fn clear(&mut self) { let now = Instant::now(); self.cache.retain(|_, v| Self::is_valid(now, v)) @@ -52,9 +130,14 @@ where } } +/// Result when requesting a cached item. +#[derive(Debug, PartialEq)] pub enum CacheResult { + /// Item is cached and still valid Cached(T), + /// Item is cached but invalid Invalid, + /// Item is not in the cache Empty, } From 501257684ec2e3ee8336ad12d27e1e5f1b9ac812 Mon Sep 17 00:00:00 2001 From: Valentin Brandl Date: Wed, 7 Aug 2019 22:19:41 +0200 Subject: [PATCH 4/6] Implement tests --- time-cache/src/lib.rs | 77 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 75 insertions(+), 2 deletions(-) diff --git a/time-cache/src/lib.rs b/time-cache/src/lib.rs index a325285..884c6d8 100644 --- a/time-cache/src/lib.rs +++ b/time-cache/src/lib.rs @@ -148,10 +148,83 @@ impl CacheEntry { CacheEntry(Instant::now() + duration, value) } } + #[cfg(test)] mod tests { + use super::{Cache, CacheResult}; + use std::time::Duration; + #[test] - fn it_works() { - assert_eq!(2 + 2, 4); + fn always_invalid() { + let key = 0; + let value = 1; + let mut cache = Cache::new(Duration::from_secs(0)); + assert_eq!(CacheResult::Empty, cache.get(&key)); + cache.store(key, value); + assert_eq!(CacheResult::Invalid, cache.get(&key)); + } + + #[test] + fn valid() { + let key = 0; + let value = 1; + let mut cache = Cache::new(Duration::from_secs(100)); + assert_eq!(CacheResult::Empty, cache.get(&key)); + cache.store(key, value); + assert_eq!(CacheResult::Cached(&value), cache.get(&key)); + } + + #[test] + fn wait_for_invalidation() { + let key = 0; + let value = 1; + let dur = Duration::from_millis(500); + let mut cache = Cache::new(dur); + assert_eq!(CacheResult::Empty, cache.get(&key)); + cache.store(key, value); + assert_eq!(CacheResult::Cached(&value), cache.get(&key)); + std::thread::sleep(dur); + assert_eq!(CacheResult::Invalid, cache.get(&key)); + } + + #[test] + fn invalidate() { + let key = 0; + let value = 1; + let dur = Duration::from_secs(100); + let mut cache = Cache::new(dur); + assert_eq!(CacheResult::Empty, cache.get(&key)); + cache.store(key, value); + assert_eq!(CacheResult::Cached(&value), cache.get(&key)); + assert!(cache.invalidate(&key)); + assert_eq!(CacheResult::Empty, cache.get(&key)); + } + + #[test] + fn invalidate_wait() { + let key = 0; + let value = 1; + let dur = Duration::from_millis(500); + let mut cache = Cache::new(dur); + assert_eq!(CacheResult::Empty, cache.get(&key)); + cache.store(key, value); + assert_eq!(CacheResult::Cached(&value), cache.get(&key)); + std::thread::sleep(dur); + assert_eq!(CacheResult::Invalid, cache.get(&key)); + assert!(cache.invalidate(&key)); + assert_eq!(CacheResult::Empty, cache.get(&key)); + } + + #[test] + fn clear() { + let key = 0; + let value = 1; + let dur = Duration::from_secs(0); + let mut cache = Cache::new(dur); + assert_eq!(CacheResult::Empty, cache.get(&key)); + cache.store(key, value); + assert_eq!(CacheResult::Invalid, cache.get(&key)); + cache.clear(); + assert_eq!(CacheResult::Empty, cache.get(&key)); } } From d97e392a936b8a2f23f293101c82284b208b3296 Mon Sep 17 00:00:00 2001 From: Valentin Brandl Date: Wed, 7 Aug 2019 22:24:20 +0200 Subject: [PATCH 5/6] Copy cache crate --- Dockerfile | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8ed2ce2..8418647 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,17 +17,21 @@ RUN ./build.sh FROM ekidd/rust-musl-builder:stable as backend # create new cargo project -RUN USER=rust cargo init --bin +RUN USER=rust cargo new --bin gitache +RUN cargo new --lib time-cache +WORKDIR /home/rust/src/gitache # copy build config -COPY --chown=rust ./backend/Cargo.lock ./Cargo.lock -COPY --chown=rust ./backend/Cargo.toml ./Cargo.toml +COPY --chown=rust ./backend/Cargo.lock ./gitache/Cargo.lock +COPY --chown=rust ./backend/Cargo.toml ./gitache/Cargo.toml +COPY --chown=rust ./time-cache/Cargo.toml ./time-cache/Cargo.toml # build to cache dependencies RUN cargo build --release # delete build cache to prevent caching issues later on RUN rm -r ./target/x86_64-unknown-linux-musl/release/.fingerprint/gitache-* -COPY ./backend/static ./static -COPY ./backend/src ./src +COPY ./backend/static ./gitache/static +COPY ./backend/src ./gitache/src +COPY ./time-cache/src ./time-cache/src # build source code RUN cargo build --release @@ -43,7 +47,7 @@ COPY --from=linuxkit/ca-certificates:v0.7 / / COPY --from=user_builder /etc/passwd /etc/passwd USER dummy -COPY --from=backend /home/rust/src/target/x86_64-unknown-linux-musl/release/gitache / +COPY --from=backend /home/rust/src/gitache/target/x86_64-unknown-linux-musl/release/gitache / COPY --from=frontend /output/index.html /public/index.html COPY --from=frontend /output/scripts /public/scripts COPY --from=frontend /output/assets /public/assets From da3592f82c543bb80a9699e5c5c3131547a540d4 Mon Sep 17 00:00:00 2001 From: Valentin Brandl Date: Wed, 7 Aug 2019 22:24:38 +0200 Subject: [PATCH 6/6] Link dockerignore --- time-cache/.dockerignore | 1 + 1 file changed, 1 insertion(+) create mode 120000 time-cache/.dockerignore diff --git a/time-cache/.dockerignore b/time-cache/.dockerignore new file mode 120000 index 0000000..3e4e48b --- /dev/null +++ b/time-cache/.dockerignore @@ -0,0 +1 @@ +.gitignore \ No newline at end of file