Move everything to backend folder

backend/.dockerignore

@@ -0,0 +1 @@
+/target

backend/.drone.yml

@@ -0,0 +1,15 @@
+---
+kind: pipeline
+name: build
+
+steps:
+  - name: docker
+    image: plugins/docker
+    ref:
+      - refs/heads/master
+      - refs/tags/v*
+---
+kind: signature
+hmac: 1cc0785083f8c74e5fab2c191c65e6ca155a5fbd7b18a559f729b1426fb9511d
+
+...

backend/.gitignore

@@ -0,0 +1,2 @@
+/target
+**/*.rs.bk

backend/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "gitache"
+version = "0.1.0"
+authors = ["Valentin Brandl <vbrandl@riseup.net>"]
+edition = "2018"
+
+
+[dependencies]
+actix-web = "1.0.5"
+awc = { version = "0.2.2", features = ["default", "ssl"] }
+bytes = "0.4.12"
+futures = "0.1.28"
+lazy_static = "1.3.0"
+mime_guess = "1.8.7"
+openssl-probe = "0.1.2"
+pretty_env_logger = "0.3.0"
+serde = "1.0.97"
+serde_derive = "1.0.97"
+serde_json = "1.0.40"

backend/Dockerfile

@@ -0,0 +1,31 @@
+FROM ekidd/rust-musl-builder:stable as builder
+
+# create new cargo project
+RUN USER=rust cargo init --bin
+# copy build config
+COPY --chown=rust ./Cargo.lock ./Cargo.lock
+COPY --chown=rust ./Cargo.toml ./Cargo.toml
+# build to cache dependencies
+RUN cargo build --release
+# delete build cache to prevent caching issues later on
+RUN rm -r ./target/x86_64-unknown-linux-musl/release/.fingerprint/gitache-*
+
+COPY ./src ./src
+# build source code
+RUN cargo build --release
+
+
+# create /etc/password for rootless scratch container
+FROM alpine:latest as user_builder
+RUN USER=root adduser -D -u 10001 dummy
+
+FROM scratch
+
+# copy certificates
+COPY --from=linuxkit/ca-certificates:v0.7 / /
+COPY --from=user_builder /etc/passwd /etc/passwd
+USER dummy
+
+COPY --from=builder /home/rust/src/target/x86_64-unknown-linux-musl/release/gitache /
+
+ENTRYPOINT ["/gitache"]

backend/README.md

@@ -0,0 +1,26 @@
+# Gitache
+
+Gitache is a web service that serves raw files from GitHub, GitLab and Bitbucket
+with the proper `Content-Type` headers. Requests to a branch will be redirected
+to the branches `HEAD`. Requests to a specific commit will also set long time
+cache headers, so the service can be put behind a CDN like Cloudflare.
+
+The endpoints follow the pattern `/<service>/<user>/<repo>/<gitref>/<file>`
+where `<service>` is one of `github`, `gitlab` or `bitbucket`, `<gitref>` is the
+name of the branch or a commit hash.
+
+## Building and Running
+
+The code can be built natively using `cargo build --release` or as a Docker
+image using `docker build .`
+
+The easiest way to run the service is by using `docker-compose up -d` and
+exposing port `8080`.
+
+
+## API Limits
+
+To get the `HEAD` of a requested branch, Gitache sends a request to the
+requested service's API. To prevent running into rate limiting issues with the
+GitHub API, an OAuth2 App should be created and the client ID and secret can be
+set via the `GITHUB_CLIENT_ID` and `GITHUB_CLIENT_SECRET` environment variables.

backend/docker-compose.yml

@@ -0,0 +1,6 @@
+version: '2'
+
+services:
+  gitache:
+    build: ./
+    restart: always

backend/src/data.rs

@@ -0,0 +1,8 @@
+#[derive(Deserialize, Debug)]
+pub(crate) struct FilePath {
+    pub(crate) user: String,
+    pub(crate) repo: String,
+    pub(crate) commit: String,
+    pub(crate) file: String,
+}
+

backend/src/error.rs

@@ -0,0 +1,67 @@
+use actix_web::{HttpResponse, ResponseError};
+use std::fmt;
+
+pub(crate) type Result<T> = std::result::Result<T, Error>;
+
+#[derive(Debug)]
+pub(crate) enum Error {
+    // HttpClient(reqwest::Error),
+    HttpClient(awc::error::SendRequestError),
+    HttpPayload(awc::error::PayloadError),
+    HttpServer(actix_web::Error),
+    Io(std::io::Error),
+}
+
+impl fmt::Display for Error {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        use crate::error::Error::*;
+        match self {
+            HttpClient(err) => write!(f, "HttpClient({})", err),
+            HttpPayload(err) => write!(f, "HttpPayload({})", err),
+            HttpServer(err) => write!(f, "HttpServer({})", err),
+            Io(err) => write!(f, "Io({})", err),
+        }
+    }
+}
+
+impl std::error::Error for Error {}
+
+// impl From<reqwest::Error> for Error {
+//     fn from(err: reqwest::Error) -> Self {
+//         Error::HttpClient(err)
+//     }
+// }
+
+impl From<actix_web::Error> for Error {
+    fn from(err: actix_web::Error) -> Self {
+        Error::HttpServer(err)
+    }
+}
+
+impl From<std::io::Error> for Error {
+    fn from(err: std::io::Error) -> Self {
+        Error::Io(err)
+    }
+}
+
+impl From<awc::error::SendRequestError> for Error {
+    fn from(err: awc::error::SendRequestError) -> Self {
+        Error::HttpClient(err)
+    }
+}
+
+impl From<awc::error::PayloadError> for Error {
+    fn from(err: awc::error::PayloadError) -> Self {
+        Error::HttpPayload(err)
+    }
+}
+
+impl ResponseError for Error {
+    fn error_response(&self) -> HttpResponse {
+        HttpResponse::InternalServerError().finish()
+    }
+
+    fn render_response(&self) -> HttpResponse {
+        self.error_response()
+    }
+}

backend/src/main.rs

@@ -0,0 +1,118 @@
+#[macro_use]
+extern crate actix_web;
+#[macro_use]
+extern crate lazy_static;
+#[macro_use]
+extern crate serde_derive;
+
+mod data;
+mod error;
+mod service;
+mod statics;
+
+use crate::{
+    data::FilePath,
+    error::Result,
+    service::{Bitbucket, GitLab, Github, Service},
+    statics::FAVICON,
+};
+use actix_web::{
+    http::header::{self, CacheControl, CacheDirective},
+    middleware, web, App, Error, HttpResponse, HttpServer,
+};
+use awc::{http::StatusCode, Client};
+use futures::Future;
+
+fn proxy_file<T: Service>(
+    client: web::Data<Client>,
+    data: web::Path<FilePath>,
+) -> Box<dyn Future<Item = HttpResponse, Error = Error>> {
+    Box::new(
+        client
+            .get(&T::raw_url(
+                &data.user,
+                &data.repo,
+                &data.commit,
+                &data.file,
+            ))
+            .header(header::USER_AGENT, statics::USER_AGENT.as_str())
+            .send()
+            .from_err()
+            .and_then(move |response| match response.status() {
+                StatusCode::OK => {
+                    let mime = mime_guess::guess_mime_type(&data.file);
+                    Ok(HttpResponse::Ok()
+                        .content_type(mime.to_string().as_str())
+                        .set(CacheControl(vec![
+                            CacheDirective::Public,
+                            CacheDirective::MaxAge(2_592_000_000),
+                        ]))
+                        .streaming(response))
+                }
+                code => Ok(HttpResponse::build(code).finish()),
+            }),
+    )
+}
+
+fn redirect<T: Service>(
+    client: web::Data<Client>,
+    data: web::Path<FilePath>,
+) -> Box<dyn Future<Item = HttpResponse, Error = Error>> {
+    Box::new(
+        client
+            .get(&T::api_url(&data))
+            .header(header::USER_AGENT, statics::USER_AGENT.as_str())
+            .send()
+            .from_err()
+            .and_then(move |response| T::request_head(response, data, client)),
+    )
+}
+
+fn handle_request<T: Service>(
+    client: web::Data<Client>,
+    data: web::Path<FilePath>,
+) -> Box<dyn Future<Item = HttpResponse, Error = Error>> {
+    if data.commit.len() == 40 {
+        proxy_file::<T>(client, data)
+    } else {
+        redirect::<T>(client, data)
+    }
+}
+
+#[get("/favicon.ico")]
+fn favicon32() -> HttpResponse {
+    HttpResponse::Ok()
+        .content_type("image/png")
+        .set(CacheControl(vec![
+            CacheDirective::Public,
+            CacheDirective::MaxAge(2_592_000_000),
+        ]))
+        .body(FAVICON)
+}
+
+fn main() -> Result<()> {
+    std::env::set_var("RUST_LOG", "actix_server=info,actix_web=trace");
+    pretty_env_logger::init();
+    openssl_probe::init_ssl_cert_env_vars();
+
+    Ok(HttpServer::new(move || {
+        App::new()
+            .data(Client::new())
+            .wrap(middleware::Logger::default())
+            .service(favicon32)
+            .route(
+                "/github/{user}/{repo}/{commit}/{file:.*}",
+                web::get().to_async(handle_request::<Github>),
+            )
+            .route(
+                "/bitbucket/{user}/{repo}/{commit}/{file:.*}",
+                web::get().to_async(handle_request::<Bitbucket>),
+            )
+            .route(
+                "/gitlab/{user}/{repo}/{commit}/{file:.*}",
+                web::get().to_async(handle_request::<GitLab>),
+            )
+    })
+    .bind("0.0.0.0:8080")?
+    .run()?)
+}

backend/src/service.rs

@@ -0,0 +1,248 @@
+use crate::data::FilePath;
+use actix_web::{
+    http::{header::LOCATION, StatusCode},
+    web, Error, HttpResponse,
+};
+use awc::{error::PayloadError, Client, ClientResponse};
+use bytes::Bytes;
+use futures::{Future, Stream};
+
+pub(crate) trait ApiResponse {
+    fn commit_ref(&self) -> &str;
+}
+
+#[derive(Deserialize)]
+pub(crate) struct GitHubApiResponse {
+    pub(crate) sha: String,
+}
+
+impl ApiResponse for GitHubApiResponse {
+    fn commit_ref(&self) -> &str {
+        &self.sha
+    }
+}
+
+#[derive(Deserialize)]
+pub(crate) struct BitbucketApiResponse {
+    values: Vec<BitbucketEntry>,
+}
+
+#[derive(Deserialize)]
+struct BitbucketEntry {
+    hash: String,
+}
+
+impl ApiResponse for BitbucketApiResponse {
+    fn commit_ref(&self) -> &str {
+        &self.values[0].hash
+    }
+}
+
+#[derive(Deserialize)]
+struct GitLabProject {
+    id: u64,
+}
+
+#[derive(Deserialize)]
+pub(crate) struct GitLabApiResponse {
+    commit: GitLabCommit,
+}
+
+#[derive(Deserialize)]
+struct GitLabCommit {
+    id: String,
+}
+
+impl ApiResponse for GitLabApiResponse {
+    fn commit_ref(&self) -> &str {
+        &self.commit.id
+    }
+}
+
+pub(crate) trait Service {
+    type Response: for<'de> serde::Deserialize<'de> + ApiResponse + 'static;
+
+    fn raw_url(user: &str, repo: &str, commit: &str, file: &str) -> String;
+
+    fn api_url(path: &FilePath) -> String;
+
+    fn redirect_url(user: &str, repo: &str, commit: &str, file: &str) -> String;
+
+    fn request_head<S>(
+        mut response: ClientResponse<S>,
+        data: web::Path<FilePath>,
+        _client: web::Data<Client>,
+    ) -> Box<dyn Future<Item = HttpResponse, Error = Error>>
+    where
+        S: 'static + Stream<Item = Bytes, Error = PayloadError>,
+    {
+        Box::new(match response.status() {
+            StatusCode::OK => Box::new(
+                response
+                    .json::<Self::Response>()
+                    .map(move |resp| {
+                        HttpResponse::SeeOther()
+                            .header(
+                                LOCATION,
+                                Self::redirect_url(
+                                    &data.user,
+                                    &data.repo,
+                                    resp.commit_ref(),
+                                    &data.file,
+                                )
+                                .as_str(),
+                            )
+                            .finish()
+                    })
+                    .from_err(),
+            ) as Box<dyn Future<Item = HttpResponse, Error = Error>>,
+            code => Box::new(futures::future::ok(HttpResponse::build(code).finish()))
+                as Box<dyn Future<Item = HttpResponse, Error = Error>>,
+        })
+    }
+}
+
+pub(crate) struct Github;
+
+impl Github {
+    fn auth_query() -> Option<String> {
+        use std::env::var;
+        var("GITHUB_CLIENT_ID").ok().and_then(|id| {
+            var("GITHUB_CLIENT_SECRET")
+                .ok()
+                .map(|secret| format!("?client_id={}&client_secret={}", id, secret))
+        })
+    }
+}
+
+impl Service for Github {
+    type Response = GitHubApiResponse;
+
+    fn raw_url(user: &str, repo: &str, commit: &str, file: &str) -> String {
+        format!(
+            "https://raw.githubusercontent.com/{}/{}/{}/{}",
+            user, repo, commit, file
+        )
+    }
+
+    fn api_url(path: &FilePath) -> String {
+        format!(
+            "https://api.github.com/repos/{}/{}/commits/{}{}",
+            path.user,
+            path.repo,
+            path.commit,
+            Self::auth_query().unwrap_or_default()
+        )
+    }
+
+    fn redirect_url(user: &str, repo: &str, commit: &str, file: &str) -> String {
+        format!("/github/{}/{}/{}/{}", user, repo, commit, file)
+    }
+}
+
+pub(crate) struct Bitbucket;
+
+impl Service for Bitbucket {
+    type Response = BitbucketApiResponse;
+
+    fn raw_url(user: &str, repo: &str, commit: &str, file: &str) -> String {
+        format!(
+            "https://bitbucket.org/{}/{}/raw/{}/{}",
+            user, repo, commit, file
+        )
+    }
+
+    fn api_url(path: &FilePath) -> String {
+        format!(
+            "https://api.bitbucket.org/2.0/repositories/{}/{}/commits/{}?pagelen=1",
+            path.user, path.repo, path.commit
+        )
+    }
+
+    fn redirect_url(user: &str, repo: &str, commit: &str, file: &str) -> String {
+        format!("/bitbucket/{}/{}/{}/{}", user, repo, commit, file)
+    }
+}
+
+pub(crate) struct GitLab;
+
+impl Service for GitLab {
+    type Response = GitLabApiResponse;
+
+    fn raw_url(user: &str, repo: &str, commit: &str, file: &str) -> String {
+        format!(
+            "https://gitlab.com/{}/{}/raw/{}/{}",
+            user, repo, commit, file
+        )
+    }
+
+    fn api_url(path: &FilePath) -> String {
+        let repo_pattern = format!("{}/{}", path.user, path.repo).replace("/", "%2F");
+        format!("https://gitlab.com/api/v4/projects/{}", repo_pattern)
+        // format!(
+        //     "https://gitlab.com/api/v4/projects/{}/repository/branches/{}",
+        //     path.repo, path.commit
+        // )
+    }
+
+    fn redirect_url(user: &str, repo: &str, commit: &str, file: &str) -> String {
+        format!("/gitlab/{}/{}/{}/{}", user, repo, commit, file)
+    }
+
+    fn request_head<S>(
+        mut response: ClientResponse<S>,
+        data: web::Path<FilePath>,
+        client: web::Data<Client>,
+    ) -> Box<dyn Future<Item = HttpResponse, Error = Error>>
+    where
+        S: 'static + Stream<Item = Bytes, Error = PayloadError>,
+    {
+        // "https://gitlab.com/api/v4/projects/{}/repository/branches/{}",
+        Box::new(match response.status() {
+            StatusCode::OK => Box::new(
+                response
+                    .json::<GitLabProject>()
+                    .map(move |resp| resp.id)
+                    .from_err()
+                    .and_then(move |repo_id| {
+                        client
+                            .get(format!(
+                                "https://gitlab.com/api/v4/projects/{}/repository/branches/{}",
+                                repo_id, data.commit
+                            ))
+                            .send()
+                            .from_err()
+                            .and_then(|mut respo| match respo.status() {
+                                StatusCode::OK => Box::new(
+                                    respo
+                                        .json::<Self::Response>()
+                                        .map(move |resp| {
+                                            HttpResponse::SeeOther()
+                                                .header(
+                                                    LOCATION,
+                                                    Self::redirect_url(
+                                                        &data.user,
+                                                        &data.repo,
+                                                        resp.commit_ref(),
+                                                        &data.file,
+                                                    )
+                                                    .as_str(),
+                                                )
+                                                .finish()
+                                        })
+                                        .from_err(),
+                                )
+                                    as Box<dyn Future<Item = HttpResponse, Error = Error>>,
+                                code => Box::new(futures::future::ok(
+                                    HttpResponse::build(code).finish(),
+                                ))
+                                    as Box<dyn Future<Item = HttpResponse, Error = Error>>,
+                            })
+                            .from_err()
+                    }),
+            ) as Box<dyn Future<Item = HttpResponse, Error = Error>>,
+            code => Box::new(futures::future::ok(HttpResponse::build(code).finish()))
+                as Box<dyn Future<Item = HttpResponse, Error = Error>>,
+        })
+    }
+}

backend/src/statics.rs

@@ -0,0 +1,5 @@
+const VERSION: &str = env!("CARGO_PKG_VERSION");
+pub(crate) const FAVICON: &[u8] = include_bytes!("../static/favicon32.png");
+lazy_static! {
+    pub(crate) static ref USER_AGENT: String = format!("gitache/{}", VERSION);
+}