actix/actix-extras
1
0
Fork 0
mirror of https://github.com/actix/actix-extras.git synced 2025-01-23 15:24:36 +01:00
Code Issues Releases Wiki Activity

Switch brotli compressor to rust. (#1197)

Browse Source 
* Switch to a rustified version of brotli.

* Some memory optimizations.

* Make brotli not optional anymore.
This commit is contained in:
daxpedda 2019-12-07 16:55:41 +01:00 committed by Nikolay Kim
parent 0ba125444a
commit e5f3d88a4e
10 changed files with 53 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Cargo.toml
View File

@ -16,7 +16,7 @@ exclude = [".gitignore", ".travis.yml", ".cargo/config", "appveyor.yml"]
edition = "2018" edition = "2018"
[package.metadata.docs.rs] [package.metadata.docs.rs]
features = ["openssl", "brotli", "flate2-zlib", "secure-cookies", "client"] features = ["openssl", "flate2-zlib", "secure-cookies", "client"]
[badges] [badges]
travis-ci = { repository = "actix/actix-web", branch = "master" } travis-ci = { repository = "actix/actix-web", branch = "master" }
@ -43,14 +43,11 @@ members = [
] ]
[features] [features]
default = ["brotli", "flate2-zlib", "client", "fail"] default = ["flate2-zlib", "client", "fail"]
# http client # http client
client = ["awc"] client = ["awc"]
# brotli encoding, requires c compiler
brotli = ["actix-http/brotli", "awc/brotli"]
# miniz-sys backend for flate2 crate # miniz-sys backend for flate2 crate
flate2-zlib = ["actix-http/flate2-zlib", "awc/flate2-zlib"] flate2-zlib = ["actix-http/flate2-zlib", "awc/flate2-zlib"]
@ -111,7 +108,7 @@ actix-http-test = "1.0.0-alpha.3"
rand = "0.7" rand = "0.7"
env_logger = "0.6" env_logger = "0.6"
serde_derive = "1.0" serde_derive = "1.0"
brotli2 = "0.3.2" brotli = "3.3.0"
flate2 = "1.0.2" flate2 = "1.0.2"
[profile.release] [profile.release]

7
actix-http/Cargo.toml
View File

@ -16,7 +16,7 @@ edition = "2018"
workspace = ".." workspace = ".."
[package.metadata.docs.rs] [package.metadata.docs.rs]
features = ["openssl", "rustls", "fail", "brotli", "flate2-zlib", "secure-cookies"] features = ["openssl", "rustls", "fail", "flate2-zlib", "secure-cookies"]
[lib] [lib]
name = "actix_http" name = "actix_http"
@ -31,9 +31,6 @@ openssl = ["actix-tls/openssl", "actix-connect/openssl"]
# rustls support # rustls support
rustls = ["actix-tls/rustls", "actix-connect/rustls"] rustls = ["actix-tls/rustls", "actix-connect/rustls"]
# brotli encoding, requires c compiler
brotli = ["brotli2"]
# miniz-sys backend for flate2 crate # miniz-sys backend for flate2 crate
flate2-zlib = ["flate2/miniz-sys"] flate2-zlib = ["flate2/miniz-sys"]
@ -88,7 +85,7 @@ time = "0.1.42"
ring = { version = "0.16.9", optional = true } ring = { version = "0.16.9", optional = true }
# compression # compression
brotli2 = { version="0.3.2", optional = true } brotli = "3.3.0"
flate2 = { version="1.0.7", optional = true, default-features = false } flate2 = { version="1.0.7", optional = true, default-features = false }
# optional deps # optional deps

17
actix-http/src/encoding/decoder.rs
View File

@ -4,8 +4,7 @@ use std::pin::Pin;
use std::task::{Context, Poll}; use std::task::{Context, Poll};
use actix_threadpool::{run, CpuFuture}; use actix_threadpool::{run, CpuFuture};
#[cfg(feature = "brotli")] use brotli::DecompressorWriter;
use brotli2::write::BrotliDecoder;
use bytes::Bytes; use bytes::Bytes;
#[cfg(any(feature = "flate2-zlib", feature = "flate2-rust"))] #[cfg(any(feature = "flate2-zlib", feature = "flate2-rust"))]
use flate2::write::{GzDecoder, ZlibDecoder}; use flate2::write::{GzDecoder, ZlibDecoder};
@ -32,9 +31,8 @@ where
#[inline] #[inline]
pub fn new(stream: S, encoding: ContentEncoding) -> Decoder<S> { pub fn new(stream: S, encoding: ContentEncoding) -> Decoder<S> {
let decoder = match encoding { let decoder = match encoding {
#[cfg(feature = "brotli")]
ContentEncoding::Br => Some(ContentDecoder::Br(Box::new( ContentEncoding::Br => Some(ContentDecoder::Br(Box::new(
BrotliDecoder::new(Writer::new()), DecompressorWriter::new(Writer::new(), 0),
))), ))),
#[cfg(any(feature = "flate2-zlib", feature = "flate2-rust"))] #[cfg(any(feature = "flate2-zlib", feature = "flate2-rust"))]
ContentEncoding::Deflate => Some(ContentDecoder::Deflate(Box::new( ContentEncoding::Deflate => Some(ContentDecoder::Deflate(Box::new(
@ -144,18 +142,16 @@ enum ContentDecoder {
Deflate(Box<ZlibDecoder<Writer>>), Deflate(Box<ZlibDecoder<Writer>>),
#[cfg(any(feature = "flate2-zlib", feature = "flate2-rust"))] #[cfg(any(feature = "flate2-zlib", feature = "flate2-rust"))]
Gzip(Box<GzDecoder<Writer>>), Gzip(Box<GzDecoder<Writer>>),
#[cfg(feature = "brotli")] Br(Box<DecompressorWriter<Writer>>),
Br(Box<BrotliDecoder<Writer>>),
} }
impl ContentDecoder { impl ContentDecoder {
#[allow(unreachable_patterns)] #[allow(unreachable_patterns)]
fn feed_eof(&mut self) -> io::Result<Option<Bytes>> { fn feed_eof(&mut self) -> io::Result<Option<Bytes>> {
match self { match self {
#[cfg(feature = "brotli")] ContentDecoder::Br(ref mut decoder) => match decoder.flush() {
ContentDecoder::Br(ref mut decoder) => match decoder.finish() { Ok(()) => {
Ok(mut writer) => { let b = decoder.get_mut().take();
let b = writer.take();
if !b.is_empty() { if !b.is_empty() {
Ok(Some(b)) Ok(Some(b))
} else { } else {
@ -195,7 +191,6 @@ impl ContentDecoder {
#[allow(unreachable_patterns)] #[allow(unreachable_patterns)]
fn feed_data(&mut self, data: Bytes) -> io::Result<Option<Bytes>> { fn feed_data(&mut self, data: Bytes) -> io::Result<Option<Bytes>> {
match self { match self {
#[cfg(feature = "brotli")]
ContentDecoder::Br(ref mut decoder) => match decoder.write_all(&data) { ContentDecoder::Br(ref mut decoder) => match decoder.write_all(&data) {
Ok(_) => { Ok(_) => {
decoder.flush()?; decoder.flush()?;

30
actix-http/src/encoding/encoder.rs
View File

@ -5,8 +5,7 @@ use std::pin::Pin;
use std::task::{Context, Poll}; use std::task::{Context, Poll};
use actix_threadpool::{run, CpuFuture}; use actix_threadpool::{run, CpuFuture};
#[cfg(feature = "brotli")] use brotli::CompressorWriter;
use brotli2::write::BrotliEncoder;
use bytes::Bytes; use bytes::Bytes;
#[cfg(any(feature = "flate2-zlib", feature = "flate2-rust"))] #[cfg(any(feature = "flate2-zlib", feature = "flate2-rust"))]
use flate2::write::{GzEncoder, ZlibEncoder}; use flate2::write::{GzEncoder, ZlibEncoder};
@ -177,8 +176,7 @@ enum ContentEncoder {
Deflate(ZlibEncoder<Writer>), Deflate(ZlibEncoder<Writer>),
#[cfg(any(feature = "flate2-zlib", feature = "flate2-rust"))] #[cfg(any(feature = "flate2-zlib", feature = "flate2-rust"))]
Gzip(GzEncoder<Writer>), Gzip(GzEncoder<Writer>),
#[cfg(feature = "brotli")] Br(CompressorWriter<Writer>),
Br(BrotliEncoder<Writer>),
} }
impl ContentEncoder { impl ContentEncoder {
@ -194,10 +192,12 @@ impl ContentEncoder {
Writer::new(), Writer::new(),
flate2::Compression::fast(), flate2::Compression::fast(),
))), ))),
#[cfg(feature = "brotli")] ContentEncoding::Br => Some(ContentEncoder::Br(CompressorWriter::new(
ContentEncoding::Br => { Writer::new(),
Some(ContentEncoder::Br(BrotliEncoder::new(Writer::new(), 3))) 0,
} 3,
0,
))),
_ => None, _ => None,
} }
} }
@ -205,8 +205,11 @@ impl ContentEncoder {
#[inline] #[inline]
pub(crate) fn take(&mut self) -> Bytes { pub(crate) fn take(&mut self) -> Bytes {
match *self { match *self {
#[cfg(feature = "brotli")] ContentEncoder::Br(ref mut encoder) => {
ContentEncoder::Br(ref mut encoder) => encoder.get_mut().take(), let mut encoder_new = CompressorWriter::new(Writer::new(), 0, 3, 0);
std::mem::swap(encoder, &mut encoder_new);
encoder_new.into_inner().freeze()
}
#[cfg(any(feature = "flate2-zlib", feature = "flate2-rust"))] #[cfg(any(feature = "flate2-zlib", feature = "flate2-rust"))]
ContentEncoder::Deflate(ref mut encoder) => encoder.get_mut().take(), ContentEncoder::Deflate(ref mut encoder) => encoder.get_mut().take(),
#[cfg(any(feature = "flate2-zlib", feature = "flate2-rust"))] #[cfg(any(feature = "flate2-zlib", feature = "flate2-rust"))]
@ -216,11 +219,7 @@ impl ContentEncoder {
fn finish(self) -> Result<Bytes, io::Error> { fn finish(self) -> Result<Bytes, io::Error> {
match self { match self {
#[cfg(feature = "brotli")] ContentEncoder::Br(encoder) => Ok(encoder.into_inner().buf.freeze()),
ContentEncoder::Br(encoder) => match encoder.finish() {
Ok(writer) => Ok(writer.buf.freeze()),
Err(err) => Err(err),
},
#[cfg(any(feature = "flate2-zlib", feature = "flate2-rust"))] #[cfg(any(feature = "flate2-zlib", feature = "flate2-rust"))]
ContentEncoder::Gzip(encoder) => match encoder.finish() { ContentEncoder::Gzip(encoder) => match encoder.finish() {
Ok(writer) => Ok(writer.buf.freeze()), Ok(writer) => Ok(writer.buf.freeze()),
@ -236,7 +235,6 @@ impl ContentEncoder {
fn write(&mut self, data: &[u8]) -> Result<(), io::Error> { fn write(&mut self, data: &[u8]) -> Result<(), io::Error> {
match *self { match *self {
#[cfg(feature = "brotli")]
ContentEncoder::Br(ref mut encoder) => match encoder.write_all(data) { ContentEncoder::Br(ref mut encoder) => match encoder.write_all(data) {
Ok(_) => Ok(()), Ok(_) => Ok(()),
Err(err) => { Err(err) => {

3
actix-http/src/encoding/mod.rs
View File

@ -22,6 +22,9 @@ impl Writer {
fn take(&mut self) -> Bytes { fn take(&mut self) -> Bytes {
self.buf.split().freeze() self.buf.split().freeze()
} }
fn freeze(self) -> Bytes {
self.buf.freeze()
}
} }
impl io::Write for Writer { impl io::Write for Writer {

9
awc/Cargo.toml
View File

@ -21,10 +21,10 @@ name = "awc"
path = "src/lib.rs" path = "src/lib.rs"
[package.metadata.docs.rs] [package.metadata.docs.rs]
features = ["openssl", "rustls", "brotli", "flate2-zlib"] features = ["openssl", "rustls", "flate2-zlib"]
[features] [features]
default = ["brotli", "flate2-zlib"] default = ["flate2-zlib"]
# openssl # openssl
openssl = ["open-ssl", "actix-http/openssl"] openssl = ["open-ssl", "actix-http/openssl"]
@ -32,9 +32,6 @@ openssl = ["open-ssl", "actix-http/openssl"]
# rustls # rustls
rustls = ["rust-tls", "actix-http/rustls"] rustls = ["rust-tls", "actix-http/rustls"]
# brotli encoding, requires c compiler
brotli = ["actix-http/brotli"]
# miniz-sys backend for flate2 crate # miniz-sys backend for flate2 crate
flate2-zlib = ["actix-http/flate2-zlib"] flate2-zlib = ["actix-http/flate2-zlib"]
@ -69,7 +66,7 @@ actix-http-test = { version = "1.0.0-alpha.3", features=["openssl"] }
actix-utils = "1.0.0-alpha.3" actix-utils = "1.0.0-alpha.3"
actix-server = { version = "1.0.0-alpha.3" } actix-server = { version = "1.0.0-alpha.3" }
actix-tls = { version = "1.0.0-alpha.3", features=["openssl", "rustls"] } actix-tls = { version = "1.0.0-alpha.3", features=["openssl", "rustls"] }
brotli2 = { version="0.3.2" } brotli = "3.3.0"
flate2 = { version="1.0.2" } flate2 = { version="1.0.2" }
env_logger = "0.6" env_logger = "0.6"
webpki = { version = "0.21" } webpki = { version = "0.21" }

49
awc/src/request.rs
View File

@ -23,13 +23,10 @@ use crate::frozen::FrozenClientRequest;
use crate::sender::{PrepForSendingError, RequestSender, SendClientRequest}; use crate::sender::{PrepForSendingError, RequestSender, SendClientRequest};
use crate::ClientConfig; use crate::ClientConfig;
#[cfg(any(feature = "brotli", feature = "flate2-zlib", feature = "flate2-rust"))] #[cfg(any(feature = "flate2-zlib", feature = "flate2-rust"))]
const HTTPS_ENCODING: &str = "br, gzip, deflate"; const HTTPS_ENCODING: &str = "br, gzip, deflate";
#[cfg(all( #[cfg(not(any(feature = "flate2-zlib", feature = "flate2-rust")))]
any(feature = "flate2-zlib", feature = "flate2-rust"), const HTTPS_ENCODING: &str = "br";
not(feature = "brotli")
))]
const HTTPS_ENCODING: &str = "gzip, deflate";
/// An HTTP Client request builder /// An HTTP Client request builder
/// ///
@ -544,31 +541,23 @@ impl ClientRequest {
let mut slf = self; let mut slf = self;
// enable br only for https if slf.response_decompress {
#[cfg(any( let https = slf
feature = "brotli", .head
feature = "flate2-zlib", .uri
feature = "flate2-rust" .scheme()
))] .map(|s| s == &uri::Scheme::HTTPS)
{ .unwrap_or(true);
if slf.response_decompress {
let https = slf
.head
.uri
.scheme()
.map(|s| s == &uri::Scheme::HTTPS)
.unwrap_or(true);
if https { if https {
slf = slf.set_header_if_none(header::ACCEPT_ENCODING, HTTPS_ENCODING) slf = slf.set_header_if_none(header::ACCEPT_ENCODING, HTTPS_ENCODING)
} else { } else {
#[cfg(any(feature = "flate2-zlib", feature = "flate2-rust"))] #[cfg(any(feature = "flate2-zlib", feature = "flate2-rust"))]
{ {
slf = slf slf =
.set_header_if_none(header::ACCEPT_ENCODING, "gzip, deflate") slf.set_header_if_none(header::ACCEPT_ENCODING, "gzip, deflate")
} }
}; };
}
} }
Ok(slf) Ok(slf)

3
awc/tests/test_client.rs
View File

@ -4,7 +4,7 @@ use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc; use std::sync::Arc;
use std::time::Duration; use std::time::Duration;
use brotli2::write::BrotliEncoder; use brotli::write::BrotliEncoder;
use bytes::Bytes; use bytes::Bytes;
use flate2::read::GzDecoder; use flate2::read::GzDecoder;
use flate2::write::GzEncoder; use flate2::write::GzEncoder;
@ -568,7 +568,6 @@ async fn test_client_brotli_encoding() {
// assert_eq!(bytes, Bytes::from(data)); // assert_eq!(bytes, Bytes::from(data));
// } // }
// #[cfg(feature = "brotli")]
// #[actix_rt::test] // #[actix_rt::test]
// async fn test_client_deflate_encoding() { // async fn test_client_deflate_encoding() {
// let srv = test::TestServer::start(|app| { // let srv = test::TestServer::start(|app| {

2
src/lib.rs
View File

@ -72,8 +72,6 @@
//! * `rustls` - enables ssl support via `rustls` crate, supports `http/2` //! * `rustls` - enables ssl support via `rustls` crate, supports `http/2`
//! * `secure-cookies` - enables secure cookies support, includes `ring` crate as //! * `secure-cookies` - enables secure cookies support, includes `ring` crate as
//! dependency //! dependency
//! * `brotli` - enables `brotli` compression support, requires `c`
//! compiler (default enabled)
//! * `flate2-zlib` - enables `gzip`, `deflate` compression support, requires //! * `flate2-zlib` - enables `gzip`, `deflate` compression support, requires
//! `c` compiler (default enabled) //! `c` compiler (default enabled)
//! * `flate2-rust` - experimental rust based implementation for //! * `flate2-rust` - experimental rust based implementation for

6
tests/test_server.rs
View File

@ -6,7 +6,7 @@ use actix_http::http::header::{
}; };
use actix_http::{Error, HttpService, Response}; use actix_http::{Error, HttpService, Response};
use actix_http_test::TestServer; use actix_http_test::TestServer;
use brotli2::write::{BrotliDecoder, BrotliEncoder}; use brotli::write::{BrotliDecoder, BrotliEncoder};
use bytes::Bytes; use bytes::Bytes;
use flate2::read::GzDecoder; use flate2::read::GzDecoder;
use flate2::write::{GzEncoder, ZlibDecoder, ZlibEncoder}; use flate2::write::{GzEncoder, ZlibDecoder, ZlibEncoder};
@ -296,7 +296,6 @@ async fn test_body_chunked_implicit() {
} }
#[actix_rt::test] #[actix_rt::test]
#[cfg(feature = "brotli")]
async fn test_body_br_streaming() { async fn test_body_br_streaming() {
let srv = TestServer::start(move || { let srv = TestServer::start(move || {
HttpService::build() HttpService::build()
@ -411,7 +410,6 @@ async fn test_body_deflate() {
} }
#[actix_rt::test] #[actix_rt::test]
#[cfg(any(feature = "brotli"))]
async fn test_body_brotli() { async fn test_body_brotli() {
let srv = TestServer::start(move || { let srv = TestServer::start(move || {
HttpService::build() HttpService::build()
@ -717,7 +715,7 @@ async fn test_brotli_encoding_large() {
assert_eq!(bytes, Bytes::from(data)); assert_eq!(bytes, Bytes::from(data));
} }
// #[cfg(all(feature = "brotli", feature = "ssl"))] // #[cfg(feature = "ssl")]
// #[actix_rt::test] // #[actix_rt::test]
// async fn test_brotli_encoding_large_ssl() { // async fn test_brotli_encoding_large_ssl() {
// use actix::{Actor, System}; // use actix::{Actor, System};