From 6c765739d08c4e414369cc220456a70a7f0c6fb7 Mon Sep 17 00:00:00 2001 From: Dursun Akkurt Date: Wed, 13 Jun 2018 20:43:03 +0300 Subject: [PATCH 1/6] add HttpMessage::readlines() --- src/httpmessage.rs | 97 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 96 insertions(+), 1 deletion(-) diff --git a/src/httpmessage.rs b/src/httpmessage.rs index a9d68d3a..a8ae50ed 100644 --- a/src/httpmessage.rs +++ b/src/httpmessage.rs @@ -3,7 +3,7 @@ use encoding::all::UTF_8; use encoding::label::encoding_from_whatwg_label; use encoding::types::{DecoderTrap, Encoding}; use encoding::EncodingRef; -use futures::{Future, Poll, Stream}; +use futures::{Future, Poll, Stream, Async}; use http::{header, HeaderMap}; use http_range::HttpRange; use mime::Mime; @@ -260,6 +260,101 @@ pub trait HttpMessage { let boundary = Multipart::boundary(self.headers()); Multipart::new(boundary, self) } + + /// Return stream of lines. + fn readlines(self) -> Readlines + where + Self: Stream + Sized, + { + Readlines::new(self) + } +} + +/// Stream to read request line by line. +pub struct Readlines +where + T: HttpMessage + Stream + 'static, +{ + req: T, + buff: Vec, + limit: usize, +} + +impl Readlines +where + T: HttpMessage + Stream + 'static, +{ + /// Create a new stream to read request line by line. + fn new(req: T) -> Self { + Readlines { + req, + buff: Vec::with_capacity(256), + limit: 262_144, + } + } + + /// Change max size of payload. By default max size is 256Kb + pub fn limit(mut self, limit: usize) -> Self { + self.limit = limit; + self + } +} + +impl Stream for Readlines +where + T: HttpMessage + Stream + 'static, +{ + type Item = String; + type Error = ReadlinesError; + + fn poll(&mut self) -> Poll, Self::Error> { + match self.req.poll() { + Ok(Async::Ready(Some(bytes))) => { + for b in bytes.iter() { + if *b == '\n' as u8 { + self.buff.push(*b); + let line = str::from_utf8(&*self.buff)?.to_owned(); + self.buff.clear(); + return Ok(Async::Ready(Some(line))); + } else { + self.buff.push(*b); + } + if self.limit < self.buff.len() { + return Err(ReadlinesError::LimitOverflow); + } + } + Ok(Async::NotReady) + }, + Ok(Async::NotReady) => Ok(Async::NotReady), + Ok(Async::Ready(None)) => { + if self.buff.len() == 0 { + return Ok(Async::Ready(None)); + } + let line = str::from_utf8(&*self.buff)?.to_owned(); + self.buff.clear(); + return Ok(Async::Ready(Some(line))) + }, + Err(e) => Err(ReadlinesError::from(e)), + } + } +} + +pub enum ReadlinesError { + EncodingError, + PayloadError(PayloadError), + LimitOverflow, +} + +impl From for ReadlinesError { + fn from(err: PayloadError) -> Self { + ReadlinesError::PayloadError(err) + } +} + +impl From for ReadlinesError { + fn from(_: str::Utf8Error) -> Self { + ReadlinesError::EncodingError + } } /// Future that resolves to a complete http message body. From 6d95e34552e48b79aa6319a1ee71a8208ac985b9 Mon Sep 17 00:00:00 2001 From: Dursun Akkurt Date: Wed, 13 Jun 2018 20:45:31 +0300 Subject: [PATCH 2/6] add HttpMessage::readlines() --- src/httpmessage.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/httpmessage.rs b/src/httpmessage.rs index a8ae50ed..7f6b50c6 100644 --- a/src/httpmessage.rs +++ b/src/httpmessage.rs @@ -288,7 +288,7 @@ where fn new(req: T) -> Self { Readlines { req, - buff: Vec::with_capacity(256), + buff: Vec::with_capacity(262_144), limit: 262_144, } } From f8854f951c27d9266fdabf857d55d77a7581db6a Mon Sep 17 00:00:00 2001 From: Daniel McNab <36049421+DJMcNab@users.noreply.github.com> Date: Wed, 13 Jun 2018 20:31:20 +0100 Subject: [PATCH 3/6] remove duplication of `App::with_state` in `App::new` --- src/application.rs | 46 ++++++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/src/application.rs b/src/application.rs index b9fa3e32..5555ee39 100644 --- a/src/application.rs +++ b/src/application.rs @@ -192,20 +192,7 @@ impl App<()> { /// Create application with empty state. Application can /// be configured with a builder-like pattern. pub fn new() -> App<()> { - App { - parts: Some(ApplicationParts { - state: (), - prefix: "/".to_owned(), - settings: ServerSettings::default(), - default: ResourceHandler::default_not_found(), - resources: Vec::new(), - handlers: Vec::new(), - external: HashMap::new(), - encoding: ContentEncoding::Auto, - filters: Vec::new(), - middlewares: Vec::new(), - }), - } + App::with_state(()) } } @@ -737,7 +724,7 @@ impl Iterator for App { #[cfg(test)] mod tests { use super::*; - use body::{Body, Binary}; + use body::{Binary, Body}; use http::StatusCode; use httprequest::HttpRequest; use httpresponse::HttpResponse; @@ -811,7 +798,9 @@ mod tests { #[test] fn test_handler() { - let mut app = App::new().handler("/test", |_| HttpResponse::Ok()).finish(); + let mut app = App::new() + .handler("/test", |_| HttpResponse::Ok()) + .finish(); let req = TestRequest::with_uri("/test").finish(); let resp = app.run(req); @@ -836,7 +825,9 @@ mod tests { #[test] fn test_handler2() { - let mut app = App::new().handler("test", |_| HttpResponse::Ok()).finish(); + let mut app = App::new() + .handler("test", |_| HttpResponse::Ok()) + .finish(); let req = TestRequest::with_uri("/test").finish(); let resp = app.run(req); @@ -890,21 +881,29 @@ mod tests { #[test] fn test_route() { let mut app = App::new() - .route("/test", Method::GET, |_: HttpRequest| HttpResponse::Ok()) + .route("/test", Method::GET, |_: HttpRequest| { + HttpResponse::Ok() + }) .route("/test", Method::POST, |_: HttpRequest| { HttpResponse::Created() }) .finish(); - let req = TestRequest::with_uri("/test").method(Method::GET).finish(); + let req = TestRequest::with_uri("/test") + .method(Method::GET) + .finish(); let resp = app.run(req); assert_eq!(resp.as_msg().status(), StatusCode::OK); - let req = TestRequest::with_uri("/test").method(Method::POST).finish(); + let req = TestRequest::with_uri("/test") + .method(Method::POST) + .finish(); let resp = app.run(req); assert_eq!(resp.as_msg().status(), StatusCode::CREATED); - let req = TestRequest::with_uri("/test").method(Method::HEAD).finish(); + let req = TestRequest::with_uri("/test") + .method(Method::HEAD) + .finish(); let resp = app.run(req); assert_eq!(resp.as_msg().status(), StatusCode::NOT_FOUND); } @@ -973,6 +972,9 @@ mod tests { let req = TestRequest::with_uri("/some").finish(); let resp = app.run(req); assert_eq!(resp.as_msg().status(), StatusCode::OK); - assert_eq!(resp.as_msg().body(), &Body::Binary(Binary::Slice(b"some"))); + assert_eq!( + resp.as_msg().body(), + &Body::Binary(Binary::Slice(b"some")) + ); } } From ad9aacf5213440a11f5a20021586729c4b31d8a0 Mon Sep 17 00:00:00 2001 From: Dursun Akkurt Date: Wed, 13 Jun 2018 22:41:35 +0300 Subject: [PATCH 4/6] change poll method of Readlines --- src/httpmessage.rs | 96 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 79 insertions(+), 17 deletions(-) diff --git a/src/httpmessage.rs b/src/httpmessage.rs index 7f6b50c6..91ea09f6 100644 --- a/src/httpmessage.rs +++ b/src/httpmessage.rs @@ -13,6 +13,7 @@ use std::str; use error::{ ContentTypeError, HttpRangeError, ParseError, PayloadError, UrlencodedError, + Error, ErrorBadRequest }; use header::Header; use json::JsonBody; @@ -276,7 +277,7 @@ where T: HttpMessage + Stream + 'static, { req: T, - buff: Vec, + buff: BytesMut, limit: usize, } @@ -288,12 +289,12 @@ where fn new(req: T) -> Self { Readlines { req, - buff: Vec::with_capacity(262_144), + buff: BytesMut::with_capacity(262_144), limit: 262_144, } } - /// Change max size of payload. By default max size is 256Kb + /// Change max line size. By default max size is 256Kb pub fn limit(mut self, limit: usize) -> Self { self.limit = limit; self @@ -308,21 +309,63 @@ where type Error = ReadlinesError; fn poll(&mut self) -> Poll, Self::Error> { + let encoding = self.req.encoding()?; + // check if there is a newline in the buffer + let mut found: Option = None; + for (ind, b) in self.buff.iter().enumerate() { + if *b == '\n' as u8 { + found = Some(ind); + break; + } + } + if let Some(ind) = found { + // check if line is longer than limit + if ind+1 > self.limit { + return Err(ReadlinesError::LimitOverflow); + } + let enc: *const Encoding = encoding as *const Encoding; + let line = if enc == UTF_8 { + str::from_utf8(&self.buff.split_to(ind+1)) + .map_err(|_| ErrorBadRequest("Can not decode body"))? + .to_owned() + } else { + encoding + .decode(&self.buff.split_to(ind+1), DecoderTrap::Strict) + .map_err(|_| ErrorBadRequest("Can not decode body"))? + }; + return Ok(Async::Ready(Some(line))); + } + // poll req for more bytes match self.req.poll() { - Ok(Async::Ready(Some(bytes))) => { - for b in bytes.iter() { + Ok(Async::Ready(Some(mut bytes))) => { + // check if there is a newline in bytes + let mut found: Option = None; + for (ind, b) in bytes.iter().enumerate() { if *b == '\n' as u8 { - self.buff.push(*b); - let line = str::from_utf8(&*self.buff)?.to_owned(); - self.buff.clear(); - return Ok(Async::Ready(Some(line))); - } else { - self.buff.push(*b); - } - if self.limit < self.buff.len() { - return Err(ReadlinesError::LimitOverflow); + found = Some(ind); + break; } } + if let Some(ind) = found { + // check if line is longer than limit + if ind+1 > self.limit { + return Err(ReadlinesError::LimitOverflow); + } + let enc: *const Encoding = encoding as *const Encoding; + let line = if enc == UTF_8 { + str::from_utf8(&bytes.split_to(ind+1)) + .map_err(|_| ErrorBadRequest("Can not decode body"))? + .to_owned() + } else { + encoding + .decode(&bytes.split_to(ind+1), DecoderTrap::Strict) + .map_err(|_| ErrorBadRequest("Can not decode body"))? + }; + // extend buffer with rest of the bytes; + self.buff.extend_from_slice(&bytes); + return Ok(Async::Ready(Some(line))); + } + self.buff.extend_from_slice(&bytes); Ok(Async::NotReady) }, Ok(Async::NotReady) => Ok(Async::NotReady), @@ -330,7 +373,19 @@ where if self.buff.len() == 0 { return Ok(Async::Ready(None)); } - let line = str::from_utf8(&*self.buff)?.to_owned(); + if self.buff.len() > self.limit { + return Err(ReadlinesError::LimitOverflow); + } + let enc: *const Encoding = encoding as *const Encoding; + let line = if enc == UTF_8 { + str::from_utf8(&self.buff) + .map_err(|_| ErrorBadRequest("Can not decode body"))? + .to_owned() + } else { + encoding + .decode(&self.buff, DecoderTrap::Strict) + .map_err(|_| ErrorBadRequest("Can not decode body"))? + }; self.buff.clear(); return Ok(Async::Ready(Some(line))) }, @@ -343,6 +398,7 @@ pub enum ReadlinesError { EncodingError, PayloadError(PayloadError), LimitOverflow, + ContentTypeError(ContentTypeError), } impl From for ReadlinesError { @@ -351,12 +407,18 @@ impl From for ReadlinesError { } } -impl From for ReadlinesError { - fn from(_: str::Utf8Error) -> Self { +impl From for ReadlinesError { + fn from(_: Error) -> Self { ReadlinesError::EncodingError } } +impl From for ReadlinesError { + fn from(err: ContentTypeError) -> Self { + ReadlinesError::ContentTypeError(err) + } +} + /// Future that resolves to a complete http message body. pub struct MessageBody { limit: usize, From 1bee528018605f0eec143fcf6bd432e2c4b124c0 Mon Sep 17 00:00:00 2001 From: Dursun Akkurt Date: Wed, 13 Jun 2018 22:59:36 +0300 Subject: [PATCH 5/6] move ReadlinesError to error module --- src/error.rs | 26 ++++++++++++++++++++++++++ src/httpmessage.rs | 27 +-------------------------- 2 files changed, 27 insertions(+), 26 deletions(-) diff --git a/src/error.rs b/src/error.rs index bbafb1c4..4cbfe39e 100644 --- a/src/error.rs +++ b/src/error.rs @@ -590,6 +590,32 @@ impl From for JsonPayloadError { } } +/// Error type returned when reading body as lines. +pub enum ReadlinesError { + EncodingError, + PayloadError(PayloadError), + LimitOverflow, + ContentTypeError(ContentTypeError), +} + +impl From for ReadlinesError { + fn from(err: PayloadError) -> Self { + ReadlinesError::PayloadError(err) + } +} + +impl From for ReadlinesError { + fn from(_: Error) -> Self { + ReadlinesError::EncodingError + } +} + +impl From for ReadlinesError { + fn from(err: ContentTypeError) -> Self { + ReadlinesError::ContentTypeError(err) + } +} + /// Errors which can occur when attempting to interpret a segment string as a /// valid path segment. #[derive(Fail, Debug, PartialEq)] diff --git a/src/httpmessage.rs b/src/httpmessage.rs index 91ea09f6..a380f3b9 100644 --- a/src/httpmessage.rs +++ b/src/httpmessage.rs @@ -13,7 +13,7 @@ use std::str; use error::{ ContentTypeError, HttpRangeError, ParseError, PayloadError, UrlencodedError, - Error, ErrorBadRequest + Error, ErrorBadRequest, ReadlinesError }; use header::Header; use json::JsonBody; @@ -394,31 +394,6 @@ where } } -pub enum ReadlinesError { - EncodingError, - PayloadError(PayloadError), - LimitOverflow, - ContentTypeError(ContentTypeError), -} - -impl From for ReadlinesError { - fn from(err: PayloadError) -> Self { - ReadlinesError::PayloadError(err) - } -} - -impl From for ReadlinesError { - fn from(_: Error) -> Self { - ReadlinesError::EncodingError - } -} - -impl From for ReadlinesError { - fn from(err: ContentTypeError) -> Self { - ReadlinesError::ContentTypeError(err) - } -} - /// Future that resolves to a complete http message body. pub struct MessageBody { limit: usize, From cb77f7e688985fa1f8d677c2a3013994ed487a60 Mon Sep 17 00:00:00 2001 From: Dursun Akkurt Date: Thu, 14 Jun 2018 00:19:48 +0300 Subject: [PATCH 6/6] Add `HttpMessage::readlines()` --- CHANGES.md | 2 ++ src/error.rs | 10 +++--- src/httpmessage.rs | 84 ++++++++++++++++++++++++++++++++-------------- 3 files changed, 64 insertions(+), 32 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 5265d9d7..ee109fe0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,8 @@ ### Added +* Add `HttpMessage::readlines()` for reading line by line. + * Add `ClientRequestBuilder::form()` for sending `application/x-www-form-urlencoded` requests. * Add method to configure custom error handler to Form extractor. diff --git a/src/error.rs b/src/error.rs index 4cbfe39e..c272a2dc 100644 --- a/src/error.rs +++ b/src/error.rs @@ -592,9 +592,13 @@ impl From for JsonPayloadError { /// Error type returned when reading body as lines. pub enum ReadlinesError { + /// Error when decoding a line. EncodingError, + /// Payload error. PayloadError(PayloadError), + /// Line limit exceeded. LimitOverflow, + /// ContentType error. ContentTypeError(ContentTypeError), } @@ -604,12 +608,6 @@ impl From for ReadlinesError { } } -impl From for ReadlinesError { - fn from(_: Error) -> Self { - ReadlinesError::EncodingError - } -} - impl From for ReadlinesError { fn from(err: ContentTypeError) -> Self { ReadlinesError::ContentTypeError(err) diff --git a/src/httpmessage.rs b/src/httpmessage.rs index a380f3b9..82c50d77 100644 --- a/src/httpmessage.rs +++ b/src/httpmessage.rs @@ -13,7 +13,7 @@ use std::str; use error::{ ContentTypeError, HttpRangeError, ParseError, PayloadError, UrlencodedError, - Error, ErrorBadRequest, ReadlinesError + ReadlinesError }; use header::Header; use json::JsonBody; @@ -279,6 +279,7 @@ where req: T, buff: BytesMut, limit: usize, + checked_buff: bool, } impl Readlines @@ -291,6 +292,7 @@ where req, buff: BytesMut::with_capacity(262_144), limit: 262_144, + checked_buff: true, } } @@ -311,29 +313,32 @@ where fn poll(&mut self) -> Poll, Self::Error> { let encoding = self.req.encoding()?; // check if there is a newline in the buffer - let mut found: Option = None; - for (ind, b) in self.buff.iter().enumerate() { - if *b == '\n' as u8 { - found = Some(ind); - break; + if !self.checked_buff { + let mut found: Option = None; + for (ind, b) in self.buff.iter().enumerate() { + if *b == '\n' as u8 { + found = Some(ind); + break; + } } - } - if let Some(ind) = found { - // check if line is longer than limit - if ind+1 > self.limit { - return Err(ReadlinesError::LimitOverflow); + if let Some(ind) = found { + // check if line is longer than limit + if ind+1 > self.limit { + return Err(ReadlinesError::LimitOverflow); + } + let enc: *const Encoding = encoding as *const Encoding; + let line = if enc == UTF_8 { + str::from_utf8(&self.buff.split_to(ind+1)) + .map_err(|_| ReadlinesError::EncodingError)? + .to_owned() + } else { + encoding + .decode(&self.buff.split_to(ind+1), DecoderTrap::Strict) + .map_err(|_| ReadlinesError::EncodingError)? + }; + return Ok(Async::Ready(Some(line))); } - let enc: *const Encoding = encoding as *const Encoding; - let line = if enc == UTF_8 { - str::from_utf8(&self.buff.split_to(ind+1)) - .map_err(|_| ErrorBadRequest("Can not decode body"))? - .to_owned() - } else { - encoding - .decode(&self.buff.split_to(ind+1), DecoderTrap::Strict) - .map_err(|_| ErrorBadRequest("Can not decode body"))? - }; - return Ok(Async::Ready(Some(line))); + self.checked_buff = true; } // poll req for more bytes match self.req.poll() { @@ -354,15 +359,16 @@ where let enc: *const Encoding = encoding as *const Encoding; let line = if enc == UTF_8 { str::from_utf8(&bytes.split_to(ind+1)) - .map_err(|_| ErrorBadRequest("Can not decode body"))? + .map_err(|_| ReadlinesError::EncodingError)? .to_owned() } else { encoding .decode(&bytes.split_to(ind+1), DecoderTrap::Strict) - .map_err(|_| ErrorBadRequest("Can not decode body"))? + .map_err(|_| ReadlinesError::EncodingError)? }; // extend buffer with rest of the bytes; self.buff.extend_from_slice(&bytes); + self.checked_buff = false; return Ok(Async::Ready(Some(line))); } self.buff.extend_from_slice(&bytes); @@ -379,12 +385,12 @@ where let enc: *const Encoding = encoding as *const Encoding; let line = if enc == UTF_8 { str::from_utf8(&self.buff) - .map_err(|_| ErrorBadRequest("Can not decode body"))? + .map_err(|_| ReadlinesError::EncodingError)? .to_owned() } else { encoding .decode(&self.buff, DecoderTrap::Strict) - .map_err(|_| ErrorBadRequest("Can not decode body"))? + .map_err(|_| ReadlinesError::EncodingError)? }; self.buff.clear(); return Ok(Async::Ready(Some(line))) @@ -799,4 +805,30 @@ mod tests { _ => unreachable!("error"), } } + + #[test] + fn test_readlines() { + let mut req = HttpRequest::default(); + req.payload_mut().unread_data(Bytes::from_static( + b"Lorem Ipsum is simply dummy text of the printing and typesetting\n\ + industry. Lorem Ipsum has been the industry's standard dummy\n\ + Contrary to popular belief, Lorem Ipsum is not simply random text." + )); + let mut r = Readlines::new(req); + match r.poll().ok().unwrap() { + Async::Ready(Some(s)) => assert_eq!(s, + "Lorem Ipsum is simply dummy text of the printing and typesetting\n"), + _ => unreachable!("error"), + } + match r.poll().ok().unwrap() { + Async::Ready(Some(s)) => assert_eq!(s, + "industry. Lorem Ipsum has been the industry's standard dummy\n"), + _ => unreachable!("error"), + } + match r.poll().ok().unwrap() { + Async::Ready(Some(s)) => assert_eq!(s, + "Contrary to popular belief, Lorem Ipsum is not simply random text."), + _ => unreachable!("error"), + } + } }