diff --git a/CHANGES.md b/CHANGES.md
index ef9ee900a..5df0b6d6d 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -4,11 +4,16 @@
 ### Fixed
 * Double ampersand in Logger format is escaped correctly. [#2067]
 
+### Changed
+* `CustomResponder` would return error as `HttpResponse` when `CustomResponder::with_header` failed instead of skipping.
+  (Only the first error is kept when multiple error occur) [#2093]
+
 ### Removed
 * The `client` mod was removed. Clients should now use `awc` directly.
   [871ca5e4](https://github.com/actix/actix-web/commit/871ca5e4ae2bdc22d1ea02701c2992fa8d04aed7)
 
 [#2067]: https://github.com/actix/actix-web/pull/2067
+[#2093]: https://github.com/actix/actix-web/pull/2093
 
 
 ## 4.0.0-beta.4 - 2021-03-09
diff --git a/src/responder.rs b/src/responder.rs
index 92945cdaa..0b6f47c6b 100644
--- a/src/responder.rs
+++ b/src/responder.rs
@@ -155,8 +155,7 @@ impl Responder for BytesMut {
 pub struct CustomResponder<T> {
     responder: T,
     status: Option<StatusCode>,
-    headers: Option<HeaderMap>,
-    error: Option<HttpError>,
+    headers: Result<HeaderMap, HttpError>,
 }
 
 impl<T: Responder> CustomResponder<T> {
@@ -164,8 +163,7 @@ impl<T: Responder> CustomResponder<T> {
         CustomResponder {
             responder,
             status: None,
-            headers: None,
-            error: None,
+            headers: Ok(HeaderMap::new()),
         }
     }
 
@@ -206,32 +204,33 @@ impl<T: Responder> CustomResponder<T> {
     where
         H: IntoHeaderPair,
     {
-        if self.headers.is_none() {
-            self.headers = Some(HeaderMap::new());
+        if let Ok(ref mut headers) = self.headers {
+            match header.try_into_header_pair() {
+                Ok((key, value)) => headers.append(key, value),
+                Err(e) => self.headers = Err(e.into()),
+            };
         }
 
-        match header.try_into_header_pair() {
-            Ok((key, value)) => self.headers.as_mut().unwrap().append(key, value),
-            Err(e) => self.error = Some(e.into()),
-        };
-
         self
     }
 }
 
 impl<T: Responder> Responder for CustomResponder<T> {
     fn respond_to(self, req: &HttpRequest) -> HttpResponse {
+        let headers = match self.headers {
+            Ok(headers) => headers,
+            Err(err) => return HttpResponse::from_error(Error::from(err)),
+        };
+
         let mut res = self.responder.respond_to(req);
 
         if let Some(status) = self.status {
             *res.status_mut() = status;
         }
 
-        if let Some(ref headers) = self.headers {
-            for (k, v) in headers {
-                // TODO: before v4, decide if this should be append instead
-                res.headers_mut().insert(k.clone(), v.clone());
-            }
+        for (k, v) in headers {
+            // TODO: before v4, decide if this should be append instead
+            res.headers_mut().insert(k, v);
         }
 
         res