[BUG] CompressHandler if combined with http.Fileserver misses "Content-Encoding: deflate" on 404 status codes

[Lercher]

Is there an existing issue for this?

• I have searched the existing issues

Current Behavior

curl --output - -i --compressed http://localhost:9000/notfound.html outputs

HTTP/1.1 404 Not Found
Content-Type: text/plain; charset=utf-8
Vary: Accept-Encoding
X-Content-Type-Options: nosniff
Date: Mon, 18 Nov 2024 13:26:14 GMT
Content-Length: 25

210Q(HLOU��/QH�/�K�☻♦��

for the program listed in "steps to reproduce".

Expected Behavior

It should output either "deflate and len 25 bytes"

HTTP/1.1 404 Not Found
Content-Encoding: deflate
Content-Type: text/plain; charset=utf-8
Vary: Accept-Encoding
X-Content-Type-Options: nosniff
Date: Mon, 18 Nov 2024 12:52:02 GMT
Content-Length: 25

404 page not found

or "not compressed and len 19 bytes"

HTTP/1.1 404 Not Found
Content-Type: text/plain; charset=utf-8
X-Content-Type-Options: nosniff
Date: Mon, 18 Nov 2024 12:54:26 GMT
Content-Length: 19

404 page not found

Steps To Reproduce

• go version go1.23.3 windows/amd64
• github.com/gorilla/handlers v1.5.2
• github.com/gorilla/mux v1.8.1

package main

import (
	"fmt"
	"log"
	"net/http"

	"github.com/gorilla/handlers"
	"github.com/gorilla/mux"
)

func main() {
	log.Println("Listening on http://localhost:9000/index.html")

	mux := mux.NewRouter()
	mux.Use(handlers.CompressHandler)

	mux.HandleFunc("/index.html", func(w http.ResponseWriter, r *http.Request) {
		_, _ = fmt.Fprintln(w, `<a href="/error404.html">Click me for a 404 error.</a> or use <pre>curl --output - -i --compressed http://localhost:9000</pre>`)
	})
	mux.PathPrefix("/").Methods("GET").Handler(http.FileServer(http.Dir(".")))

	log.Fatal(http.ListenAndServe(":9000", mux))
}

The "expected behavior" is present if the line mux.PathPrefix("/").Methods("GET").Handler(http.FileServer(http.Dir("."))) is commented out. It degrades to the "current behavior" if the FileServer is activated.

It works in both cases if a normal OK status code is produced (/index.html).

Chrome Dev tools behave similarly but different in details b/c of gzip / deflate request headers, I guess.

Anything else?

Sorry for not investigating more detail for the sample program. My more complex code worked with Go 1.22.x and handlers v1.5.1 and I don't know if it's an issue of the newer Go std lib, of gorilla/mux or gorilla/handlers.

Thanks
Martin

[Lercher]

More analysis: I guess, it's b/c of this Go std lib code in C:\Program Files\Go\src\net\http\fs.go which explicitly resets the header field "Content-Encoding":

// serveError serves an error from ServeFile, ServeFileFS, and ServeContent.
// Because those can all be configured by the caller by setting headers like
// Etag, Last-Modified, and Cache-Control to send on a successful response,
// the error path needs to clear them, since they may not be meant for errors.
func serveError(w ResponseWriter, text string, code int) {
	h := w.Header()

	nonDefault := false
	for _, k := range []string{
		"Cache-Control",
		"Content-Encoding",                               // <-------------- this line is "offensive"
		"Etag",
		"Last-Modified",
	} {
		if !h.has(k) {
			continue
		}
		if httpservecontentkeepheaders.Value() == "1" {
			nonDefault = true
		} else {
			h.Del(k)
		}
	}
	if nonDefault {
		httpservecontentkeepheaders.IncNonDefault()
	}

	Error(w, text, code)
}

So, I'm really unsure what indeed should be the correct behavior for the compress handler. Maybe it's defensive to just turn off compression in case of non OK http Status codes. Or, to add the compression-algorithm-compatible "Content-Encoding" header later, i.e. right before emitting the compressed body.

Or as the client app, honor this:

// GODEBUG=httpservecontentkeepheaders=1 restores the pre-1.23 behavior of not deleting
// Cache-Control, Content-Encoding, Etag, or Last-Modified headers on ServeContent errors.
var httpservecontentkeepheaders = godebug.New("httpservecontentkeepheaders")

[aronatkins]

This is a change with Go 1.23; the Content-Encoding header is removed from error responses. The Go release notes recommend Transfer-Encoding.

The previous behavior can be returned by setting GODEBUG=httpservecontentkeepheaders=1

https://go.dev/doc/go1.23#nethttppkgnethttp

[isaacwein]

do to this problem

i ended up writing my own gzip middleware
if the status code is more then 400 it will not compress the respose

package middleware

import (
	"bufio"
	"compress/gzip"
	"io"
	"net"
	"net/http"
	"path/filepath"
	"strings"
)

// isCompressibleContentType returns true if the given content type is likely text.
func isCompressibleContentType(ct string) bool {
	// If empty, assume compressible.
	if ct == "" {
		return true
	}
	ct = strings.ToLower(ct)
	// Text types and common compressible MIME types.
	if strings.HasPrefix(ct, "text/") ||
		strings.HasPrefix(ct, "application/json") ||
		strings.HasPrefix(ct, "application/javascript") ||
		strings.HasPrefix(ct, "application/xml") ||
		strings.HasPrefix(ct, "image/svg") ||
		strings.HasPrefix(ct, "application/x-javascript") {
		return true
	}
	// Otherwise (e.g. image, video) assume not compressible.
	return false
}

// GzipMiddleware is a middleware that compresses the response using gzip.
type gzipMiddlewareStruct struct {
	Next http.Handler
}

// GzipMiddleware returns a new GzipMiddleware.
func GzipMiddleware(h http.Handler) http.Handler {
	return &gzipMiddlewareStruct{h}
}

// ServeHTTP compresses the response using gzip.
func (gm *gzipMiddlewareStruct) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if gm.Next == nil {
		gm.Next = http.DefaultServeMux
	}

	switch strings.ToUpper(filepath.Ext(r.URL.Path)) {
	case ".MP3", ".MP4", ".WMA", ".WAV", ".WMV", ".APK":
		gm.Next.ServeHTTP(w, r)
		return
	}

	// Check if the client accepts gzip.
	if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
		gm.Next.ServeHTTP(w, r)
		return
	}

	gzipWriter := gzip.NewWriter(w)

	rw := &gzipResponseWriter{
		responseWriter: w,
		compressor:     gzipWriter,
	}
	defer rw.Close()
	gm.Next.ServeHTTP(rw, r)
}

// gzipResponseWriter wraps http.ResponseWriter to intercept the status code
// and the headers so we can decide whether to compress.
type gzipResponseWriter struct {
	responseWriter     http.ResponseWriter
	compressor         *gzip.Writer
	status             int
	disableCompression bool
}

func (grw *gzipResponseWriter) Header() http.Header {
	return grw.responseWriter.Header()
}

// WriteHeader captures the status code and checks the Content-Type.
// If the status is >= 400 or the Content-Type indicates binary data, disable compression.
func (grw *gzipResponseWriter) WriteHeader(code int) {
	grw.status = code

	// Disable compression for error responses.
	if code >= http.StatusBadRequest {
		grw.disableCompression = true
		grw.responseWriter.Header().Del("Content-Encoding")
	} else {
		// Check Content-Type header if available.
		ct := grw.responseWriter.Header().Get("Content-Type")
		if !isCompressibleContentType(ct) {
			grw.disableCompression = true
			grw.responseWriter.Header().Del("Content-Encoding")
		} else {
			// Set Content-Encoding only if compression is enabled.
			grw.responseWriter.Header().Set("Content-Encoding", "gzip")
		}
	}

	grw.responseWriter.WriteHeader(code)
}

// Write writes data to the connection. If compression is disabled, write directly.
func (grw *gzipResponseWriter) Write(data []byte) (int, error) {
	// if you try to write without WriteHeader() go will write it with out calling WriteHeader()
	if grw.status == 0 {
		grw.WriteHeader(http.StatusOK)
	}

	if grw.disableCompression {
		return grw.responseWriter.Write(data)
	}

	return grw.compressor.Write(data)
}

// making a custom closer to run only if the gzip writer was used
func (grw *gzipResponseWriter) Close() error {
	// if there is no body the system go will call the 200 and it will send gzip bytes without the gzip headers
	if grw.status == 0 {
		grw.WriteHeader(http.StatusOK)
	}

	if grw.disableCompression {
		if f, ok := grw.responseWriter.(io.Closer); ok {
			return f.Close()
		}
		return nil
	}
	return grw.compressor.Close()
}

func (grw *gzipResponseWriter) Flush() {
	if f, ok := grw.responseWriter.(http.Flusher); ok {
		f.Flush()
	}
}

func (grw *gzipResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
	if hj, ok := grw.responseWriter.(http.Hijacker); ok {
		return hj.Hijack()
	}
	return nil, nil, http.ErrNotSupported
}

func (grw *gzipResponseWriter) Push(target string, opts *http.PushOptions) error {
	if p, ok := grw.responseWriter.(http.Pusher); ok {
		return p.Push(target, opts)
	}
	return http.ErrNotSupported
}

// Ensure our gzipResponseWriter implements the necessary interfaces.
var _ io.Closer = &gzipResponseWriter{}
var _ http.ResponseWriter = &gzipResponseWriter{}
var _ http.Pusher = &gzipResponseWriter{}
var _ http.Hijacker = &gzipResponseWriter{}
var _ http.Flusher = &gzipResponseWriter{}

[aronatkins]

I also wrote my own middleware, but relied on the incoming Content-Encoding to determine if it should restore an outgoing Content-Encoding.

import (
	"net/http"
)

// compressHandler is an HTTP handler that adds the Content-Encoding header
// back to responses when removed by the http.FileServer.
//
// handlers.CompressHandler(newCompressHandler(http.FileServer(...)))
type compressHandler struct {
	// handler is an HTTP handler, usually an http.FileServer.
	handler http.Handler
}

var _ http.Handler = &compressHandler{}

func newCompressHandler(handler http.Handler) *compressHandler {
	return &compressHandler{
		handler: handler,
	}
}

func (h *compressHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	// The wrapped response writer saves the incoming content encoding so
	// it can be restored when writing the response headers.
	cw := &compressedResponseWriter{
		encoding:       w.Header().Get("Content-Encoding"),
		fixed:          false,
		responseWriter: w,
	}
	h.handler.ServeHTTP(cw, r)
}

// compressedResponseWriter is an http.ResponseWriter that ensures that a
// previously-set Content-Encoding header is in place before writing the
// response.
type compressedResponseWriter struct {
	encoding       string
	fixed          bool
	responseWriter http.ResponseWriter
}

var _ http.ResponseWriter = &compressedResponseWriter{}

func (w *compressedResponseWriter) Header() http.Header {
	return w.responseWriter.Header()
}

func (w *compressedResponseWriter) fixContentEncoding() {
	if w.fixed {
		return
	}
	w.fixed = true
	// The Go 1.23 http.FileServer() removes headers like Content-Encoding
	// from error responses. This breaks gzip and deflate encoding.
	// https://github.com/gorilla/handlers/issues/259
	// https://github.com/golang/go/issues/66343
	if w.encoding == "gzip" || w.encoding == "deflate" {
		if w.Header().Get("Content-Encoding") == "" {
			w.Header().Set("Content-Encoding", w.encoding)
		}
	}
}

func (w *compressedResponseWriter) Write(data []byte) (int, error) {
	w.fixContentEncoding()
	return w.responseWriter.Write(data)
}

func (w *compressedResponseWriter) WriteHeader(statusCode int) {
	w.fixContentEncoding()
	w.responseWriter.WriteHeader(statusCode)
}