fix: don't fail responses whose body was capped by -response-size-to-read

common/httpx/httpx.go

@@ -278,6 +278,9 @@ get_response:
 	// 304 - Not Modified => no body the response terminates with latest header newline
 	shouldSkipBodyRead := generic.EqualsAny(httpresp.StatusCode, http.StatusSwitchingProtocols, http.StatusNotModified)
 
+	// the body is capped before dumping the response to avoid loading unbounded
+	// bodies (or infinite streams) in memory
+	bodyTruncated := h.Options.MaxResponseBodySizeToRead > 0 && httpresp.ContentLength > h.Options.MaxResponseBodySizeToRead
 	if h.Options.MaxResponseBodySizeToRead > 0 {
 		httpresp.Body = io.NopCloser(io.LimitReader(httpresp.Body, h.Options.MaxResponseBodySizeToRead))
 		if !shouldSkipBodyRead {
@@ -296,6 +299,13 @@ get_response:
 			shouldIgnoreBodyErrors = true
 		}
 
+		// Serializing a response whose body was capped fails with "ContentLength=x with
+		// Body length y", although headers and the truncated body are dumped correctly.
+		// An intentional truncation must not turn a valid response into a failed one.
+		if bodyTruncated && stringsutil.ContainsAny(err.Error(), "with Body length") {
+			shouldIgnoreErrors = true
+		}
+
 		// Edge case - some servers respond with gzip encoding header but uncompressed body, in this case the standard library configures the reader as gzip, triggering an error when read.
 		// The bytes slice is not accessible because of abstraction, therefore we need to perform the request again tampering the Accept-Encoding header
 		if !gzipRetry && strings.Contains(err.Error(), "gzip: invalid header") {

common/httpx/response_memory_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"net/http"
 	"net/http/httptest"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -188,3 +189,34 @@ func TestBodyMetricsCountingDoesNotAllocate(t *testing.T) {
 	require.NotZero(t, lines)
 	require.Zerof(t, allocs, "word/line counting must not allocate, got %v allocs/op", allocs)
 }
+
+// TestDoTruncatedBodyIsNotAFailure guards against the regression where capping
+// the body before dumping the response made the dump fail with
+// "ContentLength=x with Body length y", turning a valid response into a failed
+// one (issue #2641).
+func TestDoTruncatedBodyIsNotAFailure(t *testing.T) {
+	body := bytes.Repeat([]byte("A"), 4096)
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/pdf")
+		w.Header().Set("Content-Length", strconv.Itoa(len(body)))
+		_, _ = w.Write(body)
+	}))
+	defer ts.Close()
+
+	const readSize = 10
+	options := DefaultOptions
+	options.CdnCheck = "false"
+	options.Timeout = 5 * time.Second
+	options.RetryMax = 0
+	options.MaxResponseBodySizeToRead = readSize
+
+	ht, err := New(&options)
+	require.NoError(t, err)
+
+	resp := doLocal(t, ht, ts.URL)
+
+	require.Equal(t, http.StatusOK, resp.StatusCode)
+	require.Equal(t, len(body), resp.ContentLength, "advertised content length must be reported")
+	require.Len(t, resp.Data, readSize, "body must be capped to the configured read size")
+	require.Contains(t, resp.RawHeaders, "Content-Length: "+strconv.Itoa(len(body)))
+}