fix: keep credentials on the host the request started from

.github/workflows/ci.yml

@@ -6,6 +6,9 @@ on:
     tags: ["**"]
   pull_request:
 
+permissions:
+  contents: read
+
 jobs:
   build:
     runs-on: ubuntu-latest

README.md

@@ -12,7 +12,7 @@ _Please note: this is not a replacement for `http.Client`, but rather a companio
         http.Client{Timeout: 5*time.Second},    // set http client
         requester.MaxConcurrent(8),             // maximum number of concurrent requests
         requester.JSON,                         // set json headers
-        requester.Header("X-AUTH", "123456789"),// set some auth header
+        requester.SecretHeader("X-AUTH", "123456789"), // set some auth header, kept on the original host and its subdomains
         requester.Logger(requester.StdLogger),  // enable logging to stdout
     )
 
@@ -31,10 +31,11 @@ _Please note: this is not a replacement for `http.Client`, but rather a companio
 *Built-in middlewares:*
 
 - `Header` - appends user-defined headers to all requests. 
+- `SecretHeader` - appends a user-defined header carrying a credential, kept on the original host only
 - `MaxConcurrent` - sets maximum concurrency
 - `Retry` - sets retry on errors and status codes
 - `JSON` - sets headers `"Content-Type": "application/json"` and `"Accept": "application/json"`
-- `BasicAuth(user, passwd string)` - adds HTTP Basic Authentication
+- `BasicAuth(user, passwd string)` - adds HTTP Basic Authentication, kept on the original host only
 
 *Interfaces for external middlewares:*
 
@@ -53,7 +54,17 @@ See examples of the usage in [_example](https://github.com/go-pkgz/requester/tre
 `Header` middleware adds user-defined headers to all requests. It expects a map of headers to be added. For example:
 
 ```go
-rq := requester.New(http.Client{}, middleware.Header("X-Auth", "123456789"))
+rq := requester.New(http.Client{}, middleware.Header("X-Trace", "6f1a2b"))
+```
+
+Headers carrying credentials, i.e. `Authorization`, `Www-Authenticate`, `Cookie`, `Cookie2`, `Proxy-Authorization` and `Proxy-Authenticate`, are not set once a followed redirect leaves the host the request started from, the same way `http.Client` treats them. Any other header is set on every hop. Hosts are compared as they are written, so the unicode and the punycode form of an internationalised host count as two hosts and the credential stays behind.
+
+### SecretHeader middleware
+
+`SecretHeader` adds a header the same way `Header` does, but treats it as a credential regardless of its name. The header is set while the redirect chain stays on the host the request started from, or on one of its subdomains, and removed once the chain leaves it. Since `http.Client` copies a header it doesn't recognise as a credential to every hop, the removal covers the whole header, so a value the caller set on the request, or one a `CheckRedirect` hook set for the destination, goes with it. Use it for custom headers carrying a secret:
+
+```go
+rq := requester.New(http.Client{}, middleware.SecretHeader("X-Auth", "123456789"))
 ```
 ### MaxConcurrent middleware
 
@@ -189,6 +200,8 @@ rq := requester.New(http.Client{}, middleware.JSON)
 rq := requester.New(http.Client{}, middleware.BasicAuth("user", "passwd"))
 ```
 
+Credentials are set while the redirect chain stays on the host the request started from, or on one of its subdomains, and left out once the chain leaves it. A credential the client itself puts in for the destination, through a `CheckRedirect` hook or a cookie jar, is not touched.
+
 ----
 
 ### Logging middleware interface

_example/go.mod

@@ -3,7 +3,7 @@ module github.com/go-pkgz/requester/_example
 go 1.23
 
 require (
-	github.com/go-pkgz/lcw v1.1.0
+	github.com/go-pkgz/lcw v1.2.0
 	github.com/go-pkgz/repeater/v2 v2.2.0
 	github.com/go-pkgz/requester v1.0.0
 )
@@ -12,10 +12,9 @@ require (
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/google/uuid v1.6.0 // indirect
-	github.com/hashicorp/errwrap v1.1.0 // indirect
-	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/golang-lru v1.0.2 // indirect
-	github.com/redis/go-redis/v9 v9.17.3 // indirect
+	github.com/redis/go-redis/v9 v9.18.0 // indirect
+	go.uber.org/atomic v1.11.0 // indirect
 )
 
 replace github.com/go-pkgz/requester => ../

_example/go.sum

@@ -1,7 +1,5 @@
-github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a h1:HbKu58rmZpUGpz5+4FfNmIU+FmZg2P3Xaj2v2bfNWmk=
-github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
-github.com/alicebob/miniredis/v2 v2.31.1 h1:7XAt0uUg3DtwEKW5ZAGa+K7FZV2DdKQo5K/6TTnfX8Y=
-github.com/alicebob/miniredis/v2 v2.31.1/go.mod h1:UB/T2Uztp7MlFSDakaX1sTXUv5CASoprx0wulRT6HBg=
+github.com/alicebob/miniredis/v2 v2.38.0 h1:nZAzCR+Lj+Vxk4ZXzm2NuKq2O33RXj1XxJ2e2uP9jiw=
+github.com/alicebob/miniredis/v2 v2.38.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
 github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
 github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
 github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
@@ -10,24 +8,25 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
-github.com/go-pkgz/lcw v1.1.0 h1:hDJdQJZf4iw19a7cTgQvF6Poz/L7mL4E7FgG5jGs4lA=
-github.com/go-pkgz/lcw v1.1.0/go.mod h1:zwT7RSxFskQsHWHJezYq6n0iTn0n4yIYRDijXq3awM0=
+github.com/go-pkgz/lcw v1.2.0 h1:P4qr2srBKuZZRPKY7xB1CJYMRVMV6vnKSTqlsuN349A=
+github.com/go-pkgz/lcw v1.2.0/go.mod h1:DbA7qyDxoL7ZWKscm6KpxDYbex59bs/PXZ00WdNil2U=
 github.com/go-pkgz/repeater/v2 v2.2.0 h1:8nZR/NaknmLfx2YMHbr78u9OL4Xj+8+romm9dz4FpMg=
 github.com/go-pkgz/repeater/v2 v2.2.0/go.mod h1:RgX5vUbLKq7PV82QUDP5pFbQS1os4Z+U9XzKymK23A8=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
-github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
-github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
-github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
-github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
 github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c=
 github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
-github.com/redis/go-redis/v9 v9.17.3 h1:fN29NdNrE17KttK5Ndf20buqfDZwGNgoUr9qjl1DQx4=
-github.com/redis/go-redis/v9 v9.17.3/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370=
+github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
+github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs=
+github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0=
 github.com/stretchr/testify v1.12.0 h1:K6Mr6jO9JICuend/5xzTM03ydSV3vdNRYAdPSukj8uI=
 github.com/stretchr/testify v1.12.0/go.mod h1:bOYBZb5qJ00vPzWfIqBUZPaxK8jWiXc6d3ErP4Ca9Gw=
-github.com/yuin/gopher-lua v1.1.0 h1:BojcDhfyDWgU2f2TOzYK/g5p2gxMrku8oupLDqlnSqE=
-github.com/yuin/gopher-lua v1.1.0/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
+github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
+github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
+github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
+github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
+go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
+go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

middleware/header.go

@@ -2,17 +2,22 @@ package middleware
 
 import (
 	"net/http"
+	"strings"
 )
 
-// Header middleware adds a header to request
+// Header middleware adds a header to request. Headers carrying credentials, i.e. Authorization, Www-Authenticate,
+// Cookie, Cookie2, Proxy-Authorization and Proxy-Authenticate, are not set once a redirect leaves the host the
+// request started from, the same way the standard client treats them. Any other header, a custom one carrying a
+// secret included, is set on every hop, see SecretHeader for the protected version.
 func Header(key, value string) func(http.RoundTripper) http.RoundTripper {
-	return func(next http.RoundTripper) http.RoundTripper {
-		fn := func(req *http.Request) (*http.Response, error) {
-			req.Header.Set(key, value)
-			return next.RoundTrip(req)
-		}
-		return RoundTripperFunc(fn)
-	}
+	return headerHandler(key, value, credentialHeader(key))
+}
+
+// SecretHeader middleware adds a header carrying a credential to request. The header is set while the redirect chain
+// stays on the host the request started from, or on one of its subdomains, and removed once the chain leaves it,
+// including a value the caller set on the request itself.
+func SecretHeader(key, value string) func(http.RoundTripper) http.RoundTripper {
+	return headerHandler(key, value, true)
 }
 
 // JSON sets Content-Type and Accept headers to json
@@ -25,13 +30,101 @@ func JSON(next http.RoundTripper) http.RoundTripper {
 	return RoundTripperFunc(fn)
 }
 
-// BasicAuth middleware adds basic auth to request
+// BasicAuth middleware adds basic auth to request. Credentials are set while the redirect chain stays on the host
+// the request started from, or on one of its subdomains, and left out once the chain leaves it.
 func BasicAuth(user, passwd string) func(http.RoundTripper) http.RoundTripper {
 	return func(next http.RoundTripper) http.RoundTripper {
 		fn := func(req *http.Request) (*http.Response, error) {
-			req.SetBasicAuth(user, passwd)
-			return next.RoundTrip(req)
+			// off the original host the client copies no Authorization from the original request, so anything in
+			// the header belongs to the destination, put in by a CheckRedirect hook, and is left as it is
+			if onOriginalHost(req) {
+				req.SetBasicAuth(user, passwd)
+			}
+			return roundTrip(next, req)
+		}
+		return RoundTripperFunc(fn)
+	}
+}
+
+// headerHandler makes the middleware setting a single header, secret ones only while the request is on its original host
+func headerHandler(key, value string, secret bool) func(http.RoundTripper) http.RoundTripper {
+	return func(next http.RoundTripper) http.RoundTripper {
+		fn := func(req *http.Request) (*http.Response, error) {
+			if secret && !onOriginalHost(req) {
+				// the client copies headers it doesn't recognise as credentials from the original request to every
+				// hop, so the caller's own value of such a key goes as well. For the recognised ones it copies
+				// nothing off the original host and what is there belongs to the destination, left alone
+				if !credentialHeader(key) {
+					req.Header.Del(key)
+				}
+				return roundTrip(next, req)
+			}
+			req.Header.Set(key, value)
+			return roundTrip(next, req)
 		}
 		return RoundTripperFunc(fn)
 	}
 }
+
+// roundTrip passes the request down the chain and fills the request in on the response if the transport below left
+// it unset, keeping the chain onOriginalHost walks complete for any custom transport
+func roundTrip(next http.RoundTripper, req *http.Request) (*http.Response, error) {
+	resp, err := next.RoundTrip(req)
+	if resp != nil && resp.Request == nil {
+		resp.Request = req
+	}
+	return resp, err //nolint:wrapcheck // the transport's error goes through the middleware as it is
+}
+
+// credentialHeader reports if the header carries credentials, the set matching the one the standard client
+// strips on a redirect to another host
+func credentialHeader(key string) bool {
+	switch http.CanonicalHeaderKey(key) {
+	case "Authorization", "Www-Authenticate", "Cookie", "Cookie2", "Proxy-Authorization", "Proxy-Authenticate":
+		return true
+	}
+	return false
+}
+
+// onOriginalHost reports if the request is still on the host the redirect chain started from, or on one of its
+// subdomains. A request outside of a redirect chain is always on its own host. Once the chain left the original
+// host the result stays negative for the rest of it, matching the standard client.
+//
+// The chain is walked over Response.Request, which the middleware fills in for the transport below it. A redirect
+// the origin still can't be established for is treated as a hop away from it, and so is a hop between the unicode
+// and the punycode form of the same internationalised host, which is compared as it is written.
+func onOriginalHost(req *http.Request) bool {
+	if req.Response == nil { // not a redirect
+		return true
+	}
+
+	origin := req
+	for origin.Response != nil {
+		if origin.Response.Request == nil { // broken chain, the origin is unknown
+			return false
+		}
+		origin = origin.Response.Request
+	}
+
+	originHost := strings.ToLower(origin.URL.Hostname())
+	for r := req; r != origin; r = r.Response.Request {
+		if !domainOrSubdomain(strings.ToLower(r.URL.Hostname()), originHost) {
+			return false
+		}
+	}
+	return true
+}
+
+// domainOrSubdomain reports whether sub is the same domain as parent or a subdomain of it
+func domainOrSubdomain(sub, parent string) bool {
+	if sub == parent {
+		return true
+	}
+	if strings.ContainsAny(sub, ":%") { // IPv6 address or a zone, never a hostname
+		return false
+	}
+	if !strings.HasSuffix(sub, parent) {
+		return false
+	}
+	return sub[len(sub)-len(parent)-1] == '.'
+}