From c75ee0c9f32ff56450e88a5e12236711e1e48900 Mon Sep 17 00:00:00 2001 From: Nakul Bharti Date: Thu, 3 Sep 2026 00:31:41 +0700 Subject: [PATCH 01/12] fix: retry over https when a plaintext probe is rejected for missing TLS Ports above 1024 are probed as plain HTTP first, and the scheme retry only fires on a transport error. A TLS listener answers a plaintext request with a perfectly valid HTTP 400 saying TLS is required, so err == nil, the retry never happens, and a TLS-only service is reported as plain http with a 400 and no title or technologies. Detect that response and reuse the existing scheme retry. Only the distinctive server phrasings count (nginx, Netty, Apache, HAProxy); a bare "400 Bad Request" is a legitimate HTTP answer and is left alone, so the extra request is limited to targets that already told us to use TLS. Downstream this mattered: consumers that persist the probed scheme were recording TLS-only ports as http assets, and every HTTP-based scan of those targets then ran over plaintext and matched nothing. Claude-Session: https://claude.ai/code/session_015AKXsNzjd3HjGMq4B8JCNH Claude-Local-Session: claude --resume 0fe5b680-5ba5-43e6-b354-43d4388aaf9c --- runner/ports_optimization.go | 29 ++++++ runner/ports_optimization_test.go | 75 ++++++++++++++ runner/runner.go | 10 ++ runner/tls_retry_live_test.go | 157 ++++++++++++++++++++++++++++++ 4 files changed, 271 insertions(+) create mode 100644 runner/tls_retry_live_test.go diff --git a/runner/ports_optimization.go b/runner/ports_optimization.go index 4ccb9e71b..048cc1ca1 100644 --- a/runner/ports_optimization.go +++ b/runner/ports_optimization.go @@ -2,10 +2,12 @@ package runner import ( "net" + "net/http" "strconv" "github.com/projectdiscovery/httpx/common/httpx" sliceutil "github.com/projectdiscovery/utils/slice" + stringsutil "github.com/projectdiscovery/utils/strings" ) var commonHttpPorts = []string{ @@ -31,3 +33,30 @@ func determineMostLikelySchemeOrder(input string) string { return httpx.HTTPS } + +// tlsRequiredSignals are the phrases a TLS listener returns when a plaintext +// request reaches it. They arrive as a perfectly valid HTTP 400, so nothing in +// the transport layer flags them and the scheme retry never fires. +var tlsRequiredSignals = []string{ + "plain http request was sent to https port", + "http request was sent to an https port", + "combination of host and port requires tls", + "speaking plain http to an ssl-enabled server port", + "client sent an http request to an https server", +} + +// respondsOnlyOverTLS reports whether a plaintext response is a TLS listener +// rejecting the request rather than a real HTTP service. A bare +// "400 Bad Request" is a legitimate HTTP answer and is deliberately not +// matched, so only the distinctive server phrasings trigger a scheme retry. +func respondsOnlyOverTLS(resp *httpx.Response) bool { + if resp == nil || resp.StatusCode != http.StatusBadRequest { + return false + } + + haystack := resp.Raw + if haystack == "" { + haystack = string(resp.Data) + } + return stringsutil.ContainsAnyI(haystack, tlsRequiredSignals...) +} diff --git a/runner/ports_optimization_test.go b/runner/ports_optimization_test.go index 232db95ed..bc02eae33 100644 --- a/runner/ports_optimization_test.go +++ b/runner/ports_optimization_test.go @@ -113,3 +113,78 @@ func getPortForFallback(currentPort, currentProtocol string) string { return currentPort } +func TestRespondsOnlyOverTLS(t *testing.T) { + tests := []struct { + name string + response *httpx.Response + expected bool + }{ + { + name: "nil response", + response: nil, + expected: false, + }, + { + name: "netty/teamcity rejection", + response: &httpx.Response{ + StatusCode: 400, + Raw: "HTTP/1.1 400 Bad Request\r\n\r\nBad Request\r\nThis combination of host and port requires TLS.\r\n", + }, + expected: true, + }, + { + name: "nginx rejection", + response: &httpx.Response{ + StatusCode: 400, + Raw: "HTTP/1.1 400 Bad Request\r\n\r\n

400 The plain HTTP request was sent to HTTPS port

", + }, + expected: true, + }, + { + name: "apache rejection", + response: &httpx.Response{ + StatusCode: 400, + Raw: "HTTP/1.1 400 Bad Request\r\n\r\nYou're speaking plain HTTP to an SSL-enabled server port.", + }, + expected: true, + }, + { + name: "haproxy rejection", + response: &httpx.Response{ + StatusCode: 400, + Raw: "HTTP/1.1 400 Bad Request\r\n\r\nClient sent an HTTP request to an HTTPS server.", + }, + expected: true, + }, + { + name: "falls back to decoded data when raw is empty", + response: &httpx.Response{ + StatusCode: 400, + Data: []byte("This combination of host and port requires TLS."), + }, + expected: true, + }, + { + name: "genuine bad request is not a TLS rejection", + response: &httpx.Response{ + StatusCode: 400, + Raw: "HTTP/1.1 400 Bad Request\r\n\r\nBad Request", + }, + expected: false, + }, + { + name: "phrase present on a successful response", + response: &httpx.Response{ + StatusCode: 200, + Raw: "HTTP/1.1 200 OK\r\n\r\nDocs: the plain HTTP request was sent to HTTPS port", + }, + expected: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.expected, respondsOnlyOverTLS(tc.response)) + }) + } +} diff --git a/runner/runner.go b/runner/runner.go index 63dccb05d..df2856cdc 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1927,6 +1927,16 @@ retry: if r.options.ShowStatistics { r.stats.IncrementCounter("requests", 1) } + // A TLS listener answers a plaintext request with a valid HTTP response + // saying TLS is required. That is not a transport error, so the scheme + // retry further down never fires and the service ends up recorded as + // plain http. Treat it as a reason to retry over https. + if err == nil && !retried && origProtocol == httpx.HTTPorHTTPS && + protocol == httpx.HTTP && respondsOnlyOverTLS(resp) { + protocol = httpx.HTTPS + retried = true + goto retry + } var requestDump []byte if scanopts.Unsafe { var errDump error diff --git a/runner/tls_retry_live_test.go b/runner/tls_retry_live_test.go new file mode 100644 index 000000000..51c59256b --- /dev/null +++ b/runner/tls_retry_live_test.go @@ -0,0 +1,157 @@ +package runner + +import ( + "bufio" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "io" + "math/big" + "net" + "net/http" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// startTLSOnlyListener serves content over TLS and answers any plaintext +// request the way a real TLS listener does: a valid HTTP 400 saying TLS is +// required. The response is a successful HTTP transaction, which is what stops +// the transport-error scheme retry from firing. +func startTLSOnlyListener(t *testing.T) string { + t.Helper() + + const rejection = "HTTP/1.1 400 Bad Request\r\n" + + "Connection: close\r\n" + + "Content-Type: text/plain;charset=utf-8\r\n" + + "Content-Length: 62\r\n\r\n" + + "Bad Request\r\nThis combination of host and port requires TLS.\r\n" + + key, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + + template := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "tls-only.test"}, + DNSNames: []string{"localhost", "tls-only.test"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + der, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key) + require.NoError(t, err) + + tlsConfig := &tls.Config{Certificates: []tls.Certificate{{Certificate: [][]byte{der}, PrivateKey: key}}} + + mux := http.NewServeMux() + mux.HandleFunc("/", func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/html") + _, _ = io.WriteString(w, "Only Over TLSok") + }) + server := &http.Server{Handler: mux, ReadHeaderTimeout: 5 * time.Second} + + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = listener.Close() }) + + go func() { + for { + conn, err := listener.Accept() + if err != nil { + return + } + go func(conn net.Conn) { + buffered := bufio.NewReader(conn) + first, err := buffered.Peek(1) + if err != nil { + _ = conn.Close() + return + } + // 0x16 is a TLS handshake record; anything else is plaintext. + if first[0] != 0x16 { + _, _ = io.WriteString(conn, rejection) + _ = conn.Close() + return + } + server.Serve(oneShot(tls.Server(&peeked{Conn: conn, reader: buffered}, tlsConfig))) + }(conn) + } + }() + + return listener.Addr().String() +} + +type peeked struct { + net.Conn + reader *bufio.Reader +} + +func (c *peeked) Read(b []byte) (int, error) { return c.reader.Read(b) } + +type oneShotListener struct { + conn net.Conn + used bool +} + +func oneShot(conn net.Conn) net.Listener { return &oneShotListener{conn: conn} } + +func (l *oneShotListener) Accept() (net.Conn, error) { + if l.used { + return nil, io.EOF + } + l.used = true + return l.conn, nil +} +func (l *oneShotListener) Close() error { return nil } +func (l *oneShotListener) Addr() net.Addr { return l.conn.LocalAddr() } + +// TestTLSOnlyPortIsProbedOverHTTPS covers a TLS-only service on a port above +// 1024, which the scheme heuristic probes as plain HTTP first. Without the +// retry on a TLS-required response the service is reported as plain http. +func TestTLSOnlyPortIsProbedOverHTTPS(t *testing.T) { + target := startTLSOnlyListener(t) + + var ( + mu sync.Mutex + results []Result + ) + + options := &Options{ + Threads: 1, + RateLimit: 10, + Retries: 0, + Timeout: 5, + Methods: http.MethodGet, + Delay: -1, + OnResult: func(r Result) { + if r.Err != nil || r.URL == "" { + return + } + mu.Lock() + results = append(results, r) + mu.Unlock() + }, + InputTargetHost: []string{target}, + } + + // The heuristic must pick http first, otherwise this test proves nothing. + require.Equal(t, "http", determineMostLikelySchemeOrder(target)) + + r, err := New(options) + require.NoError(t, err) + defer r.Close() + r.RunEnumeration() + + mu.Lock() + defer mu.Unlock() + + require.Len(t, results, 1) + require.Equal(t, "https", results[0].Scheme) + require.Equal(t, "https://"+target, results[0].URL) + require.Equal(t, http.StatusOK, results[0].StatusCode) +} From 9c9ef009863a235913822d945be5909c55396051 Mon Sep 17 00:00:00 2001 From: Nakul Bharti Date: Thu, 3 Sep 2026 00:42:40 +0700 Subject: [PATCH 02/12] refactor: fold the TLS retry test into runner_test.go Tests for runner.go belong in runner_test.go; a per-scenario file drifts away from the code it covers. Also cut the four-line preamble on the retry down to the one fact that is not already on the next line. Claude-Session: https://claude.ai/code/session_015AKXsNzjd3HjGMq4B8JCNH Claude-Local-Session: claude --resume 0fe5b680-5ba5-43e6-b354-43d4388aaf9c --- runner/runner.go | 5 +- runner/runner_test.go | 148 ++++++++++++++++++++++++++++++++ runner/tls_retry_live_test.go | 157 ---------------------------------- 3 files changed, 149 insertions(+), 161 deletions(-) delete mode 100644 runner/tls_retry_live_test.go diff --git a/runner/runner.go b/runner/runner.go index df2856cdc..bc0f505ef 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1927,10 +1927,7 @@ retry: if r.options.ShowStatistics { r.stats.IncrementCounter("requests", 1) } - // A TLS listener answers a plaintext request with a valid HTTP response - // saying TLS is required. That is not a transport error, so the scheme - // retry further down never fires and the service ends up recorded as - // plain http. Treat it as a reason to retry over https. + // A TLS-required 400 is a successful transaction, so the retry below never fires. if err == nil && !retried && origProtocol == httpx.HTTPorHTTPS && protocol == httpx.HTTP && respondsOnlyOverTLS(resp) { protocol = httpx.HTTPS diff --git a/runner/runner_test.go b/runner/runner_test.go index 5a847852e..596560b42 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -1,7 +1,17 @@ package runner import ( + "bufio" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" "fmt" + "io" + "math/big" + "net" + "net/http" "os" "strings" "sync" @@ -768,3 +778,141 @@ func TestCreateNetworkpolicyInstance_AllowDenyFlags(t *testing.T) { }) } } + +// startTLSOnlyListener serves content over TLS and answers any plaintext +// request the way a real TLS listener does: a valid HTTP 400 saying TLS is +// required. The response is a successful HTTP transaction, which is what stops +// the transport-error scheme retry from firing. +func startTLSOnlyListener(t *testing.T) string { + t.Helper() + + const rejection = "HTTP/1.1 400 Bad Request\r\n" + + "Connection: close\r\n" + + "Content-Type: text/plain;charset=utf-8\r\n" + + "Content-Length: 62\r\n\r\n" + + "Bad Request\r\nThis combination of host and port requires TLS.\r\n" + + key, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + + template := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "tls-only.test"}, + DNSNames: []string{"localhost", "tls-only.test"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + der, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key) + require.NoError(t, err) + + tlsConfig := &tls.Config{Certificates: []tls.Certificate{{Certificate: [][]byte{der}, PrivateKey: key}}} + + mux := http.NewServeMux() + mux.HandleFunc("/", func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/html") + _, _ = io.WriteString(w, "Only Over TLSok") + }) + server := &http.Server{Handler: mux, ReadHeaderTimeout: 5 * time.Second} + + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = listener.Close() }) + + go func() { + for { + conn, err := listener.Accept() + if err != nil { + return + } + go func(conn net.Conn) { + buffered := bufio.NewReader(conn) + first, err := buffered.Peek(1) + if err != nil { + _ = conn.Close() + return + } + // 0x16 is a TLS handshake record; anything else is plaintext. + if first[0] != 0x16 { + _, _ = io.WriteString(conn, rejection) + _ = conn.Close() + return + } + server.Serve(oneShot(tls.Server(&peeked{Conn: conn, reader: buffered}, tlsConfig))) + }(conn) + } + }() + + return listener.Addr().String() +} + +type peeked struct { + net.Conn + reader *bufio.Reader +} + +func (c *peeked) Read(b []byte) (int, error) { return c.reader.Read(b) } + +type oneShotListener struct { + conn net.Conn + used bool +} + +func oneShot(conn net.Conn) net.Listener { return &oneShotListener{conn: conn} } + +func (l *oneShotListener) Accept() (net.Conn, error) { + if l.used { + return nil, io.EOF + } + l.used = true + return l.conn, nil +} +func (l *oneShotListener) Close() error { return nil } +func (l *oneShotListener) Addr() net.Addr { return l.conn.LocalAddr() } + +// TestTLSOnlyPortIsProbedOverHTTPS covers a TLS-only service on a port above +// 1024, which the scheme heuristic probes as plain HTTP first. Without the +// retry on a TLS-required response the service is reported as plain http. +func TestTLSOnlyPortIsProbedOverHTTPS(t *testing.T) { + target := startTLSOnlyListener(t) + + var ( + mu sync.Mutex + results []Result + ) + + options := &Options{ + Threads: 1, + RateLimit: 10, + Retries: 0, + Timeout: 5, + Methods: http.MethodGet, + Delay: -1, + OnResult: func(r Result) { + if r.Err != nil || r.URL == "" { + return + } + mu.Lock() + results = append(results, r) + mu.Unlock() + }, + InputTargetHost: []string{target}, + } + + // The heuristic must pick http first, otherwise this test proves nothing. + require.Equal(t, "http", determineMostLikelySchemeOrder(target)) + + r, err := New(options) + require.NoError(t, err) + defer r.Close() + r.RunEnumeration() + + mu.Lock() + defer mu.Unlock() + + require.Len(t, results, 1) + require.Equal(t, "https", results[0].Scheme) + require.Equal(t, "https://"+target, results[0].URL) + require.Equal(t, http.StatusOK, results[0].StatusCode) +} diff --git a/runner/tls_retry_live_test.go b/runner/tls_retry_live_test.go deleted file mode 100644 index 51c59256b..000000000 --- a/runner/tls_retry_live_test.go +++ /dev/null @@ -1,157 +0,0 @@ -package runner - -import ( - "bufio" - "crypto/rand" - "crypto/rsa" - "crypto/tls" - "crypto/x509" - "crypto/x509/pkix" - "io" - "math/big" - "net" - "net/http" - "sync" - "testing" - "time" - - "github.com/stretchr/testify/require" -) - -// startTLSOnlyListener serves content over TLS and answers any plaintext -// request the way a real TLS listener does: a valid HTTP 400 saying TLS is -// required. The response is a successful HTTP transaction, which is what stops -// the transport-error scheme retry from firing. -func startTLSOnlyListener(t *testing.T) string { - t.Helper() - - const rejection = "HTTP/1.1 400 Bad Request\r\n" + - "Connection: close\r\n" + - "Content-Type: text/plain;charset=utf-8\r\n" + - "Content-Length: 62\r\n\r\n" + - "Bad Request\r\nThis combination of host and port requires TLS.\r\n" - - key, err := rsa.GenerateKey(rand.Reader, 2048) - require.NoError(t, err) - - template := x509.Certificate{ - SerialNumber: big.NewInt(1), - Subject: pkix.Name{CommonName: "tls-only.test"}, - DNSNames: []string{"localhost", "tls-only.test"}, - NotBefore: time.Now().Add(-time.Hour), - NotAfter: time.Now().Add(time.Hour), - KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - } - der, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key) - require.NoError(t, err) - - tlsConfig := &tls.Config{Certificates: []tls.Certificate{{Certificate: [][]byte{der}, PrivateKey: key}}} - - mux := http.NewServeMux() - mux.HandleFunc("/", func(w http.ResponseWriter, _ *http.Request) { - w.Header().Set("Content-Type", "text/html") - _, _ = io.WriteString(w, "Only Over TLSok") - }) - server := &http.Server{Handler: mux, ReadHeaderTimeout: 5 * time.Second} - - listener, err := net.Listen("tcp", "127.0.0.1:0") - require.NoError(t, err) - t.Cleanup(func() { _ = listener.Close() }) - - go func() { - for { - conn, err := listener.Accept() - if err != nil { - return - } - go func(conn net.Conn) { - buffered := bufio.NewReader(conn) - first, err := buffered.Peek(1) - if err != nil { - _ = conn.Close() - return - } - // 0x16 is a TLS handshake record; anything else is plaintext. - if first[0] != 0x16 { - _, _ = io.WriteString(conn, rejection) - _ = conn.Close() - return - } - server.Serve(oneShot(tls.Server(&peeked{Conn: conn, reader: buffered}, tlsConfig))) - }(conn) - } - }() - - return listener.Addr().String() -} - -type peeked struct { - net.Conn - reader *bufio.Reader -} - -func (c *peeked) Read(b []byte) (int, error) { return c.reader.Read(b) } - -type oneShotListener struct { - conn net.Conn - used bool -} - -func oneShot(conn net.Conn) net.Listener { return &oneShotListener{conn: conn} } - -func (l *oneShotListener) Accept() (net.Conn, error) { - if l.used { - return nil, io.EOF - } - l.used = true - return l.conn, nil -} -func (l *oneShotListener) Close() error { return nil } -func (l *oneShotListener) Addr() net.Addr { return l.conn.LocalAddr() } - -// TestTLSOnlyPortIsProbedOverHTTPS covers a TLS-only service on a port above -// 1024, which the scheme heuristic probes as plain HTTP first. Without the -// retry on a TLS-required response the service is reported as plain http. -func TestTLSOnlyPortIsProbedOverHTTPS(t *testing.T) { - target := startTLSOnlyListener(t) - - var ( - mu sync.Mutex - results []Result - ) - - options := &Options{ - Threads: 1, - RateLimit: 10, - Retries: 0, - Timeout: 5, - Methods: http.MethodGet, - Delay: -1, - OnResult: func(r Result) { - if r.Err != nil || r.URL == "" { - return - } - mu.Lock() - results = append(results, r) - mu.Unlock() - }, - InputTargetHost: []string{target}, - } - - // The heuristic must pick http first, otherwise this test proves nothing. - require.Equal(t, "http", determineMostLikelySchemeOrder(target)) - - r, err := New(options) - require.NoError(t, err) - defer r.Close() - r.RunEnumeration() - - mu.Lock() - defer mu.Unlock() - - require.Len(t, results, 1) - require.Equal(t, "https", results[0].Scheme) - require.Equal(t, "https://"+target, results[0].URL) - require.Equal(t, http.StatusOK, results[0].StatusCode) -} From 52a53529b895562d79f3aaeaf38545e448a18fa6 Mon Sep 17 00:00:00 2001 From: Nakul Bharti Date: Thu, 3 Sep 2026 00:47:07 +0700 Subject: [PATCH 03/12] fix(test): check the error from server.Serve errcheck flagged the unchecked Serve in the TLS-only test listener. It always returns io.EOF there, since the listener yields a single connection, so discard it explicitly. Claude-Session: https://claude.ai/code/session_015AKXsNzjd3HjGMq4B8JCNH Claude-Local-Session: claude --resume 0fe5b680-5ba5-43e6-b354-43d4388aaf9c --- runner/runner_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/runner/runner_test.go b/runner/runner_test.go index 596560b42..664e969be 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -839,7 +839,8 @@ func startTLSOnlyListener(t *testing.T) string { _ = conn.Close() return } - server.Serve(oneShot(tls.Server(&peeked{Conn: conn, reader: buffered}, tlsConfig))) + // Always returns io.EOF: the listener yields this one connection. + _ = server.Serve(oneShot(tls.Server(&peeked{Conn: conn, reader: buffered}, tlsConfig))) }(conn) } }() From c2d9c6bbba786839768d135fbaa65cbd80a9dd74 Mon Sep 17 00:00:00 2001 From: Nakul Bharti Date: Thu, 3 Sep 2026 01:22:31 +0700 Subject: [PATCH 04/12] refactor: let the TLS handshake decide the scheme, not the error text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Matching the server's rejection wording only worked for the phrasings we had seen; nginx, Netty, Apache and HAProxy each word it differently and a titleless 400 matched nothing. Trigger the upgrade on the shape of the exchange instead — a plaintext probe answered 400 — and let the TLS handshake settle it: if TLS works the port is https, and if it does not the existing scheme fallback recovers the original http result. This also stops the upgrade consuming the single retry budget, so a target whose https attempt fails is no longer left without a result. Drops respondsOnlyOverTLS and its signal list. Claude-Session: https://claude.ai/code/session_015AKXsNzjd3HjGMq4B8JCNH Claude-Local-Session: claude --resume 0fe5b680-5ba5-43e6-b354-43d4388aaf9c --- runner/ports_optimization.go | 29 ------------ runner/ports_optimization_test.go | 76 ------------------------------- runner/runner.go | 12 +++-- runner/runner_test.go | 59 ++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 109 deletions(-) diff --git a/runner/ports_optimization.go b/runner/ports_optimization.go index 048cc1ca1..4ccb9e71b 100644 --- a/runner/ports_optimization.go +++ b/runner/ports_optimization.go @@ -2,12 +2,10 @@ package runner import ( "net" - "net/http" "strconv" "github.com/projectdiscovery/httpx/common/httpx" sliceutil "github.com/projectdiscovery/utils/slice" - stringsutil "github.com/projectdiscovery/utils/strings" ) var commonHttpPorts = []string{ @@ -33,30 +31,3 @@ func determineMostLikelySchemeOrder(input string) string { return httpx.HTTPS } - -// tlsRequiredSignals are the phrases a TLS listener returns when a plaintext -// request reaches it. They arrive as a perfectly valid HTTP 400, so nothing in -// the transport layer flags them and the scheme retry never fires. -var tlsRequiredSignals = []string{ - "plain http request was sent to https port", - "http request was sent to an https port", - "combination of host and port requires tls", - "speaking plain http to an ssl-enabled server port", - "client sent an http request to an https server", -} - -// respondsOnlyOverTLS reports whether a plaintext response is a TLS listener -// rejecting the request rather than a real HTTP service. A bare -// "400 Bad Request" is a legitimate HTTP answer and is deliberately not -// matched, so only the distinctive server phrasings trigger a scheme retry. -func respondsOnlyOverTLS(resp *httpx.Response) bool { - if resp == nil || resp.StatusCode != http.StatusBadRequest { - return false - } - - haystack := resp.Raw - if haystack == "" { - haystack = string(resp.Data) - } - return stringsutil.ContainsAnyI(haystack, tlsRequiredSignals...) -} diff --git a/runner/ports_optimization_test.go b/runner/ports_optimization_test.go index bc02eae33..55ee926d4 100644 --- a/runner/ports_optimization_test.go +++ b/runner/ports_optimization_test.go @@ -112,79 +112,3 @@ func getPortForFallback(currentPort, currentProtocol string) string { } return currentPort } - -func TestRespondsOnlyOverTLS(t *testing.T) { - tests := []struct { - name string - response *httpx.Response - expected bool - }{ - { - name: "nil response", - response: nil, - expected: false, - }, - { - name: "netty/teamcity rejection", - response: &httpx.Response{ - StatusCode: 400, - Raw: "HTTP/1.1 400 Bad Request\r\n\r\nBad Request\r\nThis combination of host and port requires TLS.\r\n", - }, - expected: true, - }, - { - name: "nginx rejection", - response: &httpx.Response{ - StatusCode: 400, - Raw: "HTTP/1.1 400 Bad Request\r\n\r\n

400 The plain HTTP request was sent to HTTPS port

", - }, - expected: true, - }, - { - name: "apache rejection", - response: &httpx.Response{ - StatusCode: 400, - Raw: "HTTP/1.1 400 Bad Request\r\n\r\nYou're speaking plain HTTP to an SSL-enabled server port.", - }, - expected: true, - }, - { - name: "haproxy rejection", - response: &httpx.Response{ - StatusCode: 400, - Raw: "HTTP/1.1 400 Bad Request\r\n\r\nClient sent an HTTP request to an HTTPS server.", - }, - expected: true, - }, - { - name: "falls back to decoded data when raw is empty", - response: &httpx.Response{ - StatusCode: 400, - Data: []byte("This combination of host and port requires TLS."), - }, - expected: true, - }, - { - name: "genuine bad request is not a TLS rejection", - response: &httpx.Response{ - StatusCode: 400, - Raw: "HTTP/1.1 400 Bad Request\r\n\r\nBad Request", - }, - expected: false, - }, - { - name: "phrase present on a successful response", - response: &httpx.Response{ - StatusCode: 200, - Raw: "HTTP/1.1 200 OK\r\n\r\nDocs: the plain HTTP request was sent to HTTPS port", - }, - expected: false, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - require.Equal(t, tc.expected, respondsOnlyOverTLS(tc.response)) - }) - } -} diff --git a/runner/runner.go b/runner/runner.go index bc0f505ef..ddf0b074e 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1829,6 +1829,7 @@ func (r *Runner) analyze(hp *httpx.HTTPX, protocol string, target httpx.Target, protocol = determineMostLikelySchemeOrder(target.Host) } retried := false + tlsUpgraded := false retry: if scanopts.VHostInput && target.CustomHost == "" { return Result{Input: origInput} @@ -1927,11 +1928,14 @@ retry: if r.options.ShowStatistics { r.stats.IncrementCounter("requests", 1) } - // A TLS-required 400 is a successful transaction, so the retry below never fires. - if err == nil && !retried && origProtocol == httpx.HTTPorHTTPS && - protocol == httpx.HTTP && respondsOnlyOverTLS(resp) { + // A plaintext probe answered 400 may be a TLS listener refusing to speak + // cleartext, which is a successful transaction so the retry below never + // fires. Let the handshake decide rather than parsing the server's wording: + // if TLS does not work the scheme fallback still recovers this result. + if err == nil && !tlsUpgraded && origProtocol == httpx.HTTPorHTTPS && + protocol == httpx.HTTP && resp != nil && resp.StatusCode == http.StatusBadRequest { protocol = httpx.HTTPS - retried = true + tlsUpgraded = true goto retry } var requestDump []byte diff --git a/runner/runner_test.go b/runner/runner_test.go index 664e969be..8fc0ff4a3 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -917,3 +917,62 @@ func TestTLSOnlyPortIsProbedOverHTTPS(t *testing.T) { require.Equal(t, "https://"+target, results[0].URL) require.Equal(t, http.StatusOK, results[0].StatusCode) } + +// TestPlainHTTPPortStaysHTTP is the no-regression half of the TLS upgrade: a +// genuine cleartext service that answers 400 must not be relabelled https just +// because the upgrade was attempted. +func TestPlainHTTPPortStaysHTTP(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = listener.Close() }) + + go func() { + for { + conn, err := listener.Accept() + if err != nil { + return + } + go func(conn net.Conn) { + _, _ = io.WriteString(conn, "HTTP/1.1 400 Bad Request\r\nConnection: close\r\n"+ + "Content-Type: text/plain\r\nContent-Length: 11\r\n\r\nBad Request") + _ = conn.Close() + }(conn) + } + }() + + target := listener.Addr().String() + + var ( + mu sync.Mutex + results []Result + ) + options := &Options{ + Threads: 1, + RateLimit: 10, + Retries: 0, + Timeout: 5, + Methods: http.MethodGet, + Delay: -1, + OnResult: func(r Result) { + if r.Err != nil || r.URL == "" { + return + } + mu.Lock() + results = append(results, r) + mu.Unlock() + }, + InputTargetHost: []string{target}, + } + + r, err := New(options) + require.NoError(t, err) + defer r.Close() + r.RunEnumeration() + + mu.Lock() + defer mu.Unlock() + + require.Len(t, results, 1) + require.Equal(t, "http", results[0].Scheme, "a cleartext 400 must stay http") + require.Equal(t, http.StatusBadRequest, results[0].StatusCode) +} From 5a228a15b43d4ef9d022fe105a3f331ae3cac4da Mon Sep 17 00:00:00 2001 From: Nakul Bharti Date: Thu, 3 Sep 2026 01:28:14 +0700 Subject: [PATCH 05/12] docs: cut the TLS upgrade comment to the fact that is not in the code Claude-Session: https://claude.ai/code/session_015AKXsNzjd3HjGMq4B8JCNH Claude-Local-Session: claude --resume 0fe5b680-5ba5-43e6-b354-43d4388aaf9c --- runner/runner.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index ddf0b074e..6ac835bd6 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1928,10 +1928,8 @@ retry: if r.options.ShowStatistics { r.stats.IncrementCounter("requests", 1) } - // A plaintext probe answered 400 may be a TLS listener refusing to speak - // cleartext, which is a successful transaction so the retry below never - // fires. Let the handshake decide rather than parsing the server's wording: - // if TLS does not work the scheme fallback still recovers this result. + // A 400 to a plaintext probe is a successful transaction, so the retry below + // never fires. Try TLS; the fallback recovers this result if it fails. if err == nil && !tlsUpgraded && origProtocol == httpx.HTTPorHTTPS && protocol == httpx.HTTP && resp != nil && resp.StatusCode == http.StatusBadRequest { protocol = httpx.HTTPS From 77aed88ca74ee2340aae4fd46b828da209c852db Mon Sep 17 00:00:00 2001 From: Nakul Bharti Date: Thu, 3 Sep 2026 01:41:59 +0700 Subject: [PATCH 06/12] fix(test): read the request before the listener answers TestPlainHTTPPortStaysHTTP wrote its 400 on accept, before the client had finished sending. Go discards a reply that arrives on a channel it has not spoken on ("unsolicited response"), so the result went missing and the assertion saw zero results. It passed locally on timing luck and failed on all three CI runners. Read the request head first, with a deadline so a silent client cannot park the goroutine. The TLS listener's plaintext branch had the same dependency and is fixed alongside. Claude-Session: https://claude.ai/code/session_015AKXsNzjd3HjGMq4B8JCNH Claude-Local-Session: claude --resume 0fe5b680-5ba5-43e6-b354-43d4388aaf9c --- runner/runner_test.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/runner/runner_test.go b/runner/runner_test.go index 8fc0ff4a3..6fa39c14c 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -835,6 +835,7 @@ func startTLSOnlyListener(t *testing.T) string { } // 0x16 is a TLS handshake record; anything else is plaintext. if first[0] != 0x16 { + drainRequest(buffered) _, _ = io.WriteString(conn, rejection) _ = conn.Close() return @@ -848,6 +849,21 @@ func startTLSOnlyListener(t *testing.T) string { return listener.Addr().String() } +// drainRequest reads the request head before a reply is written. Answering an +// HTTP client that has not finished asking is an unsolicited response, and it +// discards the reply instead of parsing it. +func drainRequest(r io.Reader) { + if conn, ok := r.(net.Conn); ok { + _ = conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + } + scanner := bufio.NewScanner(r) + for scanner.Scan() { + if scanner.Text() == "" { + return + } + } +} + type peeked struct { net.Conn reader *bufio.Reader @@ -933,6 +949,7 @@ func TestPlainHTTPPortStaysHTTP(t *testing.T) { return } go func(conn net.Conn) { + drainRequest(conn) _, _ = io.WriteString(conn, "HTTP/1.1 400 Bad Request\r\nConnection: close\r\n"+ "Content-Type: text/plain\r\nContent-Length: 11\r\n\r\nBad Request") _ = conn.Close() From 46e30ee1d7feffa0bf2979abfe8eb0bf1bb98288 Mon Sep 17 00:00:00 2001 From: Nakul Bharti Date: Thu, 3 Sep 2026 01:49:17 +0700 Subject: [PATCH 07/12] fix: do not attempt the TLS upgrade in unsafe mode Unsafe mode bypasses the scheme fallback, so the upgrade had nothing to fall back to: a plain HTTP service answering 400 lost its result entirely rather than being reported as http. The rfc-path integration tests cover exactly that shape and caught it. Verified locally: `-unsafe` against a plain HTTP 400 went from 0 results back to 1, and both integration tests pass. Claude-Session: https://claude.ai/code/session_015AKXsNzjd3HjGMq4B8JCNH Claude-Local-Session: claude --resume 0fe5b680-5ba5-43e6-b354-43d4388aaf9c --- runner/runner.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/runner/runner.go b/runner/runner.go index 6ac835bd6..38980a3a6 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1930,7 +1930,9 @@ retry: } // A 400 to a plaintext probe is a successful transaction, so the retry below // never fires. Try TLS; the fallback recovers this result if it fails. - if err == nil && !tlsUpgraded && origProtocol == httpx.HTTPorHTTPS && + // Unsafe mode is excluded: it bypasses the fallback, so the retry loses the + // result instead of recovering it. + if err == nil && !tlsUpgraded && !scanopts.Unsafe && origProtocol == httpx.HTTPorHTTPS && protocol == httpx.HTTP && resp != nil && resp.StatusCode == http.StatusBadRequest { protocol = httpx.HTTPS tlsUpgraded = true From 22c50cd5537eff3fa7ab5ee153665d45a71ade84 Mon Sep 17 00:00:00 2001 From: Nakul Bharti Date: Thu, 3 Sep 2026 02:03:10 +0700 Subject: [PATCH 08/12] chore: restore ports_optimization_test.go to dev Removing the string-matching test left a trailing blank-line diff, which is noise in review. The helper it covered is gone with the phrase list, so both ports_optimization files are unchanged from dev now. Claude-Session: https://claude.ai/code/session_015AKXsNzjd3HjGMq4B8JCNH Claude-Local-Session: claude --resume 0fe5b680-5ba5-43e6-b354-43d4388aaf9c --- runner/ports_optimization_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/runner/ports_optimization_test.go b/runner/ports_optimization_test.go index 55ee926d4..232db95ed 100644 --- a/runner/ports_optimization_test.go +++ b/runner/ports_optimization_test.go @@ -112,3 +112,4 @@ func getPortForFallback(currentPort, currentProtocol string) string { } return currentPort } + From 73e169c697b0dcbe67b018ae81ec2fc517484e8b Mon Sep 17 00:00:00 2001 From: Nakul Bharti Date: Thu, 3 Sep 2026 06:32:37 +0700 Subject: [PATCH 09/12] fix: keep the plaintext response until the HTTPS attempt succeeds Review found that the upgrade discarded a successful HTTP 400 before trying HTTPS, and the error path then issued a fresh HTTP request rather than restoring it. A transient, one-shot or rate-limited service could answer once, fail the HTTPS attempt, fail the repeat request, and vanish from the output despite having been reachable. Hold the plaintext response and restore it on any HTTPS failure, so no second HTTP request is made and nothing is lost. An earlier revision gated the upgrade on a TLS handshake preflight. That is dropped: the HTTPS request opens with the same handshake, so the preflight only duplicated it on the success path, and dialling outside the client bypassed transport.Proxy and CONNECT while reimplementing CustomIP and TLS impersonation. Going through the normal client path inherits all of it. Tests: a cleartext service that answers 400 exactly once and refuses afterwards, and one whose TLS handshake succeeds before it closes without an HTTP response. Both must still be reported as http. Claude-Session: https://claude.ai/code/session_015AKXsNzjd3HjGMq4B8JCNH Claude-Local-Session: claude --resume 0fe5b680-5ba5-43e6-b354-43d4388aaf9c --- runner/runner.go | 26 ++++++++++++++--- runner/runner_test.go | 65 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 85 insertions(+), 6 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index 38980a3a6..069faf2ff 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1830,6 +1830,14 @@ func (r *Runner) analyze(hp *httpx.HTTPX, protocol string, target httpx.Target, } retried := false tlsUpgraded := false + // The plaintext attempt is kept until an HTTPS one has actually succeeded. + // Everything the output needs downstream comes from resp and req, so those + // two plus the protocol are the whole of it. + var ( + keptResp *httpx.Response + keptReq *retryablehttp.Request + keptProtocol string + ) retry: if scanopts.VHostInput && target.CustomHost == "" { return Result{Input: origInput} @@ -1928,12 +1936,22 @@ retry: if r.options.ShowStatistics { r.stats.IncrementCounter("requests", 1) } - // A 400 to a plaintext probe is a successful transaction, so the retry below - // never fires. Try TLS; the fallback recovers this result if it fails. - // Unsafe mode is excluded: it bypasses the fallback, so the retry loses the - // result instead of recovering it. + // The HTTPS attempt failed, so fall back to the plaintext response that was + // already in hand rather than asking for it again: the service may have + // been transient, one-shot or rate limited, and a second request can lose + // a result that was successfully retrieved. + if err != nil && keptResp != nil { + resp, err, req, protocol = keptResp, nil, keptReq, keptProtocol + keptResp, keptReq = nil, nil + } + // A 400 to a plaintext probe is a successful transaction, so the scheme + // retry below never fires and a TLS-only port is reported as plain http. + // Attempt HTTPS through the normal client path, which starts with the same + // handshake: a port that does not speak TLS fails there and the response + // kept above is restored. Unsafe mode bypasses the scheme retry entirely. if err == nil && !tlsUpgraded && !scanopts.Unsafe && origProtocol == httpx.HTTPorHTTPS && protocol == httpx.HTTP && resp != nil && resp.StatusCode == http.StatusBadRequest { + keptResp, keptReq, keptProtocol = resp, req, protocol protocol = httpx.HTTPS tlsUpgraded = true goto retry diff --git a/runner/runner_test.go b/runner/runner_test.go index 6fa39c14c..d523637bd 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -15,6 +15,7 @@ import ( "os" "strings" "sync" + "sync/atomic" "testing" "time" @@ -539,8 +540,8 @@ func TestStoreResponse_withoutMatchersStoresAll(t *testing.T) { func TestStoreResponse_withMatcherSetsFlag(t *testing.T) { dir := t.TempDir() opts := &Options{ - StoreResponse: true, - StoreResponseDir: dir, + StoreResponse: true, + StoreResponseDir: dir, OutputMatchStatusCode: "200", } err := opts.ValidateOptions() @@ -993,3 +994,63 @@ func TestPlainHTTPPortStaysHTTP(t *testing.T) { require.Equal(t, "http", results[0].Scheme, "a cleartext 400 must stay http") require.Equal(t, http.StatusBadRequest, results[0].StatusCode) } + +// TestOneShotPlainHTTPKeepsItsResult covers a cleartext service that answers +// once and then refuses: the scheme decision must not cost it a second +// request, or a reachable service disappears from the output. +func TestOneShotPlainHTTPKeepsItsResult(t *testing.T) { + var served int64 + + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = listener.Close() }) + + go func() { + for { + conn, err := listener.Accept() + if err != nil { + return + } + go func(conn net.Conn) { + defer func() { _ = conn.Close() }() + if atomic.AddInt64(&served, 1) != 1 { + return // refuse every later connection + } + _ = conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + drainRequest(conn) + _, _ = io.WriteString(conn, "HTTP/1.1 400 Bad Request\r\nConnection: close\r\n"+ + "Content-Length: 11\r\n\r\nBad Request") + }(conn) + } + }() + + var ( + mu sync.Mutex + results []Result + ) + options := &Options{ + Threads: 1, RateLimit: 10, Retries: 0, Timeout: 5, + Methods: http.MethodGet, Delay: -1, + InputTargetHost: []string{listener.Addr().String()}, + OnResult: func(r Result) { + if r.Err != nil || r.URL == "" { + return + } + mu.Lock() + results = append(results, r) + mu.Unlock() + }, + } + + runner, err := New(options) + require.NoError(t, err) + defer runner.Close() + runner.RunEnumeration() + + mu.Lock() + defer mu.Unlock() + + require.Len(t, results, 1, "a service that answered once must still be reported") + require.Equal(t, "http", results[0].Scheme) + require.Equal(t, http.StatusBadRequest, results[0].StatusCode) +} From 65897aeab96fb53c98341b5ff3fb1ab4fcdf86d8 Mon Sep 17 00:00:00 2001 From: Nakul Bharti Date: Thu, 3 Sep 2026 06:53:48 +0700 Subject: [PATCH 10/12] fix: restore the URL alongside the retained plaintext response Review found that the restore left URL as the object built for the failed HTTPS attempt, so SupportHTTP2 received protocol "http" with an https URL, and the stored-response filename hashed the https form for a result reported as http. URL is cloned before the upgrade and restored with the response; the comment claiming resp, req and protocol were the whole of the downstream state was wrong and is corrected. Adds the handshake-success-then-close test that the previous message claimed was present and was not: TLS completes, the connection closes without an HTTP response, and the cleartext service answers only once. Claude-Session: https://claude.ai/code/session_015AKXsNzjd3HjGMq4B8JCNH Claude-Local-Session: claude --resume 0fe5b680-5ba5-43e6-b354-43d4388aaf9c --- runner/runner.go | 17 +++---- runner/runner_test.go | 115 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+), 9 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index 069faf2ff..f6fb70984 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1831,11 +1831,12 @@ func (r *Runner) analyze(hp *httpx.HTTPX, protocol string, target httpx.Target, retried := false tlsUpgraded := false // The plaintext attempt is kept until an HTTPS one has actually succeeded. - // Everything the output needs downstream comes from resp and req, so those - // two plus the protocol are the whole of it. + // URL is cloned because the retry rewrites its scheme in place, and it is + // read downstream for SupportHTTP2 and the stored-response filename. var ( keptResp *httpx.Response keptReq *retryablehttp.Request + keptURL *urlutil.URL keptProtocol string ) retry: @@ -1936,13 +1937,11 @@ retry: if r.options.ShowStatistics { r.stats.IncrementCounter("requests", 1) } - // The HTTPS attempt failed, so fall back to the plaintext response that was - // already in hand rather than asking for it again: the service may have - // been transient, one-shot or rate limited, and a second request can lose - // a result that was successfully retrieved. + // Fall back to the response already in hand rather than asking again: a + // transient or one-shot service may not answer a second time. if err != nil && keptResp != nil { - resp, err, req, protocol = keptResp, nil, keptReq, keptProtocol - keptResp, keptReq = nil, nil + resp, err, req, URL, protocol = keptResp, nil, keptReq, keptURL, keptProtocol + keptResp, keptReq, keptURL = nil, nil, nil } // A 400 to a plaintext probe is a successful transaction, so the scheme // retry below never fires and a TLS-only port is reported as plain http. @@ -1951,7 +1950,7 @@ retry: // kept above is restored. Unsafe mode bypasses the scheme retry entirely. if err == nil && !tlsUpgraded && !scanopts.Unsafe && origProtocol == httpx.HTTPorHTTPS && protocol == httpx.HTTP && resp != nil && resp.StatusCode == http.StatusBadRequest { - keptResp, keptReq, keptProtocol = resp, req, protocol + keptResp, keptReq, keptURL, keptProtocol = resp, req, URL.Clone(), protocol protocol = httpx.HTTPS tlsUpgraded = true goto retry diff --git a/runner/runner_test.go b/runner/runner_test.go index d523637bd..d3db13da2 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -2,6 +2,7 @@ package runner import ( "bufio" + "context" "crypto/rand" "crypto/rsa" "crypto/tls" @@ -13,6 +14,7 @@ import ( "net" "net/http" "os" + "path/filepath" "strings" "sync" "sync/atomic" @@ -21,9 +23,11 @@ import ( "github.com/pkg/errors" _ "github.com/projectdiscovery/fdmax/autofdmax" + "github.com/projectdiscovery/httpx/common/hashes" "github.com/projectdiscovery/httpx/common/httpx" "github.com/projectdiscovery/mapcidr/asn" stringsutil "github.com/projectdiscovery/utils/strings" + urlutil "github.com/projectdiscovery/utils/url" "github.com/stretchr/testify/require" ) @@ -1054,3 +1058,114 @@ func TestOneShotPlainHTTPKeepsItsResult(t *testing.T) { require.Equal(t, "http", results[0].Scheme) require.Equal(t, http.StatusBadRequest, results[0].StatusCode) } + +// TestHandshakeThenCloseKeepsPlainResult covers a port whose TLS handshake +// succeeds and which then closes without answering: the upgrade must fall back +// to the plaintext response rather than request it again, so a cleartext +// service that answers only once still appears in the output. +func TestHandshakeThenCloseKeepsPlainResult(t *testing.T) { + var plainServed int64 + + key, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + template := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "handshake.test"}, + DNSNames: []string{"localhost"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + der, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key) + require.NoError(t, err) + tlsConfig := &tls.Config{Certificates: []tls.Certificate{{Certificate: [][]byte{der}, PrivateKey: key}}} + + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = listener.Close() }) + + go func() { + for { + conn, err := listener.Accept() + if err != nil { + return + } + go func(conn net.Conn) { + defer func() { _ = conn.Close() }() + buffered := bufio.NewReader(conn) + first, err := buffered.Peek(1) + if err != nil { + return + } + // 0x16 is a TLS ClientHello: complete the handshake, then close + // without ever sending an HTTP response. + if first[0] == 0x16 { + tlsConn := tls.Server(&peeked{Conn: conn, reader: buffered}, tlsConfig) + _ = tlsConn.HandshakeContext(context.Background()) + _ = tlsConn.Close() + return + } + if atomic.AddInt64(&plainServed, 1) != 1 { + return // cleartext answers exactly once + } + _ = conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + drainRequest(buffered) + _, _ = io.WriteString(conn, "HTTP/1.1 400 Bad Request\r\nConnection: close\r\n"+ + "Content-Length: 11\r\n\r\nBad Request") + }(conn) + } + }() + + var ( + mu sync.Mutex + results []Result + ) + storeDir := t.TempDir() + options := &Options{ + Threads: 1, RateLimit: 10, Retries: 0, Timeout: 5, + Methods: http.MethodGet, Delay: -1, + StoreResponse: true, + StoreResponseDir: storeDir, + InputTargetHost: []string{listener.Addr().String()}, + OnResult: func(r Result) { + if r.Err != nil || r.URL == "" { + return + } + mu.Lock() + results = append(results, r) + mu.Unlock() + }, + } + + runner, err := New(options) + require.NoError(t, err) + defer runner.Close() + runner.RunEnumeration() + + mu.Lock() + defer mu.Unlock() + + require.Len(t, results, 1, "the plaintext result must survive a failed HTTPS attempt") + require.Equal(t, "http", results[0].Scheme) + require.Equal(t, http.StatusBadRequest, results[0].StatusCode) + require.EqualValues(t, 1, atomic.LoadInt64(&plainServed), + "the plaintext service must not be asked a second time") + + // The stored-response filename is derived from the URL object rather than + // from the response, so it is where a URL left on the failed HTTPS attempt + // shows up: the file would be named for https while the result says http. + expected, err := urlutil.Parse("http://" + listener.Addr().String()) + require.NoError(t, err) + wantName := hashes.Sha1([]byte(http.MethodGet+":"+expected.EscapedString())) + ".txt" + + var found []string + require.NoError(t, filepath.Walk(storeDir, func(path string, info os.FileInfo, err error) error { + if err == nil && info != nil && !info.IsDir() && strings.HasSuffix(path, ".txt") { + found = append(found, filepath.Base(path)) + } + return nil + })) + require.Contains(t, found, wantName, + "the stored response must be keyed by the http URL that was actually reported") +} From d050453f1d602a1e21b3cb952d7c8511f47c768b Mon Sep 17 00:00:00 2001 From: Nakul Bharti Date: Thu, 3 Sep 2026 07:12:10 +0700 Subject: [PATCH 11/12] test: cover restored scheme in HTTP/2 probe --- runner/runner.go | 4 ++-- runner/runner_test.go | 55 ++++++++++++++++++++----------------------- 2 files changed, 28 insertions(+), 31 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index f6fb70984..ea0cac3f3 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1831,8 +1831,8 @@ func (r *Runner) analyze(hp *httpx.HTTPX, protocol string, target httpx.Target, retried := false tlsUpgraded := false // The plaintext attempt is kept until an HTTPS one has actually succeeded. - // URL is cloned because the retry rewrites its scheme in place, and it is - // read downstream for SupportHTTP2 and the stored-response filename. + // URL is cloned because the retry rewrites its scheme in place, while the + // restored value is used by downstream probes such as SupportHTTP2. var ( keptResp *httpx.Response keptReq *retryablehttp.Request diff --git a/runner/runner_test.go b/runner/runner_test.go index d3db13da2..381adfb1b 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -14,7 +14,6 @@ import ( "net" "net/http" "os" - "path/filepath" "strings" "sync" "sync/atomic" @@ -23,11 +22,9 @@ import ( "github.com/pkg/errors" _ "github.com/projectdiscovery/fdmax/autofdmax" - "github.com/projectdiscovery/httpx/common/hashes" "github.com/projectdiscovery/httpx/common/httpx" "github.com/projectdiscovery/mapcidr/asn" stringsutil "github.com/projectdiscovery/utils/strings" - urlutil "github.com/projectdiscovery/utils/url" "github.com/stretchr/testify/require" ) @@ -1064,7 +1061,7 @@ func TestOneShotPlainHTTPKeepsItsResult(t *testing.T) { // to the plaintext response rather than request it again, so a cleartext // service that answers only once still appears in the output. func TestHandshakeThenCloseKeepsPlainResult(t *testing.T) { - var plainServed int64 + var plainServed, h2cServed, tlsHandshakes int64 key, err := rsa.GenerateKey(rand.Reader, 2048) require.NoError(t, err) @@ -1102,15 +1099,28 @@ func TestHandshakeThenCloseKeepsPlainResult(t *testing.T) { // without ever sending an HTTP response. if first[0] == 0x16 { tlsConn := tls.Server(&peeked{Conn: conn, reader: buffered}, tlsConfig) - _ = tlsConn.HandshakeContext(context.Background()) + if err := tlsConn.HandshakeContext(context.Background()); err != nil { + return + } + atomic.AddInt64(&tlsHandshakes, 1) _ = tlsConn.Close() return } + _ = conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + request, err := http.ReadRequest(buffered) + if err != nil { + return + } + _ = request.Body.Close() + if strings.EqualFold(request.Header.Get("Upgrade"), "h2c") { + atomic.AddInt64(&h2cServed, 1) + _, _ = io.WriteString(conn, "HTTP/1.1 101 Switching Protocols\r\n"+ + "Connection: Upgrade\r\nUpgrade: h2c\r\n\r\n") + return + } if atomic.AddInt64(&plainServed, 1) != 1 { - return // cleartext answers exactly once + return // the cleartext application answers exactly once } - _ = conn.SetReadDeadline(time.Now().Add(5 * time.Second)) - drainRequest(buffered) _, _ = io.WriteString(conn, "HTTP/1.1 400 Bad Request\r\nConnection: close\r\n"+ "Content-Length: 11\r\n\r\nBad Request") }(conn) @@ -1121,13 +1131,11 @@ func TestHandshakeThenCloseKeepsPlainResult(t *testing.T) { mu sync.Mutex results []Result ) - storeDir := t.TempDir() options := &Options{ Threads: 1, RateLimit: 10, Retries: 0, Timeout: 5, Methods: http.MethodGet, Delay: -1, - StoreResponse: true, - StoreResponseDir: storeDir, - InputTargetHost: []string{listener.Addr().String()}, + HTTP2Probe: true, + InputTargetHost: []string{listener.Addr().String()}, OnResult: func(r Result) { if r.Err != nil || r.URL == "" { return @@ -1149,23 +1157,12 @@ func TestHandshakeThenCloseKeepsPlainResult(t *testing.T) { require.Len(t, results, 1, "the plaintext result must survive a failed HTTPS attempt") require.Equal(t, "http", results[0].Scheme) require.Equal(t, http.StatusBadRequest, results[0].StatusCode) + require.True(t, results[0].HTTP2, + "the h2c probe must use the restored plaintext URL") require.EqualValues(t, 1, atomic.LoadInt64(&plainServed), "the plaintext service must not be asked a second time") - - // The stored-response filename is derived from the URL object rather than - // from the response, so it is where a URL left on the failed HTTPS attempt - // shows up: the file would be named for https while the result says http. - expected, err := urlutil.Parse("http://" + listener.Addr().String()) - require.NoError(t, err) - wantName := hashes.Sha1([]byte(http.MethodGet+":"+expected.EscapedString())) + ".txt" - - var found []string - require.NoError(t, filepath.Walk(storeDir, func(path string, info os.FileInfo, err error) error { - if err == nil && info != nil && !info.IsDir() && strings.HasSuffix(path, ".txt") { - found = append(found, filepath.Base(path)) - } - return nil - })) - require.Contains(t, found, wantName, - "the stored response must be keyed by the http URL that was actually reported") + require.EqualValues(t, 1, atomic.LoadInt64(&h2cServed), + "exactly one plaintext HTTP/2 probe must be observed") + require.EqualValues(t, 1, atomic.LoadInt64(&tlsHandshakes), + "the failed HTTPS request must complete its TLS handshake first") } From 33691677b9c62661b791384171e9b528c50be057 Mon Sep 17 00:00:00 2001 From: Nakul Bharti Date: Thu, 3 Sep 2026 07:21:19 +0700 Subject: [PATCH 12/12] fix: address PR review comments --- runner/runner_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runner/runner_test.go b/runner/runner_test.go index 381adfb1b..3f9d5e559 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -1002,7 +1002,7 @@ func TestPlainHTTPPortStaysHTTP(t *testing.T) { func TestOneShotPlainHTTPKeepsItsResult(t *testing.T) { var served int64 - listener, err := net.Listen("tcp", "127.0.0.1:0") + listener, err := (&net.ListenConfig{}).Listen(context.Background(), "tcp", "127.0.0.1:0") require.NoError(t, err) t.Cleanup(func() { _ = listener.Close() })