From be8322da518756b8e35a43c72c8dc8801022cac0 Mon Sep 17 00:00:00 2001
From: krishna28238-arch <319297638+krishna28238-arch@users.noreply.github.com>
Date: Sun, 6 Sep 2026 14:30:16 +0000
Subject: [PATCH] fix: handle quoted charset and non-canonical content-type in
title extraction
---
common/httpx/encodings.go | 2 ++
common/httpx/title.go | 2 +-
common/httpx/title_test.go | 72 ++++++++++++++++++++++++++++++++++++++
3 files changed, 75 insertions(+), 1 deletion(-)
create mode 100644 common/httpx/title_test.go
diff --git a/common/httpx/encodings.go b/common/httpx/encodings.go
index 4ce49a350..df03348c5 100644
--- a/common/httpx/encodings.go
+++ b/common/httpx/encodings.go
@@ -58,6 +58,8 @@ func DecodeData(data []byte, headers http.Header) ([]byte, error) {
// Non UTF-8
if contentTypes, ok := headers["Content-Type"]; ok {
contentType := strings.ToLower(strings.Join(contentTypes, ";"))
+ // the charset parameter value can be a quoted string (charset="gbk")
+ contentType = strings.ReplaceAll(contentType, `"`, "")
switch {
case stringsutil.ContainsAny(contentType, "charset=gb2312", "charset=gbk"):
diff --git a/common/httpx/title.go b/common/httpx/title.go
index e38ba7d3a..7c0bb6d24 100644
--- a/common/httpx/title.go
+++ b/common/httpx/title.go
@@ -51,7 +51,7 @@ func ExtractTitle(r *Response) (title string) {
}
func CanHaveTitleTag(mimeType string) bool {
- return slices.Contains(supportedTitleMimeTypes, mimeType)
+ return slices.Contains(supportedTitleMimeTypes, strings.ToLower(strings.TrimSpace(mimeType)))
}
func getTitleWithDom(r *Response) (*html.Node, error) {
diff --git a/common/httpx/title_test.go b/common/httpx/title_test.go
new file mode 100644
index 000000000..12353546a
--- /dev/null
+++ b/common/httpx/title_test.go
@@ -0,0 +1,72 @@
+package httpx
+
+import (
+ "net/http"
+ "net/http/httptest"
+ "testing"
+ "time"
+
+ "github.com/projectdiscovery/retryablehttp-go"
+ "github.com/stretchr/testify/require"
+)
+
+func TestCanHaveTitleTag(t *testing.T) {
+ tests := []struct {
+ mimeType string
+ expected bool
+ }{
+ {"text/html", true},
+ {"TEXT/HTML", true},
+ {"Text/Html", true},
+ {"text/html ", true},
+ {" application/xhtml+xml", true},
+ {"text/plain", false},
+ {"application/json", false},
+ {"", false},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.mimeType, func(t *testing.T) {
+ require.Equal(t, tt.expected, CanHaveTitleTag(tt.mimeType))
+ })
+ }
+}
+
+func TestExtractTitleDecodesCharset(t *testing.T) {
+ options := DefaultOptions
+ options.CdnCheck = "false"
+ options.Timeout = 2 * time.Second
+ options.RetryMax = 0
+
+ ht, err := New(&options)
+ require.Nil(t, err)
+
+ // 中文 with the title text encoded in GBK
+ gbk := []byte{0xd6, 0xd0, 0xce, 0xc4}
+ body := append([]byte(""), gbk...)
+ body = append(body, []byte("")...)
+
+ tests := []struct {
+ name string
+ contentType string
+ }{
+ {"quoted charset", `text/html; charset="gbk"`},
+ {"unquoted charset", "text/html; charset=gbk"},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", tt.contentType)
+ _, _ = w.Write(body)
+ }))
+ defer srv.Close()
+
+ req, err := retryablehttp.NewRequest(http.MethodGet, srv.URL, nil)
+ require.Nil(t, err)
+ resp, err := ht.Do(req, UnsafeOptions{})
+ require.Nil(t, err)
+ require.Equal(t, "中文", ExtractTitle(resp))
+ })
+ }
+}