diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 120bb02ca..5de7c598f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -18,8 +18,13 @@ updates: labels: - "Type: Maintenance" groups: - modules: + # Internal PD libraries, usually safe to review as a batch + projectdiscovery: patterns: ["github.com/projectdiscovery/*"] + # Other packages, separate from PD bumps + external: + patterns: ["*"] + exclude-patterns: ["github.com/projectdiscovery/*"] # # Maintain dependencies for GitHub Actions # - package-ecosystem: "github-actions" diff --git a/Dockerfile b/Dockerfile index 279594ac2..205b1353f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Base -FROM golang:1.25.7-alpine AS builder +FROM golang:1.26.5-alpine AS builder RUN apk add --no-cache git build-base gcc musl-dev WORKDIR /app diff --git a/README.md b/README.md index e41811d17..298c87333 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,7 @@ PROBES: -server, -web-server display server name -td, -tech-detect display technology in use based on wappalyzer dataset -cff, -custom-fingerprint-file string path to a custom fingerprint file for technology detection + -kb, -knowledge-base enable knowledge base classification -method display http request method -ws, -websocket display server using websocket -ip display host ip @@ -238,7 +239,7 @@ CONFIGURATIONS: -ldp, -leave-default-ports leave default http/https ports in host header (eg. http://host:80 - https://host:443 -ztls use ztls library with autofallback to standard one for tls13 -no-decode avoid decoding body - -tlsi, -tls-impersonate enable experimental client hello (ja3) tls randomization + -tlsi, -tls-impersonate string enable experimental client hello (ja3) tls impersonation (chrome, or ja3 full string) -no-stdin Disable Stdin processing -hae, -http-api-endpoint string experimental http api endpoint -sf, -secret-file string path to secret file for authentication @@ -285,6 +286,81 @@ For details about running httpx, see https://docs.projectdiscovery.io/tools/http ### Using `httpx` as a library `httpx` can be used as a library by creating an instance of the `Option` struct and populating it with the same options that would be specified via CLI. Once validated, the struct should be passed to a runner instance (to be closed at the end of the program) and the `RunEnumeration` method should be called. A minimal example of how to do it is in the [examples](examples/) folder. +## Common Recipes + +Below are practical one-liners for common use cases leveraging httpx's composable primitives. These recipes are validated in `runner/wellknown_recipes_test.go`. + +Use `-mdc` with DSL helpers such as `contains(content_type, ...)` and `contains(body, ...)` to match response metadata and body content. (`-mr`/`-ms` match the full raw response; use `-mdc` for structured field matching.) + +### Well-known files + +**security.txt** +Probe for a valid [RFC 9116](https://www.rfc-editor.org/rfc/rfc9116.html) security.txt file at the standard paths (`/.well-known/security.txt`, `/security.txt`): + +```bash +echo target.com | httpx -path '/.well-known/security.txt,/security.txt' -mc 200 -mdc 'contains(content_type, "text/plain") && contains(body, "Contact:") && contains_any(body, "mailto:", "https://")' +``` + +- `-path` tests custom path(s) +- `-mc 200` matches HTTP 200 +- `-mdc` matches using DSL expressions on response fields such as `content_type` and `body` + +**robots.txt** + +```bash +echo target.com | httpx -path '/robots.txt' -mc 200 -mdc 'contains(content_type, "text/plain")' +``` + +**sitemap.xml** + +```bash +echo target.com | httpx -path '/sitemap.xml' -mc 200 -mdc 'contains_any(content_type, "application/xml", "text/xml") && contains(body, " 0 { + switch { + case looksLikeHTML(r.Data): + inlineScripts := extractDomainsFromHTML(r.Data, domains, fqdns, r.Input) + for _, script := range inlineScripts { + if len(script) <= maxInlineScriptSize { + extractDomainsFromJS(script, domains, fqdns, r.Input) + } + } + case looksLikeJavaScript(r.Data, r.GetHeader("Content-Type")): + if len(r.Data) <= maxInlineScriptSize { + extractDomainsFromJS(string(r.Data), domains, fqdns, r.Input) + } else { + extractDomainsFromRegex(string(r.Data), domains, fqdns, r.Input) + } + default: + extractDomainsFromRegex(r.Raw, domains, fqdns, r.Input) + } + } else { + extractDomainsFromRegex(r.Raw, domains, fqdns, r.Input) + } + + return &BodyDomain{Domains: mapsutil.GetKeys(domains), Fqdns: mapsutil.GetKeys(fqdns)} +} + +func looksLikeHTML(data []byte) bool { + trimmed := trimmedBodyPrefix(data) + return len(trimmed) > 0 && trimmed[0] == '<' +} + +func looksLikeJavaScript(data []byte, contentType string) bool { + ct := strings.ToLower(contentType) + if strings.Contains(ct, "javascript") || strings.Contains(ct, "ecmascript") { + return true + } + + trimmed := trimmedBodyPrefix(data) + if len(trimmed) == 0 { + return false + } + + s := string(trimmed) + return strings.HasPrefix(s, "var ") || + strings.HasPrefix(s, "const ") || + strings.HasPrefix(s, "let ") || + strings.HasPrefix(s, "function") || + strings.HasPrefix(s, "(function") || + strings.HasPrefix(s, "/*") || + strings.HasPrefix(s, "//") || + strings.HasPrefix(s, "import ") || + strings.HasPrefix(s, "export ") || + strings.HasPrefix(s, "!") || + strings.HasPrefix(s, "\"use strict\"") || + strings.HasPrefix(s, "'use strict'") +} + +func trimmedBodyPrefix(data []byte) []byte { + prefix := data + if len(prefix) > 1024 { + prefix = prefix[:1024] + } + trimmed := bytes.TrimSpace(prefix) + return bytes.TrimPrefix(trimmed, []byte{0xEF, 0xBB, 0xBF}) +} + +func inputHostname(input string) string { + input = strings.ToLower(strings.TrimSpace(input)) + if input == "" { + return "" + } + if host, _, err := net.SplitHostPort(input); err == nil { + return host + } + return input +} + +// extractDomainsFromHTML parses HTML and extracts hostnames from URL-bearing +// attributes (href, src, action, etc.), meta tags, and srcset values. +// It returns the text content of inline ` + response := &Response{ + Raw: html, + Data: []byte(html), + } + bd := ht.BodyDomainGrab(response) + require.NotNil(t, bd) +} + +func TestExtractDomainsFromJS(t *testing.T) { + domains := make(map[string]struct{}) + fqdns := make(map[string]struct{}) + + script := ` + var url = "https://api.test.example.com/v1/users"; + var cdn = "https://cdn.test.example.net/assets/main.js"; + var num = 42; + var noDomain = "just a plain string"; + ` + extractDomainsFromJS(script, domains, fqdns, "") + + require.Contains(t, fqdns, "api.test.example.com") + require.Contains(t, fqdns, "cdn.test.example.net") + require.Equal(t, 2, len(fqdns)) +} + +func TestExtractDomainsFromHTML(t *testing.T) { + domains := make(map[string]struct{}) + fqdns := make(map[string]struct{}) + + html := []byte(` + + link + + + + `) + + inlineScripts := extractDomainsFromHTML(html, domains, fqdns, "") + + require.Contains(t, fqdns, "link.test.example.com") + require.Contains(t, fqdns, "img.test.example.org") + require.Len(t, inlineScripts, 1) + require.Contains(t, inlineScripts[0], "inline.test.example.net") +} + +// --- False positive rejection tests --- + +func TestFalsePositive_IPAddresses(t *testing.T) { + ht, err := New(&DefaultOptions) + require.Nil(t, err) + + html := ` + router + internal + + ` + response := &Response{Raw: html, Data: []byte(html)} + bd := ht.BodyDomainGrab(response) + + for _, d := range bd.Domains { + require.False(t, isAllNumericParts(d), "IP address should not appear in domains: %s", d) + } + for _, f := range bd.Fqdns { + require.False(t, isAllNumericParts(f), "IP address should not appear in fqdns: %s", f) + } +} + +func TestFalsePositive_PackageNames(t *testing.T) { + ht, err := New(&DefaultOptions) + require.Nil(t, err) + + html := ` + + ` + response := &Response{Raw: html, Data: []byte(html)} + bd := ht.BodyDomainGrab(response) + + for _, f := range bd.Fqdns { + require.NotContains(t, f, "com.google.android", "Java/Android package name should be rejected: %s", f) + require.NotContains(t, f, "org.apache.commons", "Java package name should be rejected: %s", f) + require.NotContains(t, f, "io.netty.handler", "Java package name should be rejected: %s", f) + } +} + +func TestFalsePositive_FileExtensions(t *testing.T) { + ht, err := New(&DefaultOptions) + require.Nil(t, err) + + html := ` + report + css + js + + + ` + response := &Response{Raw: html, Data: []byte(html)} + bd := ht.BodyDomainGrab(response) + + for _, f := range bd.Fqdns { + require.NotEqual(t, "report.pdf", f) + require.NotEqual(t, "style.css", f) + require.NotEqual(t, "app.js", f) + require.NotEqual(t, "logo.png", f) + } +} + +func TestFalsePositive_VersionStrings(t *testing.T) { + ht, err := New(&DefaultOptions) + require.Nil(t, err) + + html := ` + +

Running version 2.4.1 of the software

+ ` + response := &Response{Raw: html, Data: []byte(html)} + bd := ht.BodyDomainGrab(response) + + require.Empty(t, bd.Fqdns, "version strings should not produce fqdns") + require.Empty(t, bd.Domains, "version strings should not produce domains") +} + +func TestFalsePositive_CSSClassNames(t *testing.T) { + ht, err := New(&DefaultOptions) + require.Nil(t, err) + + html := ` +
test
+ + ` + response := &Response{Raw: html, Data: []byte(html)} + bd := ht.BodyDomainGrab(response) + + for _, f := range bd.Fqdns { + require.NotContains(t, f, "header.main") + require.NotContains(t, f, "container.fluid") + require.NotContains(t, f, "nav.active") + } +} + +func TestFalsePositive_MinifiedJSVars(t *testing.T) { + ht, err := New(&DefaultOptions) + require.Nil(t, err) + + // minified JS often has expressions like e.target, n.value, t.id + html := `` + response := &Response{Raw: html, Data: []byte(html)} + bd := ht.BodyDomainGrab(response) + + require.Empty(t, bd.Fqdns, "minified JS property accesses should not produce fqdns") + require.Empty(t, bd.Domains, "minified JS property accesses should not produce domains") +} + +func TestFalsePositive_MailtoAndTelLinks(t *testing.T) { + ht, err := New(&DefaultOptions) + require.Nil(t, err) + + html := ` + email + call + ` + response := &Response{Raw: html, Data: []byte(html)} + bd := ht.BodyDomainGrab(response) + + // mailto and tel links should not be parsed by the HTML extractor + // but the regex fallback may still catch domains from the raw text + require.NotNil(t, bd) +} + +func TestFalsePositive_DataURIs(t *testing.T) { + ht, err := New(&DefaultOptions) + require.Nil(t, err) + + html := ` + + data link + ` + response := &Response{Raw: html, Data: []byte(html)} + bd := ht.BodyDomainGrab(response) + + require.Empty(t, bd.Fqdns, "data URIs should not produce fqdns") + require.Empty(t, bd.Domains, "data URIs should not produce domains") +} + +func TestFalsePositive_WebpackChunks(t *testing.T) { + ht, err := New(&DefaultOptions) + require.Nil(t, err) + + html := ` + + + ` + response := &Response{Raw: html, Data: []byte(html)} + bd := ht.BodyDomainGrab(response) + + for _, f := range bd.Fqdns { + require.NotContains(t, f, "chunk") + require.NotContains(t, f, "runtime") + } +} + +// --- Edge case tests --- + +func TestEdgeCase_ProtocolRelativeURLs(t *testing.T) { + ht, err := New(&DefaultOptions) + require.Nil(t, err) + + html := ` + + + + ` + response := &Response{Raw: html, Data: []byte(html)} + bd := ht.BodyDomainGrab(response) + + require.Contains(t, bd.Fqdns, "cdn.proto-relative.example.com") + require.Contains(t, bd.Fqdns, "images.proto-relative.example.net") + require.Contains(t, bd.Fqdns, "fonts.proto-relative.example.org") +} + +func TestEdgeCase_MetaRefreshRedirect(t *testing.T) { + ht, err := New(&DefaultOptions) + require.Nil(t, err) + + html := ` + + ` + response := &Response{Raw: html, Data: []byte(html)} + bd := ht.BodyDomainGrab(response) + + require.Contains(t, bd.Fqdns, "redirect.meta-refresh.example.com") +} + +func TestEdgeCase_JSONLDScripts(t *testing.T) { + ht, err := New(&DefaultOptions) + require.Nil(t, err) + + html := ` + + ` + response := &Response{Raw: html, Data: []byte(html)} + bd := ht.BodyDomainGrab(response) + + require.Contains(t, bd.Fqdns, "www.jsonld-org.example.com") + require.Contains(t, bd.Fqdns, "cdn.jsonld-org.example.com") +} + +func TestEdgeCase_TrailingDots(t *testing.T) { + ht, err := New(&DefaultOptions) + require.Nil(t, err) + + html := ` + link + ` + response := &Response{Raw: html, Data: []byte(html)} + bd := ht.BodyDomainGrab(response) + + require.Contains(t, bd.Fqdns, "trailing-dot.example.com") + for _, f := range bd.Fqdns { + require.False(t, strings.HasSuffix(f, "."), "domain should not have trailing dot: %s", f) + } +} + +func TestEdgeCase_Deduplication(t *testing.T) { + ht, err := New(&DefaultOptions) + require.Nil(t, err) + + html := ` + link1 + link2 + link3 + + ` + response := &Response{Raw: html, Data: []byte(html)} + bd := ht.BodyDomainGrab(response) + + count := 0 + for _, f := range bd.Fqdns { + if f == "dedup.example.com" { + count++ + } + } + require.Equal(t, 1, count, "duplicate fqdns should be deduplicated") +} + +func TestEdgeCase_InputDomainExclusion(t *testing.T) { + ht, err := New(&DefaultOptions) + require.Nil(t, err) + + html := ` + self + other + ` + response := &Response{ + Raw: html, + Data: []byte(html), + Input: "example.com", + } + bd := ht.BodyDomainGrab(response) + + // example.com is the input domain, should be excluded from domains list + for _, d := range bd.Domains { + require.NotEqual(t, "example.com", d, "input domain should be excluded from domains") + } + // self.example.com equals the input, should be excluded from fqdns + for _, f := range bd.Fqdns { + require.NotEqual(t, "example.com", f, "input should be excluded from fqdns") + } + require.Contains(t, bd.Domains, "different.net") + require.Contains(t, bd.Fqdns, "other.different.net") +} + +func TestEdgeCase_InputDomainExclusionWithPort(t *testing.T) { + ht, err := New(&DefaultOptions) + require.Nil(t, err) + + html := ` + self + other + ` + response := &Response{ + Raw: html, + Data: []byte(html), + Input: "example.com:8080", + } + bd := ht.BodyDomainGrab(response) + + for _, d := range bd.Domains { + require.NotEqual(t, "example.com", d, "input host should be excluded from domains") + } + for _, f := range bd.Fqdns { + require.NotEqual(t, "example.com", f, "input host should be excluded from fqdns") + } + require.Contains(t, bd.Domains, "different.net") + require.Contains(t, bd.Fqdns, "other.different.net") +} + +func TestEdgeCase_EmptyAndWhitespaceScripts(t *testing.T) { + ht, err := New(&DefaultOptions) + require.Nil(t, err) + + html := ` + + + + ` + response := &Response{Raw: html, Data: []byte(html)} + bd := ht.BodyDomainGrab(response) + + require.NotNil(t, bd) + require.Empty(t, bd.Fqdns) +} + +func TestEdgeCase_MultipleScriptsSomeBroken(t *testing.T) { + ht, err := New(&DefaultOptions) + require.Nil(t, err) + + html := ` + + + + + ` + response := &Response{Raw: html, Data: []byte(html)} + bd := ht.BodyDomainGrab(response) + + require.Contains(t, bd.Fqdns, "valid.multi-script.example.com") + require.Contains(t, bd.Fqdns, "also-valid.multi-script.example.net") +} + +func TestEdgeCase_URLsWithQueryAndFragment(t *testing.T) { + ht, err := New(&DefaultOptions) + require.Nil(t, err) + + html := ` + query + fragment + both + ` + response := &Response{Raw: html, Data: []byte(html)} + bd := ht.BodyDomainGrab(response) + + require.Contains(t, bd.Fqdns, "query.example.com") + require.Contains(t, bd.Fqdns, "fragment.example.com") + require.Contains(t, bd.Fqdns, "both.example.com") +} + +func TestEdgeCase_MixedCaseURLs(t *testing.T) { + ht, err := New(&DefaultOptions) + require.Nil(t, err) + + html := ` + upper + mixed + ` + response := &Response{Raw: html, Data: []byte(html)} + bd := ht.BodyDomainGrab(response) + + require.Contains(t, bd.Fqdns, "upper.case.example.com") + require.Contains(t, bd.Fqdns, "mixed.case.example.net") +} + +func TestEdgeCase_URLsWithPorts(t *testing.T) { + ht, err := New(&DefaultOptions) + require.Nil(t, err) + + html := ` + port + + ` + response := &Response{Raw: html, Data: []byte(html)} + bd := ht.BodyDomainGrab(response) + + require.Contains(t, bd.Fqdns, "ported.example.com") + require.Contains(t, bd.Fqdns, "ported-js.example.net") + for _, f := range bd.Fqdns { + require.NotContains(t, f, ":", "port numbers should not appear in extracted domains") + } +} + +func TestEdgeCase_URLsWithAuth(t *testing.T) { + ht, err := New(&DefaultOptions) + require.Nil(t, err) + + html := ` + authed + ` + response := &Response{Raw: html, Data: []byte(html)} + bd := ht.BodyDomainGrab(response) + + require.Contains(t, bd.Fqdns, "authed.example.com") +} + +func TestEdgeCase_JSArrowFunctions(t *testing.T) { + domains := make(map[string]struct{}) + fqdns := make(map[string]struct{}) + + script := ` + const fetchData = () => fetch("https://api.arrow.example.com/data"); + const urls = ["a", "b"].map(x => "https://map." + x + ".example.net"); + const handler = async () => { + const res = await fetch("https://async-arrow.example.org/endpoint"); + return res; + }; + ` + extractDomainsFromJS(script, domains, fqdns, "") + + require.Contains(t, fqdns, "api.arrow.example.com") + require.Contains(t, fqdns, "async-arrow.example.org") +} + +func TestEdgeCase_JSObjectNesting(t *testing.T) { + domains := make(map[string]struct{}) + fqdns := make(map[string]struct{}) + + script := ` + var config = { + api: { + base: "https://nested-api.example.com/v1", + endpoints: { + users: "https://nested-users.example.com/users", + deep: { + level: "https://nested-deep.example.net/deep" + } + } + } + }; + ` + extractDomainsFromJS(script, domains, fqdns, "") + + require.Contains(t, fqdns, "nested-api.example.com") + require.Contains(t, fqdns, "nested-users.example.com") + require.Contains(t, fqdns, "nested-deep.example.net") +} + +func TestEdgeCase_JSTryCatchFinally(t *testing.T) { + domains := make(map[string]struct{}) + fqdns := make(map[string]struct{}) + + script := ` + try { + fetch("https://try-block.example.com/api"); + } catch(e) { + fetch("https://catch-block.example.net/error"); + } finally { + fetch("https://finally-block.example.org/cleanup"); + } + ` + extractDomainsFromJS(script, domains, fqdns, "") + + require.Contains(t, fqdns, "try-block.example.com") + require.Contains(t, fqdns, "catch-block.example.net") + require.Contains(t, fqdns, "finally-block.example.org") +} + +func TestEdgeCase_JSConditionalTernary(t *testing.T) { + domains := make(map[string]struct{}) + fqdns := make(map[string]struct{}) + + script := ` + var url = isProd + ? "https://prod.ternary.example.com/api" + : "https://dev.ternary.example.net/api"; + ` + extractDomainsFromJS(script, domains, fqdns, "") + + require.Contains(t, fqdns, "prod.ternary.example.com") + require.Contains(t, fqdns, "dev.ternary.example.net") +} + +func TestEdgeCase_JSSwitchCase(t *testing.T) { + domains := make(map[string]struct{}) + fqdns := make(map[string]struct{}) + + script := ` + switch(env) { + case "prod": + url = "https://prod.switch.example.com/api"; + break; + case "staging": + url = "https://staging.switch.example.net/api"; + break; + default: + url = "https://default.switch.example.org/api"; + } + ` + extractDomainsFromJS(script, domains, fqdns, "") + + require.Contains(t, fqdns, "prod.switch.example.com") + require.Contains(t, fqdns, "staging.switch.example.net") + require.Contains(t, fqdns, "default.switch.example.org") +} + +func TestEdgeCase_JSLoops(t *testing.T) { + domains := make(map[string]struct{}) + fqdns := make(map[string]struct{}) + + script := ` + for (var i = 0; i < 10; i++) { + fetch("https://for-loop.example.com/item"); + } + while (true) { + fetch("https://while-loop.example.net/poll"); + break; + } + do { + fetch("https://do-while.example.org/retry"); + } while (false); + ` + extractDomainsFromJS(script, domains, fqdns, "") + + require.Contains(t, fqdns, "for-loop.example.com") + require.Contains(t, fqdns, "while-loop.example.net") + require.Contains(t, fqdns, "do-while.example.org") +} + +func TestEdgeCase_JSIfElse(t *testing.T) { + domains := make(map[string]struct{}) + fqdns := make(map[string]struct{}) + + script := ` + if (condition) { + url = "https://if-branch.example.com/a"; + } else if (other) { + url = "https://elseif-branch.example.net/b"; + } else { + url = "https://else-branch.example.org/c"; + } + ` + extractDomainsFromJS(script, domains, fqdns, "") + + require.Contains(t, fqdns, "if-branch.example.com") + require.Contains(t, fqdns, "elseif-branch.example.net") + require.Contains(t, fqdns, "else-branch.example.org") +} + +func TestEdgeCase_HTMLEntitiesInURLs(t *testing.T) { + ht, err := New(&DefaultOptions) + require.Nil(t, err) + + // goquery auto-decodes & to & + html := ` + entity + ` + response := &Response{Raw: html, Data: []byte(html)} + bd := ht.BodyDomainGrab(response) + + require.Contains(t, bd.Fqdns, "entity.example.com") +} + +func TestEdgeCase_FormactionAttribute(t *testing.T) { + ht, err := New(&DefaultOptions) + require.Nil(t, err) + + html := ` +
+ +
+ ` + response := &Response{Raw: html, Data: []byte(html)} + bd := ht.BodyDomainGrab(response) + + require.Contains(t, bd.Fqdns, "formaction-btn.example.com") +} + +func TestEdgeCase_CiteAttribute(t *testing.T) { + ht, err := New(&DefaultOptions) + require.Nil(t, err) + + html := ` +
quoted text
+ ` + response := &Response{Raw: html, Data: []byte(html)} + bd := ht.BodyDomainGrab(response) + + require.Contains(t, bd.Fqdns, "cite-source.example.com") +} + +func TestEdgeCase_OpenGraphAndTwitterMeta(t *testing.T) { + ht, err := New(&DefaultOptions) + require.Nil(t, err) + + html := ` + + + + ` + response := &Response{Raw: html, Data: []byte(html)} + bd := ht.BodyDomainGrab(response) + + require.Contains(t, bd.Fqdns, "og-url.example.com") + require.Contains(t, bd.Fqdns, "og-image.example.net") + require.Contains(t, bd.Fqdns, "twitter-img.example.org") +} + +func TestEdgeCase_SrcsetMultipleEntries(t *testing.T) { + ht, err := New(&DefaultOptions) + require.Nil(t, err) + + html := ` + + ` + response := &Response{Raw: html, Data: []byte(html)} + bd := ht.BodyDomainGrab(response) + + require.Contains(t, bd.Fqdns, "srcset1.example.com") + require.Contains(t, bd.Fqdns, "srcset2.example.net") + require.Contains(t, bd.Fqdns, "srcset3.example.org") +} + +func TestEdgeCase_DataAttributes(t *testing.T) { + ht, err := New(&DefaultOptions) + require.Nil(t, err) + + html := ` +
+
+ ` + response := &Response{Raw: html, Data: []byte(html)} + bd := ht.BodyDomainGrab(response) + + require.Contains(t, bd.Fqdns, "data-url-attr.example.com") + require.Contains(t, bd.Fqdns, "data-href-attr.example.net") +} + +func TestEdgeCase_LargeBodyNoPanic(t *testing.T) { + ht, err := New(&DefaultOptions) + require.Nil(t, err) + + // generate a large body with repeated content + var builder strings.Builder + builder.WriteString("") + for i := 0; i < 1000; i++ { + builder.WriteString(`link`) + } + builder.WriteString("") + body := builder.String() + + response := &Response{Raw: body, Data: []byte(body)} + bd := ht.BodyDomainGrab(response) + + require.NotNil(t, bd) + require.Contains(t, bd.Fqdns, "bulk.example.com") +} + +func TestEdgeCase_OnlyRawNoData(t *testing.T) { + ht, err := New(&DefaultOptions) + require.Nil(t, err) + + // r.Data is nil but r.Raw has content — HTML parser skipped, regex catches it + raw := `HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\ntest` + response := &Response{Raw: raw, Data: nil} + bd := ht.BodyDomainGrab(response) + + require.Contains(t, bd.Fqdns, "raw-only.example.com") +} + +func TestEdgeCase_SubdomainVsDomain(t *testing.T) { + ht, err := New(&DefaultOptions) + require.Nil(t, err) + + html := ` + root domain + subdomain + deep sub + ` + response := &Response{Raw: html, Data: []byte(html)} + bd := ht.BodyDomainGrab(response) + + require.Contains(t, bd.Domains, "example.com") + require.Contains(t, bd.Fqdns, "sub.example.com") + require.Contains(t, bd.Fqdns, "deep.sub.example.com") + // root domain (example.com) should be in domains but NOT in fqdns + // (because d == val for a root domain, so the fqdn branch is skipped) + for _, f := range bd.Fqdns { + require.NotEqual(t, "example.com", f, "root domain should not appear in fqdns list") + } +} + +func TestEdgeCase_InternationalTLDs(t *testing.T) { + ht, err := New(&DefaultOptions) + require.Nil(t, err) + + html := ` + uk + au + ` + response := &Response{Raw: html, Data: []byte(html)} + bd := ht.BodyDomainGrab(response) + + require.Contains(t, bd.Fqdns, "test.example.co.uk") + require.Contains(t, bd.Fqdns, "test.example.com.au") +} + +// helper for IP check test +func isAllNumericParts(d string) bool { + for _, part := range strings.Split(d, ".") { + allDigits := true + for _, c := range part { + if c < '0' || c > '9' { + allDigits = false + break + } + } + if !allDigits { + return false + } + } + return true +} + +// --- Unit tests for hostnameFromURL --- + +func TestHostnameFromURL(t *testing.T) { + tests := []struct { + name string + input string + expected string + }{ + {"https url", "https://example.com/path", "example.com"}, + {"http url", "http://sub.example.com/path", "sub.example.com"}, + {"protocol-relative", "//cdn.example.com/file.js", "cdn.example.com"}, + {"with port", "https://example.com:8443/api", "example.com"}, + {"with auth", "https://user:pass@example.com/page", "example.com"}, + {"with query", "https://example.com/path?q=1", "example.com"}, + {"with fragment", "https://example.com/path#section", "example.com"}, + {"empty string", "", ""}, + {"hash only", "#section", ""}, + {"javascript scheme", "javascript:void(0)", ""}, + {"data uri", "data:text/html,

hi

", ""}, + {"mailto", "mailto:user@example.com", ""}, + {"tel", "tel:+1234567890", ""}, + {"blob", "blob:https://example.com/uuid", ""}, + {"about", "about:blank", ""}, + {"relative path", "/path/to/page", ""}, + {"bare domain no scheme", "example.com", ""}, + {"no dots", "https://localhost/path", ""}, + {"ip address", "https://192.168.1.1/admin", "192.168.1.1"}, + {"uppercase scheme", "HTTPS://EXAMPLE.COM/PATH", "EXAMPLE.COM"}, + {"whitespace", " https://trimmed.example.com/path ", "trimmed.example.com"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + result := hostnameFromURL(tc.input) + require.Equal(t, tc.expected, result) + }) + } +} + +// --- Unit tests for addDomainCandidate --- + +func TestAddDomainCandidate(t *testing.T) { + t.Run("valid fqdn", func(t *testing.T) { + domains := make(map[string]struct{}) + fqdns := make(map[string]struct{}) + addDomainCandidate("sub.example.com", domains, fqdns, "") + require.Contains(t, fqdns, "sub.example.com") + require.Contains(t, domains, "example.com") + }) + + t.Run("root domain only goes to domains not fqdns", func(t *testing.T) { + domains := make(map[string]struct{}) + fqdns := make(map[string]struct{}) + addDomainCandidate("example.com", domains, fqdns, "") + require.Contains(t, domains, "example.com") + require.Empty(t, fqdns) + }) + + t.Run("trailing dot is stripped", func(t *testing.T) { + domains := make(map[string]struct{}) + fqdns := make(map[string]struct{}) + addDomainCandidate("sub.example.com.", domains, fqdns, "") + require.Contains(t, fqdns, "sub.example.com") + }) + + t.Run("uppercase is lowered", func(t *testing.T) { + domains := make(map[string]struct{}) + fqdns := make(map[string]struct{}) + addDomainCandidate("SUB.EXAMPLE.COM", domains, fqdns, "") + require.Contains(t, fqdns, "sub.example.com") + }) + + t.Run("input domain excluded from domains", func(t *testing.T) { + domains := make(map[string]struct{}) + fqdns := make(map[string]struct{}) + addDomainCandidate("sub.example.com", domains, fqdns, "example.com") + require.Empty(t, domains) + require.Contains(t, fqdns, "sub.example.com") + }) + + t.Run("input fqdn excluded from fqdns", func(t *testing.T) { + domains := make(map[string]struct{}) + fqdns := make(map[string]struct{}) + addDomainCandidate("sub.example.com", domains, fqdns, "sub.example.com") + require.Empty(t, fqdns) + }) + + t.Run("input host with port excluded from domains", func(t *testing.T) { + domains := make(map[string]struct{}) + fqdns := make(map[string]struct{}) + addDomainCandidate("sub.example.com", domains, fqdns, "example.com:443") + require.Empty(t, domains) + require.Contains(t, fqdns, "sub.example.com") + }) + + t.Run("empty string rejected", func(t *testing.T) { + domains := make(map[string]struct{}) + fqdns := make(map[string]struct{}) + addDomainCandidate("", domains, fqdns, "") + require.Empty(t, domains) + require.Empty(t, fqdns) + }) + + t.Run("whitespace only rejected", func(t *testing.T) { + domains := make(map[string]struct{}) + fqdns := make(map[string]struct{}) + addDomainCandidate(" ", domains, fqdns, "") + require.Empty(t, domains) + require.Empty(t, fqdns) + }) + + t.Run("ip address rejected", func(t *testing.T) { + domains := make(map[string]struct{}) + fqdns := make(map[string]struct{}) + addDomainCandidate("192.168.1.1", domains, fqdns, "") + require.Empty(t, domains) + require.Empty(t, fqdns) + }) + + t.Run("single label rejected", func(t *testing.T) { + domains := make(map[string]struct{}) + fqdns := make(map[string]struct{}) + addDomainCandidate("localhost", domains, fqdns, "") + require.Empty(t, domains) + require.Empty(t, fqdns) + }) +} + +// --- Benchmarks --- + +func BenchmarkBodyDomainGrab_HackerOne(b *testing.B) { + ht, err := New(&DefaultOptions) + if err != nil { + b.Fatal(err) + } + response := &Response{ + Raw: rawResponse, + Data: []byte(rawResponse), + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + ht.BodyDomainGrab(response) + } +} + +func BenchmarkBodyDomainGrab_SmallPage(b *testing.B) { + ht, err := New(&DefaultOptions) + if err != nil { + b.Fatal(err) + } + response := &Response{ + Raw: sampleWithJS, + Data: []byte(sampleWithJS), + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + ht.BodyDomainGrab(response) + } +} + +func BenchmarkBodyDomainGrab_RegexOnly(b *testing.B) { + response := &Response{ + Raw: rawResponse, + Data: []byte(rawResponse), + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + domains := make(map[string]struct{}) + fqdns := make(map[string]struct{}) + extractDomainsFromRegex(response.Raw, domains, fqdns, "") + } +} + +func BenchmarkBodyDomainGrab_HTMLOnly(b *testing.B) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + domains := make(map[string]struct{}) + fqdns := make(map[string]struct{}) + extractDomainsFromHTML([]byte(rawResponse), domains, fqdns, "") + } +} + +func BenchmarkBodyDomainGrab_JSOnly(b *testing.B) { + scripts := []string{} + extractDomainsFromHTML([]byte(rawResponse), make(map[string]struct{}), make(map[string]struct{}), "") + doc, _ := goquery.NewDocumentFromReader(bytes.NewReader([]byte(rawResponse))) + doc.Find("script").Each(func(_ int, s *goquery.Selection) { + if _, ok := s.Attr("src"); !ok { + if text := s.Text(); strings.TrimSpace(text) != "" { + scripts = append(scripts, text) + } + } + }) + b.ResetTimer() + for i := 0; i < b.N; i++ { + for _, script := range scripts { + domains := make(map[string]struct{}) + fqdns := make(map[string]struct{}) + extractDomainsFromJS(script, domains, fqdns, "") + } + } +} + +func BenchmarkBodyDomainGrab_JSON(b *testing.B) { + ht, err := New(&DefaultOptions) + if err != nil { + b.Fatal(err) + } + json := `{"url":"https://api.example.com/v1","cdn":"https://cdn.example.net/assets","callback":"https://hooks.example.org/notify"}` + response := &Response{ + Raw: json, + Data: []byte(json), + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + ht.BodyDomainGrab(response) + } +} + +func BenchmarkBodyDomainGrab_PlainText(b *testing.B) { + ht, err := New(&DefaultOptions) + if err != nil { + b.Fatal(err) + } + text := `'api.example.com' and 'cdn.example.net' are the endpoints` + response := &Response{ + Raw: text, + Data: []byte(text), + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + ht.BodyDomainGrab(response) + } +} diff --git a/common/httpx/filter.go b/common/httpx/filter.go index e553abcbb..44ac72ef0 100644 --- a/common/httpx/filter.go +++ b/common/httpx/filter.go @@ -53,8 +53,11 @@ type FilterCustom struct { func (f FilterCustom) Filter(response *Response) (bool, error) { for _, callback := range f.CallBacks { ok, err := callback(response) - if ok && err == nil { - return true, err + if err != nil { + return false, err + } + if ok { + return true, nil } } diff --git a/common/httpx/filter_test.go b/common/httpx/filter_test.go new file mode 100644 index 000000000..8c96e43a4 --- /dev/null +++ b/common/httpx/filter_test.go @@ -0,0 +1,73 @@ +package httpx + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestFilterCustomErrorPropagation(t *testing.T) { + t.Run("error from callback is returned, not swallowed", func(t *testing.T) { + expectedErr := errors.New("callback failure") + callback := func(response *Response) (bool, error) { + return true, expectedErr + } + filter := FilterCustom{CallBacks: []CustomCallback{callback}} + ok, err := filter.Filter(&Response{}) + require.False(t, ok, "ok should be false when callback returns an error") + require.ErrorIs(t, err, expectedErr, "error from callback should be propagated") + }) + + t.Run("error from callback with ok=false is returned", func(t *testing.T) { + expectedErr := errors.New("callback failure") + callback := func(response *Response) (bool, error) { + return false, expectedErr + } + filter := FilterCustom{CallBacks: []CustomCallback{callback}} + ok, err := filter.Filter(&Response{}) + require.False(t, ok) + require.ErrorIs(t, err, expectedErr) + }) + + t.Run("first matching callback without error returns true", func(t *testing.T) { + callbacks := []CustomCallback{ + func(response *Response) (bool, error) { return false, nil }, + func(response *Response) (bool, error) { return true, nil }, + func(response *Response) (bool, error) { return true, nil }, + } + filter := FilterCustom{CallBacks: callbacks} + ok, err := filter.Filter(&Response{}) + require.True(t, ok) + require.NoError(t, err) + }) + + t.Run("error stops remaining callbacks", func(t *testing.T) { + called := 0 + filter := FilterCustom{CallBacks: []CustomCallback{ + func(*Response) (bool, error) { + called++ + return false, errors.New("fail") + }, + func(*Response) (bool, error) { + called++ + return true, nil + }, + }} + ok, err := filter.Filter(&Response{}) + require.False(t, ok) + require.Error(t, err) + require.Equal(t, 1, called) + }) + + t.Run("no callbacks match returns false with nil error", func(t *testing.T) { + callbacks := []CustomCallback{ + func(response *Response) (bool, error) { return false, nil }, + func(response *Response) (bool, error) { return false, nil }, + } + filter := FilterCustom{CallBacks: callbacks} + ok, err := filter.Filter(&Response{}) + require.False(t, ok) + require.NoError(t, err) + }) +} diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index 7820a0229..b7133828d 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -18,7 +18,9 @@ import ( "github.com/microcosm-cc/bluemonday" "github.com/projectdiscovery/cdncheck" "github.com/projectdiscovery/fastdialer/fastdialer" + "github.com/projectdiscovery/fastdialer/fastdialer/ja3" "github.com/projectdiscovery/fastdialer/fastdialer/ja3/impersonate" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/httpx/common/httputilz" "github.com/projectdiscovery/networkpolicy" "github.com/projectdiscovery/rawhttp" @@ -141,12 +143,7 @@ func New(options *Options) (*HTTPX, error) { } transport := &http.Transport{ DialContext: httpx.Dialer.Dial, - DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) { - if options.TlsImpersonate { - return httpx.Dialer.DialTLSWithConfigImpersonate(ctx, network, addr, &tls.Config{InsecureSkipVerify: true, MinVersion: tls.VersionTLS10}, impersonate.Random, nil) - } - return httpx.Dialer.DialTLS(ctx, network, addr) - }, + DialTLSContext: httpx.buildTLSDialer(options), MaxIdleConnsPerHost: -1, TLSClientConfig: &tls.Config{ InsecureSkipVerify: true, @@ -220,6 +217,40 @@ func New(options *Options) (*HTTPX, error) { return httpx, nil } +func (h *HTTPX) buildTLSDialer(options *Options) func(ctx context.Context, network, addr string) (net.Conn, error) { + if options.TlsImpersonate == "" { + return func(ctx context.Context, network, addr string) (net.Conn, error) { + return h.Dialer.DialTLS(ctx, network, addr) + } + } + + tlsCfg := &tls.Config{InsecureSkipVerify: true, MinVersion: tls.VersionTLS10} + + strategy, identity := resolveImpersonateStrategy(options.TlsImpersonate) + + return func(ctx context.Context, network, addr string) (net.Conn, error) { + return h.Dialer.DialTLSWithConfigImpersonate(ctx, network, addr, tlsCfg, strategy, identity) + } +} + +func resolveImpersonateStrategy(value string) (impersonate.Strategy, *impersonate.Identity) { + switch strings.ToLower(value) { + case "", "chrome": + return impersonate.Chrome, nil + case "random": + // random JA3 mode was removed due to unsupported curve picks; keep chrome for compatibility. + return impersonate.Chrome, nil + default: + spec, err := ja3.ParseWithJa3(value) + if err != nil { + gologger.Warning().Msgf("invalid tls-impersonate value %q: %v; falling back to chrome", value, err) + return impersonate.Chrome, nil + } + identity := impersonate.Identity(*spec) + return impersonate.Custom, &identity + } +} + // Do http request func (h *HTTPX) Do(req *retryablehttp.Request, unsafeOptions UnsafeOptions) (*Response, error) { timeStart := time.Now() diff --git a/common/httpx/httpx_test.go b/common/httpx/httpx_test.go index a58d61014..b2910e147 100644 --- a/common/httpx/httpx_test.go +++ b/common/httpx/httpx_test.go @@ -2,6 +2,7 @@ package httpx import ( "net/http" + "net/http/httptest" "testing" "time" @@ -22,11 +23,19 @@ func TestDo(t *testing.T) { }) t.Run("content-length with binary body", func(t *testing.T) { - req, err := retryablehttp.NewRequest(http.MethodGet, "https://www.w3schools.com/images/favicon.ico", nil) + body := []byte{0x47, 0x49, 0x46, 0x38, 0x39, 0x61, 0x00, 0x01, 0x00, 0x01, 0x80, 0x00, 0x00, 0xff} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "image/gif") + _, _ = w.Write(body) + })) + defer srv.Close() + + req, err := retryablehttp.NewRequest(http.MethodGet, srv.URL, nil) require.Nil(t, err) resp, err := ht.Do(req, UnsafeOptions{}) require.Nil(t, err) - require.Greater(t, len(resp.Raw), 800) + require.Equal(t, len(body), resp.ContentLength) + require.Equal(t, body, resp.RawData) }) } diff --git a/common/httpx/option.go b/common/httpx/option.go index ce43aad70..aa5955191 100644 --- a/common/httpx/option.go +++ b/common/httpx/option.go @@ -67,7 +67,7 @@ type Options struct { Resolvers []string customCookies []*http.Cookie SniName string - TlsImpersonate bool + TlsImpersonate string NetworkPolicy *networkpolicy.NetworkPolicy CDNCheckClient *cdncheck.Client Protocol Proto diff --git a/common/httpx/pipeline.go b/common/httpx/pipeline.go index b6b7b7817..e1f4b0188 100644 --- a/common/httpx/pipeline.go +++ b/common/httpx/pipeline.go @@ -29,6 +29,7 @@ func (h *HTTPX) SupportPipeline(protocol, method, host string, port int) bool { if err != nil { return false } + defer func() { _ = conn.Close() }() // send some probes nprobes := 10 for i := 0; i < nprobes; i++ { diff --git a/common/httpx/pipeline_test.go b/common/httpx/pipeline_test.go new file mode 100644 index 000000000..01eb683ff --- /dev/null +++ b/common/httpx/pipeline_test.go @@ -0,0 +1,51 @@ +package httpx + +import ( + "io" + "net" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestSupportPipelineClosesConn(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer func() { _ = ln.Close() }() + + _, portStr, err := net.SplitHostPort(ln.Addr().String()) + require.NoError(t, err) + port, err := strconv.Atoi(portStr) + require.NoError(t, err) + + closed := make(chan struct{}) + go func() { + conn, err := ln.Accept() + if err != nil { + return + } + defer func() { _ = conn.Close() }() + buf := make([]byte, 64*1024) + _ = conn.SetReadDeadline(time.Now().Add(2 * time.Second)) + _, _ = conn.Read(buf) + _, _ = conn.Write([]byte("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\nHTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")) + _, _ = io.Copy(io.Discard, conn) + close(closed) + }() + + h := &HTTPX{} + _ = h.SupportPipeline("http", "GET", "127.0.0.1", port) + + select { + case <-closed: + case <-time.After(3 * time.Second): + t.Fatal("pipeline probe connection was not closed") + } +} + +func TestSupportPipelineDialError(t *testing.T) { + h := &HTTPX{} + require.False(t, h.SupportPipeline("http", "GET", "127.0.0.1", 1)) +} diff --git a/common/httpx/test-data/sample_with_js.html b/common/httpx/test-data/sample_with_js.html new file mode 100644 index 000000000..93b7eab19 --- /dev/null +++ b/common/httpx/test-data/sample_with_js.html @@ -0,0 +1,52 @@ + + + + Test Page + + + + + + + Link + Another + Relative + Hash + JS + + photo + +
+ +
+ + + +
+ + + + + + + + Text with a bare domain like bare.textdomain.example.com mentioned here. + + diff --git a/common/httpx/tls_impersonate_test.go b/common/httpx/tls_impersonate_test.go new file mode 100644 index 000000000..2ff5b7ac0 --- /dev/null +++ b/common/httpx/tls_impersonate_test.go @@ -0,0 +1,377 @@ +package httpx + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "math/big" + "net" + "sync" + "testing" + "time" + + "github.com/projectdiscovery/fastdialer/fastdialer" + "github.com/projectdiscovery/fastdialer/fastdialer/ja3/impersonate" + "github.com/stretchr/testify/require" +) + +// capturedHello holds the ClientHello details captured by the test TLS server. +type capturedHello struct { + CipherSuites []uint16 + SupportedCurves []tls.CurveID + ServerName string + SupportedProtos []string +} + +// startTLSServer creates a local TLS server that captures ClientHello info +// from each incoming connection. It returns the listener address and a function +// to retrieve the most recently captured hello. +func startTLSServer(t *testing.T) (string, func() *capturedHello) { + t.Helper() + + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "localhost"}, + NotBefore: time.Now(), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + DNSNames: []string{"localhost"}, + } + certDER, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + require.NoError(t, err) + + cert := tls.Certificate{ + Certificate: [][]byte{certDER}, + PrivateKey: key, + } + + var mu sync.Mutex + var latest *capturedHello + + tlsCfg := &tls.Config{ + Certificates: []tls.Certificate{cert}, + GetConfigForClient: func(info *tls.ClientHelloInfo) (*tls.Config, error) { + hello := &capturedHello{ + CipherSuites: info.CipherSuites, + SupportedCurves: info.SupportedCurves, + ServerName: info.ServerName, + SupportedProtos: info.SupportedProtos, + } + mu.Lock() + latest = hello + mu.Unlock() + return nil, nil + }, + } + + ln, err := tls.Listen("tcp", "127.0.0.1:0", tlsCfg) + require.NoError(t, err) + t.Cleanup(func() { + _ = ln.Close() + }) + + go func() { + for { + conn, err := ln.Accept() + if err != nil { + return + } + go func() { + defer func() { + _ = conn.Close() + }() + buf := make([]byte, 1) + _, _ = conn.Read(buf) + }() + } + }() + + getHello := func() *capturedHello { + mu.Lock() + defer mu.Unlock() + return latest + } + + return ln.Addr().String(), getHello +} + +// --- Unit tests for resolveImpersonateStrategy --- + +func TestResolveImpersonateStrategy(t *testing.T) { + t.Run("empty defaults to chrome", func(t *testing.T) { + strategy, identity := resolveImpersonateStrategy("") + require.Equal(t, impersonate.Chrome, strategy) + require.Nil(t, identity) + }) + + t.Run("chrome", func(t *testing.T) { + strategy, identity := resolveImpersonateStrategy("chrome") + require.Equal(t, impersonate.Chrome, strategy) + require.Nil(t, identity) + }) + + t.Run("chrome case insensitive", func(t *testing.T) { + strategy, identity := resolveImpersonateStrategy("CHROME") + require.Equal(t, impersonate.Chrome, strategy) + require.Nil(t, identity) + }) + + t.Run("valid ja3 string", func(t *testing.T) { + ja3str := "771,49195-49196,0-23-65281-10-11-35-16-5-13-18,23-24,0" + strategy, identity := resolveImpersonateStrategy(ja3str) + require.Equal(t, impersonate.Custom, strategy) + require.NotNil(t, identity) + }) + + t.Run("random maps to chrome", func(t *testing.T) { + strategy, identity := resolveImpersonateStrategy("random") + require.Equal(t, impersonate.Chrome, strategy) + require.Nil(t, identity) + }) + + t.Run("invalid ja3 falls back to chrome", func(t *testing.T) { + strategy, identity := resolveImpersonateStrategy("not-a-ja3-string") + require.Equal(t, impersonate.Chrome, strategy) + require.Nil(t, identity) + }) + + t.Run("partial ja3 falls back to chrome", func(t *testing.T) { + strategy, identity := resolveImpersonateStrategy("771,4865") + require.Equal(t, impersonate.Chrome, strategy) + require.Nil(t, identity) + }) +} + +// Integration tests with local TLS server + +func TestTLSImpersonate_DefaultGoTLS(t *testing.T) { + addr, getHello := startTLSServer(t) + + opts := fastdialer.DefaultOptions + opts.EnableFallback = false + fd, err := fastdialer.NewDialer(opts) + require.NoError(t, err) + defer fd.Close() + + conn, err := fd.DialTLSWithConfigImpersonate( + context.Background(), "tcp", addr, + &tls.Config{InsecureSkipVerify: true}, + impersonate.None, nil, + ) + require.NoError(t, err) + _ = conn.Close() + + hello := getHello() + require.NotNil(t, hello) + require.NotEmpty(t, hello.CipherSuites, "default Go TLS should have cipher suites") +} + +func TestTLSImpersonate_Chrome(t *testing.T) { + addr, getHello := startTLSServer(t) + + opts := fastdialer.DefaultOptions + opts.EnableFallback = false + fd, err := fastdialer.NewDialer(opts) + require.NoError(t, err) + defer fd.Close() + + conn, err := fd.DialTLSWithConfigImpersonate( + context.Background(), "tcp", addr, + &tls.Config{InsecureSkipVerify: true}, + impersonate.Chrome, nil, + ) + require.NoError(t, err) + _ = conn.Close() + + hello := getHello() + require.NotNil(t, hello) + require.NotEmpty(t, hello.CipherSuites) + hasGrease := false + for _, cs := range hello.CipherSuites { + if cs&0x0f0f == 0x0a0a { + hasGrease = true + break + } + } + require.True(t, hasGrease, "Chrome impersonation should include GREASE cipher suite values") +} + +func TestTLSImpersonate_CustomJA3(t *testing.T) { + addr, getHello := startTLSServer(t) + + ja3Str := "771,49195-49196,0-23-65281-10-11-35-16-5-13-18,23-24,0" + strategy, identity := resolveImpersonateStrategy(ja3Str) + require.Equal(t, impersonate.Custom, strategy) + require.NotNil(t, identity) + + opts := fastdialer.DefaultOptions + opts.EnableFallback = false + fd, err := fastdialer.NewDialer(opts) + require.NoError(t, err) + defer fd.Close() + + conn, err := fd.DialTLSWithConfigImpersonate( + context.Background(), "tcp", addr, + &tls.Config{InsecureSkipVerify: true}, + strategy, identity, + ) + require.NoError(t, err) + require.NotNil(t, conn) + _ = conn.Close() + + hello := getHello() + require.NotNil(t, hello) + + require.Equal(t, []uint16{49195, 49196}, hello.CipherSuites, + "custom JA3 should contain exactly the specified cipher suites") + + expectedCurves := []tls.CurveID{23, 24} + require.Equal(t, expectedCurves, hello.SupportedCurves, + "custom JA3 should contain exactly the specified curves") +} + +func TestTLSImpersonate_ChromeDiffersFromDefault(t *testing.T) { + addr, getHello := startTLSServer(t) + + opts := fastdialer.DefaultOptions + opts.EnableFallback = false + fd, err := fastdialer.NewDialer(opts) + require.NoError(t, err) + defer fd.Close() + + conn, err := fd.DialTLSWithConfigImpersonate( + context.Background(), "tcp", addr, + &tls.Config{InsecureSkipVerify: true}, + impersonate.None, nil, + ) + require.NoError(t, err) + _ = conn.Close() + defaultHello := getHello() + + conn, err = fd.DialTLSWithConfigImpersonate( + context.Background(), "tcp", addr, + &tls.Config{InsecureSkipVerify: true}, + impersonate.Chrome, nil, + ) + require.NoError(t, err) + _ = conn.Close() + chromeHello := getHello() + + require.NotNil(t, defaultHello) + require.NotNil(t, chromeHello) + + require.NotEqual(t, defaultHello.CipherSuites, chromeHello.CipherSuites, + "Chrome impersonation should produce different cipher suites than default Go TLS") +} + +func TestTLSImpersonate_CustomJA3DiffersFromDefault(t *testing.T) { + addr, getHello := startTLSServer(t) + + opts := fastdialer.DefaultOptions + opts.EnableFallback = false + fd, err := fastdialer.NewDialer(opts) + require.NoError(t, err) + defer fd.Close() + + // Default (no impersonation) + conn, err := fd.DialTLSWithConfigImpersonate( + context.Background(), "tcp", addr, + &tls.Config{InsecureSkipVerify: true}, + impersonate.None, nil, + ) + require.NoError(t, err) + _ = conn.Close() + defaultHello := getHello() + + ja3Str := "771,49195-49196,0-23-65281-10-11-35-16-5-13-18,23-24,0" + strategy, identity := resolveImpersonateStrategy(ja3Str) + require.Equal(t, impersonate.Custom, strategy) + + conn, err = fd.DialTLSWithConfigImpersonate( + context.Background(), "tcp", addr, + &tls.Config{InsecureSkipVerify: true}, + strategy, identity, + ) + require.NoError(t, err) + _ = conn.Close() + customHello := getHello() + + require.NotNil(t, defaultHello) + require.NotNil(t, customHello) + + require.NotEqual(t, defaultHello.CipherSuites, customHello.CipherSuites, + "custom JA3 should produce different cipher suites than default Go TLS") +} + +func TestTLSImpersonate_EndToEnd_HTTPX(t *testing.T) { + addr, getHello := startTLSServer(t) + + tests := []struct { + name string + strategy string + wantErr bool + }{ + {"disabled", "", false}, + {"chrome", "chrome", false}, + {"ja3", "771,49195-49196,0-23-65281-10-11-35-16-5-13-18,23-24,0", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + options := DefaultOptions + options.TlsImpersonate = tt.strategy + + ht, err := New(&options) + require.NoError(t, err) + + dialer := ht.buildTLSDialer(&options) + conn, err := dialer(context.Background(), "tcp", addr) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + require.NotNil(t, conn) + _ = conn.Close() + + if tt.strategy != "" { + hello := getHello() + require.NotNil(t, hello) + require.NotEmpty(t, hello.CipherSuites) + } + }) + } +} + +func TestTLSImpersonate_EndToEnd_JA3(t *testing.T) { + addr, getHello := startTLSServer(t) + + ja3Str := "771,49195-49196,0-23-65281-10-11-35-16-5-13-18,23-24,0" + options := DefaultOptions + options.TlsImpersonate = ja3Str + + ht, err := New(&options) + require.NoError(t, err) + + dialer := ht.buildTLSDialer(&options) + conn, err := dialer(context.Background(), "tcp", addr) + require.NoError(t, err) + require.NotNil(t, conn) + _ = conn.Close() + + hello := getHello() + require.NotNil(t, hello) + + require.Equal(t, []uint16{49195, 49196}, hello.CipherSuites, + "JA3 end-to-end cipher suites should match exactly") + require.Equal(t, []tls.CurveID{23, 24}, hello.SupportedCurves, + "JA3 end-to-end curves should match exactly") +} diff --git a/go.mod b/go.mod index 538142bc6..b677c8abb 100644 --- a/go.mod +++ b/go.mod @@ -1,13 +1,13 @@ module github.com/projectdiscovery/httpx -go 1.26 +go 1.26.0 require ( github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 github.com/PuerkitoBio/goquery v1.12.0 github.com/akrylysov/pogreb v0.10.2 // indirect github.com/corona10/goimagehash v1.1.0 - github.com/go-faker/faker/v4 v4.9.0 + github.com/go-faker/faker/v4 v4.11.0 github.com/go-rod/rod v0.116.2 github.com/golang/snappy v0.0.4 // indirect github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf @@ -16,45 +16,46 @@ require ( github.com/logrusorgru/aurora v2.0.3+incompatible github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 github.com/microcosm-cc/bluemonday v1.0.27 - github.com/miekg/dns v1.1.68 // indirect + github.com/miekg/dns v1.1.72 // indirect github.com/pkg/errors v0.9.1 github.com/projectdiscovery/asnmap v1.1.1 - github.com/projectdiscovery/cdncheck v1.2.42 - github.com/projectdiscovery/clistats v0.1.4 - github.com/projectdiscovery/dsl v0.8.20 - github.com/projectdiscovery/fastdialer v0.5.11 + github.com/projectdiscovery/cdncheck v1.2.50 + github.com/projectdiscovery/clistats v0.1.5 + github.com/projectdiscovery/dsl v0.8.21 + github.com/projectdiscovery/fastdialer v0.5.17 github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/goconfig v0.0.1 - github.com/projectdiscovery/goflags v0.1.74 - github.com/projectdiscovery/gologger v1.1.71 + github.com/projectdiscovery/goflags v0.1.76 + github.com/projectdiscovery/gologger v1.1.72 github.com/projectdiscovery/hmap v0.0.101 github.com/projectdiscovery/mapcidr v1.1.97 - github.com/projectdiscovery/networkpolicy v0.1.41 + github.com/projectdiscovery/networkpolicy v0.1.46 github.com/projectdiscovery/ratelimit v0.0.88 - github.com/projectdiscovery/rawhttp v0.1.90 - github.com/projectdiscovery/retryablehttp-go v1.3.16 - github.com/projectdiscovery/tlsx v1.2.2 + github.com/projectdiscovery/rawhttp v0.1.91 + github.com/projectdiscovery/retryablehttp-go v1.3.23 + github.com/projectdiscovery/tlsx v1.3.2 github.com/projectdiscovery/useragent v0.0.108 github.com/projectdiscovery/utils v0.11.1 - github.com/projectdiscovery/wappalyzergo v0.2.87 + github.com/projectdiscovery/wappalyzergo v0.2.95 github.com/rs/xid v1.6.0 github.com/spaolacci/murmur3 v1.1.0 - github.com/stretchr/testify v1.11.1 + github.com/stretchr/testify v1.12.1 github.com/zmap/zcrypto v0.0.0-20240803002437-3a861682ac77 go.etcd.io/bbolt v1.4.0 // indirect go.uber.org/multierr v1.11.0 - golang.org/x/exp v0.0.0-20250911091902-df9299821621 - golang.org/x/net v0.56.0 - golang.org/x/sys v0.46.0 // indirect - golang.org/x/text v0.38.0 + golang.org/x/exp v0.0.0-20260112195511-716be5621a96 + golang.org/x/net v0.58.0 + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.41.0 ) require ( + github.com/dop251/goja v0.0.0-20260311135729-065cd970411c github.com/dustin/go-humanize v1.0.1 github.com/go-sql-driver/mysql v1.10.0 github.com/go-viper/mapstructure/v2 v2.5.0 github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 - github.com/happyhackingspace/dit v0.0.28 + github.com/happyhackingspace/dit v0.0.33 github.com/lib/pq v1.12.3 github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30f7193 github.com/seh-msft/burpxml v1.0.1 @@ -66,7 +67,7 @@ require ( require ( aead.dev/minisign v0.2.0 // indirect filippo.io/edwards25519 v1.2.0 // indirect - github.com/Masterminds/semver/v3 v3.4.0 // indirect + github.com/Masterminds/semver/v3 v3.5.0 // indirect github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect github.com/VividCortex/ewma v1.2.0 // indirect github.com/alecthomas/chroma/v2 v2.14.0 // indirect @@ -82,7 +83,6 @@ require ( github.com/cheggaaa/pb/v3 v3.1.6 // indirect github.com/cloudflare/cfssl v1.6.4 // indirect github.com/cnf/structhash v0.0.0-20250313080605-df4c6cc74a9a // indirect - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/djherbis/times v1.6.0 // indirect github.com/dlclark/regexp2 v1.11.5 // indirect @@ -91,18 +91,19 @@ require ( github.com/ebitengine/purego v0.10.0 // indirect github.com/fatih/color v1.18.0 // indirect github.com/felixge/fgprof v0.9.5 // indirect - github.com/gaissmai/bart v0.28.0 // indirect + github.com/gaissmai/bart v0.29.0 // indirect github.com/go-ole/go-ole v1.2.6 // indirect + github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/google/certificate-transparency-go v1.3.2 // indirect github.com/google/go-github/v30 v30.1.0 // indirect - github.com/google/go-querystring v1.1.0 // indirect + github.com/google/go-querystring v1.2.0 // indirect github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/css v1.0.1 // indirect github.com/gosimple/slug v1.15.0 // indirect github.com/gosimple/unidecode v1.0.1 // indirect - github.com/hashicorp/go-version v1.8.0 // indirect + github.com/hashicorp/go-version v1.9.0 // indirect github.com/iangcarroll/cookiemonster v1.6.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kataras/jwt v0.1.10 // indirect @@ -120,9 +121,8 @@ require ( github.com/muesli/reflow v0.3.0 // indirect github.com/muesli/termenv v0.16.0 // indirect github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect - github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect - github.com/projectdiscovery/blackrock v0.0.1 // indirect + github.com/projectdiscovery/blackrock v0.0.2 // indirect github.com/projectdiscovery/freeport v0.0.7 // indirect github.com/projectdiscovery/gostruct v0.0.2 // indirect github.com/projectdiscovery/govaluate v0.0.0-20260504230327-80320480bb6e // indirect @@ -161,12 +161,13 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/zcalusic/sysinfo v1.0.2 // indirect github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect - golang.org/x/crypto v0.53.0 // indirect - golang.org/x/mod v0.36.0 // indirect - golang.org/x/oauth2 v0.34.0 // indirect - golang.org/x/sync v0.21.0 // indirect - golang.org/x/term v0.44.0 // indirect - golang.org/x/time v0.14.0 // indirect - golang.org/x/tools v0.45.0 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/crypto v0.55.0 // indirect + golang.org/x/mod v0.38.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/term v0.45.0 // indirect + golang.org/x/time v0.15.0 // indirect + golang.org/x/tools v0.48.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) diff --git a/go.sum b/go.sum index 827d7e42c..037d6823e 100644 --- a/go.sum +++ b/go.sum @@ -3,8 +3,8 @@ aead.dev/minisign v0.2.0/go.mod h1:zdq6LdSd9TbuSxchxwhpA9zEb9YXcVGoE8JakuiGaIQ= cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= -github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= -github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE= +github.com/Masterminds/semver/v3 v3.5.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 h1:KFac3SiGbId8ub47e7kd2PLZeACxc1LkiiNoDOFRClE= github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057/go.mod h1:iLB2pivrPICvLOuROKmlqURtFIEsoJZaMidQfCG1+D4= github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 h1:ZbFL+BDfBqegi+/Ssh7im5+aQfBRx6it+kHnC7jaDU8= @@ -76,6 +76,8 @@ github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZ github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dop251/goja v0.0.0-20260311135729-065cd970411c h1:OcLmPfx1T1RmZVHHFwWMPaZDdRf0DBMZOFMVWJa7Pdk= +github.com/dop251/goja v0.0.0-20260311135729-065cd970411c/go.mod h1:MxLav0peU43GgvwVgNbLAj1s/bSGboKkhuULvq/7hx4= github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 h1:2tV76y6Q9BB+NEBasnqvs7e49aEBFI8ejC89PSnWH+4= github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= @@ -94,14 +96,16 @@ github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7z github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/gaissmai/bart v0.28.0 h1:89yZLo8NmyqD0RYgJ3QO9HhqqGGw+oWhf90cZm69Lko= -github.com/gaissmai/bart v0.28.0/go.mod h1:GREWQfTLRWz/c5FTOsIw+KkscuFkIV5t8Rp7Nd1Td5c= -github.com/go-faker/faker/v4 v4.9.0 h1:a4HXLwueuTCtgF93VpUsl8Zd2nG1VH2SgNWDPVEBg5U= -github.com/go-faker/faker/v4 v4.9.0/go.mod h1:u1dIRP5neLB6kTzgyVjdBOV5R1uP7BdxkcWk7tiKQXk= +github.com/gaissmai/bart v0.29.0 h1:wO6HGE8g9YE0Wm0bCpYxwRzfQ4+fbJKOhL64e5ACGCI= +github.com/gaissmai/bart v0.29.0/go.mod h1:GREWQfTLRWz/c5FTOsIw+KkscuFkIV5t8Rp7Nd1Td5c= +github.com/go-faker/faker/v4 v4.11.0 h1:HeIFTzafXsgrlxKE2QySGGTQocfGdQ8sGqU2CXvs120= +github.com/go-faker/faker/v4 v4.11.0/go.mod h1:VFIEwWDd16EdYDLF6NJ5gAAzEp7vz5LgKgJ2iZ17Tdg= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-rod/rod v0.116.2 h1:A5t2Ky2A+5eD/ZJQr1EfsQSe5rms5Xof/qj296e+ZqA= github.com/go-rod/rod v0.116.2/go.mod h1:H+CMO9SCNc2TJ2WfrG+pKhITz57uGNYU43qYHh438Mg= +github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU= +github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxoWCWfw= github.com/go-sql-driver/mysql v1.10.0/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk= github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= @@ -133,8 +137,9 @@ github.com/google/go-github/v30 v30.1.0 h1:VLDx+UolQICEOKu2m4uAoMti1SxuEBAl7RSEG github.com/google/go-github/v30 v30.1.0/go.mod h1:n8jBpHl45a/rlBUtRJMOG4GhNADUQFEufcolZ95JfU8= github.com/google/go-github/v50 v50.2.0/go.mod h1:VBY8FB6yPIjrtKhozXv4FQupxKLS6H4m6xFZlT43q8Q= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= -github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/google/go-querystring v1.2.0 h1:yhqkPbu2/OH+V9BfpCVPZkNmUXhb2gBxJArfhIxNtP0= +github.com/google/go-querystring v1.2.0/go.mod h1:8IFJqpSRITyJ8QhQ13bmbeMBDfmeEJZD5A0egEOmkqU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 h1:y3N7Bm7Y9/CtpiVkw/ZWj6lSlDF3F74SfKwfTCer72Q= github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= @@ -148,10 +153,10 @@ github.com/gosimple/slug v1.15.0 h1:wRZHsRrRcs6b0XnxMUBM6WK1U1Vg5B0R7VkIf1Xzobo= github.com/gosimple/slug v1.15.0/go.mod h1:UiRaFH+GEilHstLUmcBgWcI42viBN7mAb818JrYOeFQ= github.com/gosimple/unidecode v1.0.1 h1:hZzFTMMqSswvf0LBJZCZgThIZrpDHFXux9KeGmn6T/o= github.com/gosimple/unidecode v1.0.1/go.mod h1:CP0Cr1Y1kogOtx0bJblKzsVWrqYaqfNOnHzpgWw4Awc= -github.com/happyhackingspace/dit v0.0.28 h1:iuX6NjGGEiPNXXltmCrj+LObSKxAy4pwKIEImfMm6pc= -github.com/happyhackingspace/dit v0.0.28/go.mod h1:TFhaJk9bARUpwcycEUwtLK8ZRC0XKInXivg3j/QmNcY= -github.com/hashicorp/go-version v1.8.0 h1:KAkNb1HAiZd1ukkxDFGmokVZe1Xy9HG6NUp+bPle2i4= -github.com/hashicorp/go-version v1.8.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/happyhackingspace/dit v0.0.33 h1:54W6qwYz4UtlFuhAMWY5KoctYdBBjDjU/Fx9teLyWxo= +github.com/happyhackingspace/dit v0.0.33/go.mod h1:MjkhMq79Zn7LmBRa+2LErpmYZzP0BdHrGaAZa8ftO+o= +github.com/hashicorp/go-version v1.9.0 h1:CeOIz6k+LoN3qX9Z0tyQrPtiB1DFYRPfCIBtaXPSCnA= +github.com/hashicorp/go-version v1.9.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf h1:umfGUaWdFP2s6457fz1+xXYIWDxdGc7HdkLS9aJ1skk= @@ -206,8 +211,8 @@ github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 h1:bjfMeqxWEJ6IRUvG github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6/go.mod h1:WVJJvUw/pIOcwu2O8ZzHEhmigq2jzwRNfJVRMJB7bR8= github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= -github.com/miekg/dns v1.1.68 h1:jsSRkNozw7G/mnmXULynzMNIsgY2dHC8LO6U6Ij2JEA= -github.com/miekg/dns v1.1.68/go.mod h1:fujopn7TB3Pu3JM69XaawiU0wqjpL9/8xGop5UrTPps= +github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI= +github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs= github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 h1:yRZGarbxsRytL6EGgbqK2mCY+Lk5MWKQYKJT2gEglhc= github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -239,34 +244,32 @@ github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzb github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/projectdiscovery/asnmap v1.1.1 h1:ImJiKIaACOT7HPx4Pabb5dksolzaFYsD1kID2iwsDqI= github.com/projectdiscovery/asnmap v1.1.1/go.mod h1:QT7jt9nQanj+Ucjr9BqGr1Q2veCCKSAVyUzLXfEcQ60= github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30f7193 h1:UCZRqs1BP1wsvhCwQxfIQc7NJcXGBhQvAnEw3awhsng= github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30f7193/go.mod h1:nSovPcipgSx/EzAefF+iCfORolkKAuodiRWL3RCGHOM= -github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= -github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= -github.com/projectdiscovery/cdncheck v1.2.42 h1:Y1Q9MPq7uuv25+aGlgjA5nToOcsk+9gNEKjicyhIwQI= -github.com/projectdiscovery/cdncheck v1.2.42/go.mod h1:9oE9KKxCSHNvUf0UaMeqqUwWpC38FkNaTll0ScIBT3w= -github.com/projectdiscovery/clistats v0.1.4 h1:kDnXoNxIdOvQElOF7k2Mt6XosGa5GbMKPtRXdPHMVzU= -github.com/projectdiscovery/clistats v0.1.4/go.mod h1:hjJYNcUubk9T3cuFvA+JkLhZGjzYW50fkC48dqUAtbU= -github.com/projectdiscovery/dsl v0.8.20 h1:CxWcKuoHFpOSS1kzqnbJuK5No/6qoRG8IzNDMnZ6c/M= -github.com/projectdiscovery/dsl v0.8.20/go.mod h1:e1oHi7mxAxF+UhBhD5gOk90Ga6LQqvFea2voMO1E5D0= -github.com/projectdiscovery/fastdialer v0.5.11 h1:eI7jfwz0i73Ot1cowIBezQLxbg0i6INdAsFGJjfwPa0= -github.com/projectdiscovery/fastdialer v0.5.11/go.mod h1:W1ZkULr9mMR6i0oRFTztANnpVyEEzPUovK8sUM4eAw8= +github.com/projectdiscovery/blackrock v0.0.2 h1:mxXdu0uM8P2L2Qi210COlU8QiICPFW/Rxk5QUhlPO2k= +github.com/projectdiscovery/blackrock v0.0.2/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= +github.com/projectdiscovery/cdncheck v1.2.50 h1:KZBYT4jKNvlmlDKJqGMJ7r6XtwAim82oEZLaI+NAWWk= +github.com/projectdiscovery/cdncheck v1.2.50/go.mod h1:9oE9KKxCSHNvUf0UaMeqqUwWpC38FkNaTll0ScIBT3w= +github.com/projectdiscovery/clistats v0.1.5 h1:kHhRWZGTrXUBzKAnZ7sY1G15JIA+DuzxopN1k5SxaOg= +github.com/projectdiscovery/clistats v0.1.5/go.mod h1:hjJYNcUubk9T3cuFvA+JkLhZGjzYW50fkC48dqUAtbU= +github.com/projectdiscovery/dsl v0.8.21 h1:LXz0T5AjWyIyTbmsjdjutnucaRPMein1gVdoxwtJxZY= +github.com/projectdiscovery/dsl v0.8.21/go.mod h1:AX4IYLANTX7vFyBm7szaFwtdLr3awAlGpSp3cYIYBPM= +github.com/projectdiscovery/fastdialer v0.5.17 h1:XnTT9ERQ66S7sqRkXFLSDsyxGz6Eh3Y9VPpDJPRBsTo= +github.com/projectdiscovery/fastdialer v0.5.17/go.mod h1:2pO3BcUJdvCUBOzxVLQN7vpTP3ycQ4S+gPeFyYBJAkk= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= github.com/projectdiscovery/freeport v0.0.7 h1:Q6uXo/j8SaV/GlAHkEYQi8WQoPXyJWxyspx+aFmz9Qk= github.com/projectdiscovery/freeport v0.0.7/go.mod h1:cOhWKvNBe9xM6dFJ3RrrLvJ5vXx2NQ36SecuwjenV2k= github.com/projectdiscovery/goconfig v0.0.1 h1:36m3QjohZvemqh9bkJAakaHsm9iEZ2AcQSS18+0QX/s= github.com/projectdiscovery/goconfig v0.0.1/go.mod h1:CPO25zR+mzTtyBrsygqsHse0sp/4vB/PjaHi9upXlDw= -github.com/projectdiscovery/goflags v0.1.74 h1:n85uTRj5qMosm0PFBfsvOL24I7TdWRcWq/1GynhXS7c= -github.com/projectdiscovery/goflags v0.1.74/go.mod h1:UMc9/7dFz2oln+10tv6cy+7WZKTHf9UGhaNkF95emh4= -github.com/projectdiscovery/gologger v1.1.71 h1:IYU4mw9viKdSzMTIGVpYuw1Gtg7QIHIStqAQgeNXcBQ= -github.com/projectdiscovery/gologger v1.1.71/go.mod h1:mJwODZcFDg70ihINpOvZevmBtgvpP8H9/l8Y+OPhZPY= +github.com/projectdiscovery/goflags v0.1.76 h1:4OnNU4hvzVdOc+7wcIYBnPOHzo0BcTJ5lC6hS/TeUkA= +github.com/projectdiscovery/goflags v0.1.76/go.mod h1:7nAP1r2Dqgn/rwmOE3EWbZWUCEJKNIhVSBGpuzJAIns= +github.com/projectdiscovery/gologger v1.1.72 h1:PKk+aSx3jYOCBPi+FD+nTnhe+Vxx9u3K7Ega0GVlMCI= +github.com/projectdiscovery/gologger v1.1.72/go.mod h1:mJwODZcFDg70ihINpOvZevmBtgvpP8H9/l8Y+OPhZPY= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= github.com/projectdiscovery/govaluate v0.0.0-20260504230327-80320480bb6e h1:o+ulEIaC2+9V2Ezr6mI5xEhKWsf0V/+FUQIS723Aj6U= @@ -277,26 +280,26 @@ github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582 h1:eR+0 github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= github.com/projectdiscovery/mapcidr v1.1.97 h1:7FkxNNVXp+m1rIu5Nv/2SrF9k4+LwP8QuWs2puwy+2w= github.com/projectdiscovery/mapcidr v1.1.97/go.mod h1:9dgTJh1SP02gYZdpzMjm6vtYFkEHQHoTyaVNvaeJ7lA= -github.com/projectdiscovery/networkpolicy v0.1.41 h1:6daf4A8Vj1+iQ7nH2FR4+sOYd7q1WH1qfN61EyQU74c= -github.com/projectdiscovery/networkpolicy v0.1.41/go.mod h1:9ULLaMbdv9UnT0C5rmuK4nIwYs0o776xMnkPUb8TtaE= +github.com/projectdiscovery/networkpolicy v0.1.46 h1:lZblKcIQh8Kd0/JA5AxspxgjNz4ffs2NEf/KxMtXV1g= +github.com/projectdiscovery/networkpolicy v0.1.46/go.mod h1:q1KeQiHchXdElScEMWc5mShWNRNoJXI5koRnVaX6Qh8= github.com/projectdiscovery/ratelimit v0.0.88 h1:AcurW9aLRzlEyPe9kSjnOpr3XzLMWTpiWAlW/w73ALU= github.com/projectdiscovery/ratelimit v0.0.88/go.mod h1:CU1s+68UUG2mctSl2wi32/DHLJA6TMg+4rxgP59LfVk= -github.com/projectdiscovery/rawhttp v0.1.90 h1:LOSZ6PUH08tnKmWsIwvwv1Z/4zkiYKYOSZ6n+8RFKtw= -github.com/projectdiscovery/rawhttp v0.1.90/go.mod h1:VZYAM25UI/wVB3URZ95ZaftgOnsbphxyAw/XnQRRz4Y= +github.com/projectdiscovery/rawhttp v0.1.91 h1:6EIhZxCkBn27kJAl47FieCFCe4H2ZdbhoBVj0J6ZCVY= +github.com/projectdiscovery/rawhttp v0.1.91/go.mod h1:VZYAM25UI/wVB3URZ95ZaftgOnsbphxyAw/XnQRRz4Y= github.com/projectdiscovery/retryabledns v1.0.115 h1:RKV63FNIznFHUoawg/1hs53pVH3wqPtFhwstCuxVSoA= github.com/projectdiscovery/retryabledns v1.0.115/go.mod h1:+fEMWoPigw+M0lGNKY7AZ+g8FIgj+4sONjsinMmeL3k= -github.com/projectdiscovery/retryablehttp-go v1.3.16 h1:M/xICwSaiSHhd5OIarU3+5JoU7VmbiSAAwYUCK7CTjw= -github.com/projectdiscovery/retryablehttp-go v1.3.16/go.mod h1:s0azLAqAbcVCjHI9t0ezPhamevYGM1eoOvFkn4QmpZ8= +github.com/projectdiscovery/retryablehttp-go v1.3.23 h1:s5gsGP7sWJvss1Xeb06Y5TRux8pXvA/xXPCWdPBHOQ8= +github.com/projectdiscovery/retryablehttp-go v1.3.23/go.mod h1:dnzMyBzMmaTHYp5caYCd8mge23qhTGcBjIJ9SagdQ4Q= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= -github.com/projectdiscovery/tlsx v1.2.2 h1:Y96QBqeD2anpzEtBl4kqNbwzXh2TrzJuXfgiBLvK+SE= -github.com/projectdiscovery/tlsx v1.2.2/go.mod h1:ZJl9F1sSl0sdwE+lR0yuNHVX4Zx6tCSTqnNxnHCFZB4= +github.com/projectdiscovery/tlsx v1.3.2 h1:1Lh2ith79o8R+rOOSBZiMq8EiZ8j4D3T/8ENO+gAqXA= +github.com/projectdiscovery/tlsx v1.3.2/go.mod h1:2wgTGC/sourHvoR+RX8cAok0Sa8YK6QbEP3xo5SDzPI= github.com/projectdiscovery/useragent v0.0.108 h1:fb+uLuFJvC+MHZjCtxQJxtvp1X6A8n98CUGPyFcg3NE= github.com/projectdiscovery/useragent v0.0.108/go.mod h1:XdNRrlvtDmYfVL1Oybat4uMe+W6cLwsK9S18ond17CI= github.com/projectdiscovery/utils v0.11.1 h1:PWj1KjIASxt8icxommH72C0TQqNOvGkcSODRkiq0SQw= github.com/projectdiscovery/utils v0.11.1/go.mod h1:yktGrHGk2CTjNiccXovnvGrLHX9sV2bqz9nSnbA3V8M= -github.com/projectdiscovery/wappalyzergo v0.2.87 h1:KuUpeRSQ80L6tx9YaAPhfYWTF47bpEURYbAymr76zto= -github.com/projectdiscovery/wappalyzergo v0.2.87/go.mod h1:gMH0o5lBp65sKMwHx/tuUdOtW2RjodC6Ti+9QDsYMkY= +github.com/projectdiscovery/wappalyzergo v0.2.95 h1:XHjbQVrrxusezFq7ciWmtznPKWhpr/srWp7Yk2noxqg= +github.com/projectdiscovery/wappalyzergo v0.2.95/go.mod h1:E2p8L90ysTTUkU5FUOgGoCQ7ucEUgqJ/tvYgiGGAlEM= github.com/refraction-networking/utls v1.8.2 h1:j4Q1gJj0xngdeH+Ox/qND11aEfhpgoEvV+S9iJ2IdQo= github.com/refraction-networking/utls v1.8.2/go.mod h1:jkSOEkLqn+S/jtpEHPOsVv/4V4EVnelwbMQl4vCWXAM= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -331,8 +334,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI= @@ -421,6 +424,8 @@ go.mongodb.org/mongo-driver v1.17.9 h1:IexDdCuuNJ3BHrELgBlyaH9p60JXAvdzWR128q+U5 go.mongodb.org/mongo-driver v1.17.9/go.mod h1:LlOhpH5NUEfhxcAwG0UEkMqwYcc4JU18gtCdGudk/tQ= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -435,17 +440,17 @@ golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDf golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= -golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= -golang.org/x/exp v0.0.0-20250911091902-df9299821621 h1:2id6c1/gto0kaHYyrixvknJ8tUK/Qs5IsmBtrc+FtgU= -golang.org/x/exp v0.0.0-20250911091902-df9299821621/go.mod h1:TwQYMMnGpvZyc+JpB/UAuTNIsVJifOlSkrZkhcvpVUk= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= +golang.org/x/exp v0.0.0-20260112195511-716be5621a96 h1:Z/6YuSHTLOHfNFdb8zVZomZr7cqNgTJvA8+Qz75D8gU= +golang.org/x/exp v0.0.0-20260112195511-716be5621a96/go.mod h1:nzimsREAkjBCIEFtHiYkrJyT+2uy9YZJB7H1k68CXZU= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= -golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= +golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= +golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -464,12 +469,12 @@ golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= -golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= -golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= -golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= -golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -478,8 +483,8 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= -golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -507,8 +512,8 @@ golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= -golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -521,8 +526,8 @@ golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= -golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= -golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -538,18 +543,18 @@ golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= -golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= -golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= -golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= -golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= +golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= +golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= diff --git a/internal/pdcp/writer.go b/internal/pdcp/writer.go index fbad91abd..d397b06d6 100644 --- a/internal/pdcp/writer.go +++ b/internal/pdcp/writer.go @@ -131,6 +131,7 @@ func (u *UploadWriter) autoCommit(ctx context.Context) { // temporary buffer to store the results buff := &bytes.Buffer{} ticker := time.NewTicker(flushTimer) + defer ticker.Stop() for { select { @@ -169,15 +170,13 @@ func (u *UploadWriter) autoCommit(ctx context.Context) { } u.counter.Add(1) line := conversion.String(lineBytes) - if buff.Len()+len(line) > MaxChunkSize { - // flush existing buffer - if err := u.uploadChunk(buff); err != nil { + appendResultLine(buff, line, MaxChunkSize, func(b *bytes.Buffer) error { + if err := u.uploadChunk(b); err != nil { gologger.Error().Msgf("Failed to upload asset results on cloud: %v", err) + return err } - } else { - buff.WriteString(line) - buff.WriteString("\n") - } + return nil + }) } } } @@ -259,12 +258,22 @@ func (u *UploadWriter) getRequest(bin []byte) (*retryablehttp.Request, error) { return req, nil } +// appendResultLine writes line and its trailing newline to buff, flushing +// existing data first when the next write would exceed max. An empty buffer is +// never flushed, so a single oversized line is retained instead of dropped or +// uploaded as an empty chunk. +func appendResultLine(buff *bytes.Buffer, line string, max int, flush func(*bytes.Buffer) error) { + if buff.Len() > 0 && buff.Len()+len(line)+len("\n") > max { + _ = flush(buff) + } + buff.WriteString(line) + buff.WriteString("\n") +} + // Close closes the upload writer func (u *UploadWriter) Close() { - if !u.closed.Load() { - // protect to avoid channel closed twice error + if u.closed.CompareAndSwap(false, true) { close(u.data) - u.closed.Store(true) } <-u.done } diff --git a/internal/pdcp/writer_test.go b/internal/pdcp/writer_test.go new file mode 100644 index 000000000..a877eac6d --- /dev/null +++ b/internal/pdcp/writer_test.go @@ -0,0 +1,131 @@ +package pdcp + +import ( + "bytes" + "errors" + "sync" + "testing" + "time" + + "github.com/projectdiscovery/httpx/runner" + "github.com/stretchr/testify/require" +) + +func TestAppendResultLine(t *testing.T) { + t.Run("keeps lines under the limit without flushing", func(t *testing.T) { + buff := &bytes.Buffer{} + flushed := 0 + appendResultLine(buff, "ab", 10, func(*bytes.Buffer) error { + flushed++ + return nil + }) + appendResultLine(buff, "cd", 10, func(*bytes.Buffer) error { + flushed++ + return nil + }) + require.Equal(t, 0, flushed) + require.Equal(t, "ab\ncd\n", buff.String()) + }) + + t.Run("flushes existing data and keeps the overflowing line", func(t *testing.T) { + buff := &bytes.Buffer{} + flush := func(b *bytes.Buffer) error { + require.Equal(t, "aaaa\n", b.String()) + b.Reset() + return nil + } + appendResultLine(buff, "aaaa", 6, flush) + appendResultLine(buff, "bbbb", 6, flush) + require.Equal(t, "bbbb\n", buff.String()) + }) + + t.Run("does not flush an empty buffer for an oversized line", func(t *testing.T) { + buff := &bytes.Buffer{} + flushed := 0 + appendResultLine(buff, "toolong", 4, func(*bytes.Buffer) error { + flushed++ + return nil + }) + require.Equal(t, 0, flushed) + require.Equal(t, "toolong\n", buff.String()) + }) + + t.Run("newline counts towards the limit", func(t *testing.T) { + buff := &bytes.Buffer{} + const max = 6 + flush := func(b *bytes.Buffer) error { + b.Reset() + return nil + } + // "abc\n" is 4 bytes, appending "de\n" would reach 7 without counting + // the newline in the check. + appendResultLine(buff, "abc", max, flush) + appendResultLine(buff, "de", max, flush) + require.LessOrEqual(t, buff.Len(), max) + require.Equal(t, "de\n", buff.String()) + }) + + t.Run("still appends the current line when flush fails", func(t *testing.T) { + buff := bytes.NewBufferString("old\n") + appendResultLine(buff, "new", 4, func(*bytes.Buffer) error { + return errors.New("upload failed") + }) + require.Equal(t, "old\nnew\n", buff.String()) + }) +} + +func TestUploadWriterCloseWaits(t *testing.T) { + u := &UploadWriter{ + done: make(chan struct{}, 1), + data: make(chan runner.Result, 8), + } + + started := make(chan struct{}) + release := make(chan struct{}) + go func() { + for range u.data { + } + close(started) + <-release + u.done <- struct{}{} + close(u.done) + }() + + firstDone := make(chan struct{}) + go func() { + u.Close() + close(firstDone) + }() + + select { + case <-started: + case <-time.After(2 * time.Second): + t.Fatal("Close did not close the data channel") + } + + secondDone := make(chan struct{}) + go func() { + u.Close() + close(secondDone) + }() + + select { + case <-secondDone: + t.Fatal("second Close returned before autoCommit finished") + case <-time.After(50 * time.Millisecond): + } + + close(release) + + var wg sync.WaitGroup + wg.Add(2) + go func() { defer wg.Done(); <-firstDone }() + go func() { defer wg.Done(); <-secondDone }() + done := make(chan struct{}) + go func() { wg.Wait(); close(done) }() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("Close hung") + } +} diff --git a/runner/banner.go b/runner/banner.go index d127cdba2..3f021e8c7 100644 --- a/runner/banner.go +++ b/runner/banner.go @@ -16,7 +16,7 @@ const banner = ` ` // Version is the current Version of httpx -const Version = `v1.10.0` +const Version = `v1.11.0` // showBanner is used to show the banner to the user func showBanner() { diff --git a/runner/cpe.go b/runner/cpe.go index 6861dea86..5d7c91f08 100644 --- a/runner/cpe.go +++ b/runner/cpe.go @@ -3,6 +3,7 @@ package runner import ( "encoding/json" "fmt" + "slices" "strings" awesomesearchqueries "github.com/projectdiscovery/awesome-search-queries" @@ -160,6 +161,92 @@ func normalizeProductName(name string) string { return b.String() } +// cpeProductSuffixes are common awesome-search-queries product suffixes stripped +// to derive shorter lookup aliases (e.g. liferay_portal -> liferay). +var cpeProductSuffixes = []string{ + "_server", + "_portal", + "_software", + "_platform", + "_suite", + "_service", + "_manager", + "_panel", + "_cms", + "_firmware", + "_gateway", + "_proxy", + "_system", + "_application", + "_framework", + "_tower", + "_policy_manager", +} + +// productLookupKeys returns normalized lookup keys for joining a CPE product +// name to wappalyzer technology names. Keys are ordered most-specific first; +// the first key with a version match wins. +func productLookupKeys(product string) []string { + product = strings.TrimSpace(product) + if product == "" { + return nil + } + + // Compound awesome-search-queries names list multiple products; the first is + // the primary identifier for version lookup purposes. + if idx := strings.Index(product, ","); idx >= 0 { + product = strings.TrimSpace(product[:idx]) + } + + seen := make(map[string]struct{}) + keys := make([]string, 0, 4) + addKey := func(raw string) { + key := normalizeProductName(raw) + if key == "" { + return + } + if _, ok := seen[key]; ok { + return + } + seen[key] = struct{}{} + keys = append(keys, key) + } + + addKey(product) + + lower := strings.ToLower(product) + suffixes := slices.Clone(cpeProductSuffixes) + slices.SortFunc(suffixes, func(a, b string) int { + return len(b) - len(a) + }) + for _, suffix := range suffixes { + if strings.HasSuffix(lower, suffix) { + addKey(strings.TrimSuffix(lower, suffix)) + } + } + + if strings.Contains(lower, "_") { + parts := strings.Split(lower, "_") + addKey(parts[0]) + for i := 2; i <= len(parts); i++ { + addKey(strings.Join(parts[:i], "_")) + } + } + + return keys +} + +// lookupTechVersion finds a wappalyzer version for a CPE product using exact +// and alias keys derived from awesome-search-queries naming conventions. +func lookupTechVersion(product string, versions map[string]string) (string, bool) { + for _, key := range productLookupKeys(product) { + if version, ok := versions[key]; ok { + return version, true + } + } + return "", false +} + // buildTechVersionMap maps normalized technology name -> version, parsing // wappalyzer's "Name:version" entries (FormatAppVersion convention). Entries // without a version are skipped. A product reported with conflicting versions @@ -203,7 +290,7 @@ func EnrichCPEVersions(matches []CPEInfo, technologies []string) []CPEInfo { enriched := make([]CPEInfo, len(matches)) for i, match := range matches { enriched[i] = match - if version, ok := versions[normalizeProductName(match.Product)]; ok { + if version, ok := lookupTechVersion(match.Product, versions); ok { enriched[i].CPE = setCPEVersion(match.CPE, version) } } diff --git a/runner/cpe_test.go b/runner/cpe_test.go index fe163bf0c..168fc2019 100644 --- a/runner/cpe_test.go +++ b/runner/cpe_test.go @@ -159,6 +159,187 @@ func TestBuildTechVersionMap(t *testing.T) { } } +func TestProductLookupKeys(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + in string + want []string + }{ + { + name: "liferay portal strips suffix", + in: "liferay_portal", + want: []string{"liferayportal", "liferay"}, + }, + { + name: "confluence server strips suffix", + in: "confluence_server", + want: []string{"confluenceserver", "confluence"}, + }, + { + name: "tableau server strips suffix", + in: "tableau_server", + want: []string{"tableauserver", "tableau"}, + }, + { + name: "longest suffix takes priority", + in: "ansible_policy_manager", + want: []string{"ansiblepolicymanager", "ansible", "ansiblepolicy"}, + }, + { + name: "compound name uses primary product", + in: "digital_experience_platform,liferay_portal", + want: []string{"digitalexperienceplatform", "digitalexperience", "digital"}, + }, + { + name: "display name unchanged", + in: "Apache HTTP Server", + want: []string{"apachehttpserver"}, + }, + { + name: "simple product", + in: "next.js", + want: []string{"nextjs"}, + }, + { + name: "empty", + in: "", + want: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := productLookupKeys(tt.in) + if len(got) != len(tt.want) { + t.Fatalf("productLookupKeys(%q) = %v, want %v", tt.in, got, tt.want) + } + for i := range tt.want { + if got[i] != tt.want[i] { + t.Fatalf("productLookupKeys(%q)[%d] = %q, want %q (full: %v)", tt.in, i, got[i], tt.want[i], got) + } + } + }) + } +} + +func TestLookupTechVersion(t *testing.T) { + t.Parallel() + + versions := buildTechVersionMap([]string{ + "Liferay:7.3.5", + "Confluence:8.5.1", + "Tableau:2023.1", + "Apache HTTP Server:2.4.7", + "Ansible:2.14.0", + }) + + tests := []struct { + product string + want string + wantFound bool + }{ + {"liferay_portal", "7.3.5", true}, + {"liferay", "7.3.5", true}, + {"confluence_server", "8.5.1", true}, + {"tableau_server", "2023.1", true}, + {"ansible_policy_manager", "2.14.0", true}, + {"Apache HTTP Server", "2.4.7", true}, + {"phpcollab", "", false}, + {"unknown_product", "", false}, + } + for _, tt := range tests { + t.Run(tt.product, func(t *testing.T) { + got, ok := lookupTechVersion(tt.product, versions) + if ok != tt.wantFound { + t.Fatalf("lookupTechVersion(%q) found = %v, want %v", tt.product, ok, tt.wantFound) + } + if got != tt.want { + t.Fatalf("lookupTechVersion(%q) = %q, want %q", tt.product, got, tt.want) + } + }) + } +} + +func TestEnrichCPEVersionsIssue2536(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + product string + vendor string + cpe string + technologies []string + wantCPE string + }{ + { + name: "liferay portal product name from awesome-search-queries", + product: "liferay_portal", + vendor: "liferay", + cpe: "cpe:2.3:a:liferay:liferay_portal:*:*:*:*:*:*:*:*", + technologies: []string{"Liferay:7.3.5"}, + wantCPE: "cpe:2.3:a:liferay:liferay_portal:7.3.5:*:*:*:*:*:*:*", + }, + { + name: "confluence server product name", + product: "confluence_server", + vendor: "atlassian", + cpe: "cpe:2.3:a:atlassian:confluence_server:*:*:*:*:*:*:*:*", + technologies: []string{"Confluence:8.5.1"}, + wantCPE: "cpe:2.3:a:atlassian:confluence_server:8.5.1:*:*:*:*:*:*:*", + }, + { + name: "tableau server product name", + product: "tableau_server", + vendor: "tableau", + cpe: "cpe:2.3:a:tableau:tableau_server:*:*:*:*:*:*:*:*", + technologies: []string{"Tableau:2023.1"}, + wantCPE: "cpe:2.3:a:tableau:tableau_server:2023.1:*:*:*:*:*:*:*", + }, + { + name: "no false match on unrelated substring", + product: "phpcollab", + vendor: "phpcollab", + cpe: "cpe:2.3:a:phpcollab:phpcollab:*:*:*:*:*:*:*:*", + technologies: []string{"PHP:8.1.0"}, + wantCPE: "cpe:2.3:a:phpcollab:phpcollab:*:*:*:*:*:*:*:*", + }, + { + name: "ansible tower strips suffix", + product: "ansible_tower", + vendor: "redhat", + cpe: "cpe:2.3:a:redhat:ansible_tower:*:*:*:*:*:*:*:*", + technologies: []string{"Ansible:2.14.0"}, + wantCPE: "cpe:2.3:a:redhat:ansible_tower:2.14.0:*:*:*:*:*:*:*", + }, + { + name: "ansible policy manager prefers ansible alias", + product: "ansible_policy_manager", + vendor: "redhat", + cpe: "cpe:2.3:a:redhat:ansible_policy_manager:*:*:*:*:*:*:*:*", + technologies: []string{"Ansible:2.14.0"}, + wantCPE: "cpe:2.3:a:redhat:ansible_policy_manager:2.14.0:*:*:*:*:*:*:*", + }, + { + name: "conflicting tech versions leave cpe unchanged", + product: "liferay_portal", + vendor: "liferay", + cpe: "cpe:2.3:a:liferay:liferay_portal:*:*:*:*:*:*:*:*", + technologies: []string{"Liferay:7.3.5", "Liferay:7.4.0"}, + wantCPE: "cpe:2.3:a:liferay:liferay_portal:*:*:*:*:*:*:*:*", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + matches := []CPEInfo{{Product: tt.product, Vendor: tt.vendor, CPE: tt.cpe}} + got := EnrichCPEVersions(matches, tt.technologies) + if got[0].CPE != tt.wantCPE { + t.Fatalf("CPE = %q, want %q", got[0].CPE, tt.wantCPE) + } + }) + } +} + func TestBuildTechVersionMapConflict(t *testing.T) { // the same product reported with two versions must be dropped, not resolved // by random map iteration order. @@ -233,8 +414,9 @@ func TestEnrichCPEVersionsWithRealWappalyzer(t *testing.T) { t.Fatalf("expected wappalyzer to emit \"Liferay:7.3.5\", got %v", technologies) } + // awesome-search-queries uses snake_case product names; issue #2536. matches := []CPEInfo{ - {Product: "Liferay", Vendor: "liferay", CPE: "cpe:2.3:a:liferay:liferay_portal:*:*:*:*:*:*:*:*"}, + {Product: "liferay_portal", Vendor: "liferay", CPE: "cpe:2.3:a:liferay:liferay_portal:*:*:*:*:*:*:*:*"}, } got := EnrichCPEVersions(matches, technologies) if got[0].CPE != "cpe:2.3:a:liferay:liferay_portal:7.3.5:*:*:*:*:*:*:*" { diff --git a/runner/options.go b/runner/options.go index 5585659b7..f636b9d16 100644 --- a/runner/options.go +++ b/runner/options.go @@ -204,6 +204,9 @@ type Options struct { // Deprecated: use OutputFilterPageType with "error" instead. OutputFilterErrorPage bool OutputFilterPageType goflags.StringSlice + // KnowledgeBase enables knowledge base classification using dit. It is + // implied by OutputFilterPageType/OutputFilterErrorPage, which need it. + KnowledgeBase bool FilterOutDuplicates bool OutputFilterContentLength string InputRawRequest string @@ -331,7 +334,7 @@ type Options struct { NoDecode bool Screenshot bool UseInstalledChrome bool - TlsImpersonate bool + TlsImpersonate string DisableStdin bool HttpApiEndpoint string NoScreenshotBytes bool @@ -412,6 +415,7 @@ func ParseOptions() *Options { flagSet.StringVarP(&options.CustomFingerprintFile, "custom-fingerprint-file", "cff", "", "path to a custom fingerprint file for technology detection"), flagSet.BoolVar(&options.CPEDetect, "cpe", false, "display CPE (Common Platform Enumeration) with product version based on awesome-search-queries"), flagSet.BoolVarP(&options.WordPress, "wordpress", "wp", false, "display WordPress plugins and themes"), + flagSet.BoolVarP(&options.KnowledgeBase, "knowledge-base", "kb", false, "enable knowledge base classification"), flagSet.BoolVar(&options.OutputMethod, "method", false, "display http request method"), flagSet.BoolVarP(&options.OutputWebSocket, "websocket", "ws", false, "display server using websocket"), flagSet.BoolVar(&options.OutputIP, "ip", false, "display host ip"), @@ -547,7 +551,7 @@ func ParseOptions() *Options { flagSet.BoolVarP(&options.LeaveDefaultPorts, "leave-default-ports", "ldp", false, "leave default http/https ports in host header (eg. http://host:80 - https://host:443"), flagSet.BoolVar(&options.ZTLS, "ztls", false, "use ztls library with autofallback to standard one for tls13"), flagSet.BoolVar(&options.NoDecode, "no-decode", false, "avoid decoding body"), - flagSet.BoolVarP(&options.TlsImpersonate, "tls-impersonate", "tlsi", false, "enable experimental client hello (ja3) tls randomization"), + flagSet.StringVarP(&options.TlsImpersonate, "tls-impersonate", "tlsi", "", "enable experimental client hello (ja3) tls impersonation (chrome, or ja3 full string)"), flagSet.BoolVar(&options.DisableStdin, "no-stdin", false, "Disable Stdin processing"), flagSet.StringVarP(&options.HttpApiEndpoint, "http-api-endpoint", "hae", "", "experimental http api endpoint"), flagSet.StringVarP(&options.SecretFile, "secret-file", "sf", "", "path to the secret file for authentication"), @@ -720,6 +724,19 @@ func (options *Options) HasMatcherOrFilter() bool { options.OutputFilterResponseTime != "" } +// hasPageTypeFilter reports whether a filter that depends on the page type +// classification is in use. +func (options *Options) hasPageTypeFilter() bool { + return len(options.OutputFilterPageType) > 0 || options.OutputFilterErrorPage +} + +// classificationEnabled reports whether the knowledge base classifier should be +// loaded. It is opt-in via -kb, and implied by the page type filters that +// cannot work without it. +func (options *Options) classificationEnabled() bool { + return options.KnowledgeBase || options.hasPageTypeFilter() +} + func (options *Options) ValidateOptions() error { if options.InputFile != "" && !fileutilz.FileNameIsGlob(options.InputFile) && !fileutil.FileExists(options.InputFile) { return fmt.Errorf("file '%s' does not exist", options.InputFile) @@ -917,6 +934,10 @@ func (options *Options) configureOutput() { gologger.Info().Msg("-fep is deprecated, use -fpt error instead") options.OutputFilterPageType = goflags.StringSlice{"error"} } + // page type filters cannot work without the classifier + if options.hasPageTypeFilter() { + options.KnowledgeBase = true + } } func (options *Options) configureResume() error { diff --git a/runner/runner.go b/runner/runner.go index eed5d886b..63dccb05d 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -431,10 +431,10 @@ func New(options *Options) (*Runner, error) { } runner.simHashes = gcache.New[uint64, []string](1000).ARC().Build() - if options.JSONOutput || options.CSVOutput || len(options.OutputFilterPageType) > 0 { + if options.classificationEnabled() { ditClassifier, err := dit.New() if err != nil { - gologger.Warning().Msgf("Could not initialize page classifier: %s", err) + return nil, errors.Wrap(err, "could not initialize page classifier") } runner.ditClassifier = ditClassifier } diff --git a/runner/wellknown_recipes.go b/runner/wellknown_recipes.go new file mode 100644 index 000000000..905d80f77 --- /dev/null +++ b/runner/wellknown_recipes.go @@ -0,0 +1,152 @@ +package runner + +// WellKnownRecipe documents a composable httpx one-liner for probing well-known resources. +type WellKnownRecipe struct { + Name string + Paths string + MatchStatusCode string + MatchCondition string +} + +// WellKnownRecipes returns README-documented recipes validated by TestWellKnownRecipes. +func WellKnownRecipes() []WellKnownRecipe { + return []WellKnownRecipe{ + { + Name: "security.txt", + Paths: "/.well-known/security.txt,/security.txt", + MatchStatusCode: "200", + MatchCondition: `contains(content_type, "text/plain") && contains(body, "Contact:") && contains_any(body, "mailto:", "https://")`, + }, + { + Name: "robots.txt", + Paths: "/robots.txt", + MatchStatusCode: "200", + MatchCondition: `contains(content_type, "text/plain")`, + }, + { + Name: "sitemap.xml", + Paths: "/sitemap.xml", + MatchStatusCode: "200", + MatchCondition: `contains_any(content_type, "application/xml", "text/xml") && contains(body, "`, + }, + "/humans.txt": { + statusCode: 200, + contentType: "text/plain", + body: "/* TEAM */\nDeveloper: Example Dev\n", + }, + "/ads.txt": { + statusCode: 200, + contentType: "text/plain", + body: "google.com, pub-0000000000000000, DIRECT, f08c47fec0942fa0\n", + }, + "/.well-known/openid-configuration": { + statusCode: 200, + contentType: "application/json", + body: `{"issuer":"https://example.com","authorization_endpoint":"https://example.com/auth"}`, + }, + "/.well-known/apple-app-site-association": { + statusCode: 200, + contentType: "application/json", + body: `{"applinks":{"apps":[],"details":[]}}`, + }, + "/.well-known/apple-app-site-association.json": { + statusCode: 200, + contentType: "application/json", + body: `{"applinks":{"apps":[],"details":[]}}`, + }, + "/.well-known/assetlinks.json": { + statusCode: 200, + contentType: "application/json", + body: `[{"relation":["delegate_permission/common.handle_all_urls"],"target":{"namespace":"android_app","package_name":"com.example.app"}}]`, + }, + "/crossdomain.xml": { + statusCode: 200, + contentType: "text/xml", + body: ``, + }, + "/.well-known/change-password": { + statusCode: 200, + contentType: "text/html", + body: "Change password", + }, +} + +// soft404Fixture is an HTML error page that should not match strict well-known recipes. +var soft404Fixture = wellKnownFixture{ + statusCode: 200, + contentType: "text/html; charset=utf-8", + body: "Not Found

404

", +} diff --git a/runner/wellknown_recipes_test.go b/runner/wellknown_recipes_test.go new file mode 100644 index 000000000..327e29103 --- /dev/null +++ b/runner/wellknown_recipes_test.go @@ -0,0 +1,123 @@ +package runner + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/projectdiscovery/httpx/common/stringz" + sliceutil "github.com/projectdiscovery/utils/slice" + "github.com/stretchr/testify/require" +) + +func TestWellKnownRecipes(t *testing.T) { + for _, recipe := range WellKnownRecipes() { + t.Run(recipe.Name, func(t *testing.T) { + matched := false + for path, fixture := range wellKnownFixtures { + if !recipeMatchesPath(recipe, path) { + continue + } + result := resultFromWellKnownFixture(path, fixture) + if recipeMatches(recipe, result) { + matched = true + break + } + } + require.True(t, matched, "recipe %q should match at least one fixture", recipe.Name) + }) + } +} + +func TestWellKnownRecipeSecurityTxtRejectsSoft404(t *testing.T) { + recipe := WellKnownRecipes()[0] + result := resultFromWellKnownFixture("/.well-known/security.txt", soft404Fixture) + require.False(t, recipeMatches(recipe, result), "security.txt recipe should not match HTML soft-404 pages") +} + +func TestWellKnownRecipeSecurityTxtRejectsMissingContact(t *testing.T) { + recipe := WellKnownRecipes()[0] + result := resultFromWellKnownFixture("/.well-known/security.txt", wellKnownFixture{ + statusCode: http.StatusOK, + contentType: "text/plain", + body: "Preferred-Languages: en\n", + }) + require.False(t, recipeMatches(recipe, result), "security.txt recipe should require a Contact field") +} + +func TestWellKnownRecipeAdsTxtRejectsInvalidBody(t *testing.T) { + recipe := WellKnownRecipes()[4] + result := resultFromWellKnownFixture("/ads.txt", wellKnownFixture{ + statusCode: http.StatusOK, + contentType: "text/plain", + body: "example.com, DIRECT\n", + }) + require.False(t, recipeMatches(recipe, result), "ads.txt recipe should require an authorized digital seller entry") +} + +func TestWellKnownRecipesHTTPProbe(t *testing.T) { + ts := newWellKnownTestServer(t) + defer ts.Close() + + recipe := WellKnownRecipes()[len(WellKnownRecipes())-1] + for _, path := range strings.Split(recipe.Paths, ",") { + t.Run(path, func(t *testing.T) { + resp, err := http.Get(ts.URL + path) + require.NoError(t, err) + defer func() { _ = resp.Body.Close() }() + require.Equal(t, http.StatusOK, resp.StatusCode) + }) + } +} + +func recipeMatchesPath(recipe WellKnownRecipe, path string) bool { + for _, recipePath := range strings.Split(recipe.Paths, ",") { + if recipePath == path { + return true + } + } + return false +} + +func recipeMatches(recipe WellKnownRecipe, result Result) bool { + if recipe.MatchStatusCode != "" { + codes, err := stringz.StringToSliceInt(recipe.MatchStatusCode) + if err != nil || !sliceutil.Contains(codes, result.StatusCode) { + return false + } + } + if recipe.MatchCondition != "" && !evalDslExpr(result, recipe.MatchCondition) { + return false + } + return true +} + +func resultFromWellKnownFixture(path string, fixture wellKnownFixture) Result { + url := "http://example.com" + path + contentType := fixture.contentType + if idx := strings.Index(contentType, ";"); idx >= 0 { + contentType = strings.TrimSpace(contentType[:idx]) + } + return Result{ + StatusCode: fixture.statusCode, + ContentType: contentType, + ResponseBody: fixture.body, + URL: url, + str: url, + } +} + +func newWellKnownTestServer(t *testing.T) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fixture, ok := wellKnownFixtures[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", fixture.contentType) + w.WriteHeader(fixture.statusCode) + _, _ = w.Write([]byte(fixture.body)) + })) +}