diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 3080047d4..0b67940a6 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -5,6 +5,7 @@ on: pull_request: paths: - '**.go' + - '.github/workflows/codeql-analysis.yml' branches: - dev @@ -25,16 +26,16 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} - name: Autobuild - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v4 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 \ No newline at end of file + uses: github/codeql-action/analyze@v4 diff --git a/go.mod b/go.mod index b677c8abb..c51a57d37 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.26.0 require ( github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 - github.com/PuerkitoBio/goquery v1.12.0 + github.com/PuerkitoBio/goquery v1.13.0 github.com/akrylysov/pogreb v0.10.2 // indirect github.com/corona10/goimagehash v1.1.0 github.com/go-faker/faker/v4 v4.11.0 @@ -16,32 +16,32 @@ require ( github.com/logrusorgru/aurora v2.0.3+incompatible github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 github.com/microcosm-cc/bluemonday v1.0.27 - github.com/miekg/dns v1.1.72 // indirect + github.com/miekg/dns v1.1.73 // indirect github.com/pkg/errors v0.9.1 github.com/projectdiscovery/asnmap v1.1.1 - github.com/projectdiscovery/cdncheck v1.2.50 - github.com/projectdiscovery/clistats v0.1.5 - github.com/projectdiscovery/dsl v0.8.21 - github.com/projectdiscovery/fastdialer v0.5.17 + github.com/projectdiscovery/cdncheck v1.2.51 + github.com/projectdiscovery/clistats v0.1.6 + github.com/projectdiscovery/dsl v0.8.22 + github.com/projectdiscovery/fastdialer v0.5.19 github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/goconfig v0.0.1 - github.com/projectdiscovery/goflags v0.1.76 + github.com/projectdiscovery/goflags v0.2.1 github.com/projectdiscovery/gologger v1.1.72 - github.com/projectdiscovery/hmap v0.0.101 + github.com/projectdiscovery/hmap v0.0.102 github.com/projectdiscovery/mapcidr v1.1.97 - github.com/projectdiscovery/networkpolicy v0.1.46 + github.com/projectdiscovery/networkpolicy v0.1.48 github.com/projectdiscovery/ratelimit v0.0.88 - github.com/projectdiscovery/rawhttp v0.1.91 - github.com/projectdiscovery/retryablehttp-go v1.3.23 - github.com/projectdiscovery/tlsx v1.3.2 - github.com/projectdiscovery/useragent v0.0.108 - github.com/projectdiscovery/utils v0.11.1 - github.com/projectdiscovery/wappalyzergo v0.2.95 + github.com/projectdiscovery/rawhttp v0.1.92 + github.com/projectdiscovery/retryablehttp-go v1.3.25 + github.com/projectdiscovery/tlsx v1.4.0 + github.com/projectdiscovery/useragent v0.0.109 + github.com/projectdiscovery/utils v0.11.3 + github.com/projectdiscovery/wappalyzergo v0.2.96 github.com/rs/xid v1.6.0 github.com/spaolacci/murmur3 v1.1.0 github.com/stretchr/testify v1.12.1 github.com/zmap/zcrypto v0.0.0-20240803002437-3a861682ac77 - go.etcd.io/bbolt v1.4.0 // indirect + go.etcd.io/bbolt v1.4.3 // indirect go.uber.org/multierr v1.11.0 golang.org/x/exp v0.0.0-20260112195511-716be5621a96 golang.org/x/net v0.58.0 @@ -72,7 +72,7 @@ require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/alecthomas/chroma/v2 v2.14.0 // indirect github.com/andybalholm/brotli v1.2.0 // indirect - github.com/andybalholm/cascadia v1.3.3 // indirect + github.com/andybalholm/cascadia v1.3.4 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect @@ -81,7 +81,7 @@ require ( github.com/charmbracelet/lipgloss v0.13.0 // indirect github.com/charmbracelet/x/ansi v0.3.2 // indirect github.com/cheggaaa/pb/v3 v3.1.6 // indirect - github.com/cloudflare/cfssl v1.6.4 // indirect + github.com/cloudflare/cfssl v1.6.5 // indirect github.com/cnf/structhash v0.0.0-20250313080605-df4c6cc74a9a // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/djherbis/times v1.6.0 // indirect @@ -94,7 +94,7 @@ require ( github.com/gaissmai/bart v0.29.0 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect - github.com/google/certificate-transparency-go v1.3.2 // indirect + github.com/google/certificate-transparency-go v1.3.3 // indirect github.com/google/go-github/v30 v30.1.0 // indirect github.com/google/go-querystring v1.2.0 // indirect github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 // indirect @@ -127,7 +127,7 @@ require ( github.com/projectdiscovery/gostruct v0.0.2 // indirect github.com/projectdiscovery/govaluate v0.0.0-20260504230327-80320480bb6e // indirect github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582 // indirect - github.com/projectdiscovery/retryabledns v1.0.115 // indirect + github.com/projectdiscovery/retryabledns v1.0.116 // indirect github.com/refraction-networking/utls v1.8.2 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect @@ -163,11 +163,9 @@ require ( github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/crypto v0.55.0 // indirect - golang.org/x/mod v0.38.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/sync v0.22.0 // indirect golang.org/x/term v0.45.0 // indirect golang.org/x/time v0.15.0 // indirect - golang.org/x/tools v0.48.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) diff --git a/go.sum b/go.sum index 037d6823e..e13aab54f 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057/go.mod h1:iLB2piv github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 h1:ZbFL+BDfBqegi+/Ssh7im5+aQfBRx6it+kHnC7jaDU8= github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809/go.mod h1:upgc3Zs45jBDnBT4tVRgRcgm26ABpaP7MoTSdgysca4= github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g= -github.com/PuerkitoBio/goquery v1.12.0 h1:pAcL4g3WRXekcB9AU/y1mbKez2dbY2AajVhtkO8RIBo= -github.com/PuerkitoBio/goquery v1.12.0/go.mod h1:802ej+gV2y7bbIhOIoPY5sT183ZW0YFofScC4q/hIpQ= +github.com/PuerkitoBio/goquery v1.13.0 h1:mqHbjD7Jmnul4DTR24LKTjo1uUmHUh072kteGV+xpFM= +github.com/PuerkitoBio/goquery v1.13.0/go.mod h1:Hip5mdBL8K2wEGKJdr27sRaNwIdDajmCwB/ExUPwW+g= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/akrylysov/pogreb v0.10.2 h1:e6PxmeyEhWyi2AKOBIJzAEi4HkiC+lKyCocRGlnDi78= @@ -24,8 +24,8 @@ github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= -github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= -github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= +github.com/andybalholm/cascadia v1.3.4 h1:vM2lgh0Vru9Vwyfm4cQqWP2HHMW0u0+2PAW7Q38Qufg= +github.com/andybalholm/cascadia v1.3.4/go.mod h1:BLRmbRjpEtNKieZOCCvYj4RqN+KRA41GBe/5O+G93kM= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= @@ -57,8 +57,8 @@ github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moA github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ= github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk= github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8= -github.com/cloudflare/cfssl v1.6.4 h1:NMOvfrEjFfC63K3SGXgAnFdsgkmiq4kATme5BfcqrO8= -github.com/cloudflare/cfssl v1.6.4/go.mod h1:8b3CQMxfWPAeom3zBnGJ6sd+G1NkL5TXqmDXacb+1J0= +github.com/cloudflare/cfssl v1.6.5 h1:46zpNkm6dlNkMZH/wMW22ejih6gIaJbzL2du6vD7ZeI= +github.com/cloudflare/cfssl v1.6.5/go.mod h1:Bk1si7sq8h2+yVEDrFJiz3d7Aw+pfjjJSZVaD+Taky4= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= github.com/cnf/structhash v0.0.0-20250313080605-df4c6cc74a9a h1:Ohw57yVY2dBTt+gsC6aZdteyxwlxfbtgkFEMTEkwgSw= github.com/cnf/structhash v0.0.0-20250313080605-df4c6cc74a9a/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4= @@ -123,8 +123,8 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/certificate-transparency-go v1.3.2 h1:9ahSNZF2o7SYMaKaXhAumVEzXB2QaayzII9C8rv7v+A= -github.com/google/certificate-transparency-go v1.3.2/go.mod h1:H5FpMUaGa5Ab2+KCYsxg6sELw3Flkl7pGZzWdBoYLXs= +github.com/google/certificate-transparency-go v1.3.3 h1:hq/rSxztSkXN2tx/3jQqF6Xc0O565UQPdHrOWvZwybo= +github.com/google/certificate-transparency-go v1.3.3/go.mod h1:iR17ZgSaXRzSa5qvjFl8TnVD5h8ky2JMVio+dzoKMgA= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -211,8 +211,8 @@ github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 h1:bjfMeqxWEJ6IRUvG github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6/go.mod h1:WVJJvUw/pIOcwu2O8ZzHEhmigq2jzwRNfJVRMJB7bR8= github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= -github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI= -github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs= +github.com/miekg/dns v1.1.73 h1:uhT8nJxmTrPJYClxVxTCX+CVn6qnzSiybRk72Z6DgrE= +github.com/miekg/dns v1.1.73/go.mod h1:RW2Obtfd5NZHvOFe3zYG0W8koWOQtAzyHaLo8vASBuQ= github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 h1:yRZGarbxsRytL6EGgbqK2mCY+Lk5MWKQYKJT2gEglhc= github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -252,54 +252,54 @@ github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30 github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30f7193/go.mod h1:nSovPcipgSx/EzAefF+iCfORolkKAuodiRWL3RCGHOM= github.com/projectdiscovery/blackrock v0.0.2 h1:mxXdu0uM8P2L2Qi210COlU8QiICPFW/Rxk5QUhlPO2k= github.com/projectdiscovery/blackrock v0.0.2/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= -github.com/projectdiscovery/cdncheck v1.2.50 h1:KZBYT4jKNvlmlDKJqGMJ7r6XtwAim82oEZLaI+NAWWk= -github.com/projectdiscovery/cdncheck v1.2.50/go.mod h1:9oE9KKxCSHNvUf0UaMeqqUwWpC38FkNaTll0ScIBT3w= -github.com/projectdiscovery/clistats v0.1.5 h1:kHhRWZGTrXUBzKAnZ7sY1G15JIA+DuzxopN1k5SxaOg= -github.com/projectdiscovery/clistats v0.1.5/go.mod h1:hjJYNcUubk9T3cuFvA+JkLhZGjzYW50fkC48dqUAtbU= -github.com/projectdiscovery/dsl v0.8.21 h1:LXz0T5AjWyIyTbmsjdjutnucaRPMein1gVdoxwtJxZY= -github.com/projectdiscovery/dsl v0.8.21/go.mod h1:AX4IYLANTX7vFyBm7szaFwtdLr3awAlGpSp3cYIYBPM= -github.com/projectdiscovery/fastdialer v0.5.17 h1:XnTT9ERQ66S7sqRkXFLSDsyxGz6Eh3Y9VPpDJPRBsTo= -github.com/projectdiscovery/fastdialer v0.5.17/go.mod h1:2pO3BcUJdvCUBOzxVLQN7vpTP3ycQ4S+gPeFyYBJAkk= +github.com/projectdiscovery/cdncheck v1.2.51 h1:uB+/JdG39HaP3GQ3ncBsWmPh/8cbYmJ3p/UxstHZl0o= +github.com/projectdiscovery/cdncheck v1.2.51/go.mod h1:GIAA0SqjDfxzDQjHR/X+bK05BGmUGfsVplP5Zdv/kHA= +github.com/projectdiscovery/clistats v0.1.6 h1:e2Hc7cVWKJ4sCbyQ7th5zuBvgqsZcLwRq1qY0vmQXY4= +github.com/projectdiscovery/clistats v0.1.6/go.mod h1:RxRGxhMzkT344VTTvZWIrojurUXdQQRN1yfwgg/88I8= +github.com/projectdiscovery/dsl v0.8.22 h1:ZfPunLaaYUJTBffE1ShuAAV9hkSH2thvlszK48FlZS0= +github.com/projectdiscovery/dsl v0.8.22/go.mod h1:fr8WMU381As89e4Z+SQ8tYQ1NudniWAS6NLPHrhe+Vs= +github.com/projectdiscovery/fastdialer v0.5.19 h1:FQOoGGaCyreq53yz3QI+KztvOKn2M9OXJqNg7/4e03w= +github.com/projectdiscovery/fastdialer v0.5.19/go.mod h1:6I4/kPu9s/LTUgmPqIc0ZOEh8TfZGY0Nlr5M84cFAj8= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= github.com/projectdiscovery/freeport v0.0.7 h1:Q6uXo/j8SaV/GlAHkEYQi8WQoPXyJWxyspx+aFmz9Qk= github.com/projectdiscovery/freeport v0.0.7/go.mod h1:cOhWKvNBe9xM6dFJ3RrrLvJ5vXx2NQ36SecuwjenV2k= github.com/projectdiscovery/goconfig v0.0.1 h1:36m3QjohZvemqh9bkJAakaHsm9iEZ2AcQSS18+0QX/s= github.com/projectdiscovery/goconfig v0.0.1/go.mod h1:CPO25zR+mzTtyBrsygqsHse0sp/4vB/PjaHi9upXlDw= -github.com/projectdiscovery/goflags v0.1.76 h1:4OnNU4hvzVdOc+7wcIYBnPOHzo0BcTJ5lC6hS/TeUkA= -github.com/projectdiscovery/goflags v0.1.76/go.mod h1:7nAP1r2Dqgn/rwmOE3EWbZWUCEJKNIhVSBGpuzJAIns= +github.com/projectdiscovery/goflags v0.2.1 h1:XnSYvsCjUwHbZaaJUZ3YznCJPoXRk/IhTFP9Gecmx6M= +github.com/projectdiscovery/goflags v0.2.1/go.mod h1:7nAP1r2Dqgn/rwmOE3EWbZWUCEJKNIhVSBGpuzJAIns= github.com/projectdiscovery/gologger v1.1.72 h1:PKk+aSx3jYOCBPi+FD+nTnhe+Vxx9u3K7Ega0GVlMCI= github.com/projectdiscovery/gologger v1.1.72/go.mod h1:mJwODZcFDg70ihINpOvZevmBtgvpP8H9/l8Y+OPhZPY= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= github.com/projectdiscovery/govaluate v0.0.0-20260504230327-80320480bb6e h1:o+ulEIaC2+9V2Ezr6mI5xEhKWsf0V/+FUQIS723Aj6U= github.com/projectdiscovery/govaluate v0.0.0-20260504230327-80320480bb6e/go.mod h1:xH7bPwHxUlz1yx9UlVeTF+UVCUaKhTnZgaxHb5z362E= -github.com/projectdiscovery/hmap v0.0.101 h1:zXM6YtLmsn8Q0CUUw8QavhqWmiQYwaw+/U679Rr00pc= -github.com/projectdiscovery/hmap v0.0.101/go.mod h1:w6N9/a5H8kvyx53AhtPDUWe5Qq3D6NBDPA23glHpa/Q= +github.com/projectdiscovery/hmap v0.0.102 h1:ybPuf7UQvuOLiHLOUbVMvd+PSlItQU5iQpP1Erl8kaU= +github.com/projectdiscovery/hmap v0.0.102/go.mod h1:IZ92wXFmxYpdn0euZ7VFmypzOBNDwq8c3yRyIIol6XE= github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582 h1:eR+0HE//Ciyfwy3HC7fjRyKShSJHYoX2Pv7pPshjK/Q= github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= github.com/projectdiscovery/mapcidr v1.1.97 h1:7FkxNNVXp+m1rIu5Nv/2SrF9k4+LwP8QuWs2puwy+2w= github.com/projectdiscovery/mapcidr v1.1.97/go.mod h1:9dgTJh1SP02gYZdpzMjm6vtYFkEHQHoTyaVNvaeJ7lA= -github.com/projectdiscovery/networkpolicy v0.1.46 h1:lZblKcIQh8Kd0/JA5AxspxgjNz4ffs2NEf/KxMtXV1g= -github.com/projectdiscovery/networkpolicy v0.1.46/go.mod h1:q1KeQiHchXdElScEMWc5mShWNRNoJXI5koRnVaX6Qh8= +github.com/projectdiscovery/networkpolicy v0.1.48 h1:s/d0nEyfkHYVBeZI/bgRiOCS66KDxv3+s7Hb+AvepHY= +github.com/projectdiscovery/networkpolicy v0.1.48/go.mod h1:bb2ZkDnOeVuR9Ok8BdCdoBEz8S9Y9EFvFFcn5FTUV2U= github.com/projectdiscovery/ratelimit v0.0.88 h1:AcurW9aLRzlEyPe9kSjnOpr3XzLMWTpiWAlW/w73ALU= github.com/projectdiscovery/ratelimit v0.0.88/go.mod h1:CU1s+68UUG2mctSl2wi32/DHLJA6TMg+4rxgP59LfVk= -github.com/projectdiscovery/rawhttp v0.1.91 h1:6EIhZxCkBn27kJAl47FieCFCe4H2ZdbhoBVj0J6ZCVY= -github.com/projectdiscovery/rawhttp v0.1.91/go.mod h1:VZYAM25UI/wVB3URZ95ZaftgOnsbphxyAw/XnQRRz4Y= -github.com/projectdiscovery/retryabledns v1.0.115 h1:RKV63FNIznFHUoawg/1hs53pVH3wqPtFhwstCuxVSoA= -github.com/projectdiscovery/retryabledns v1.0.115/go.mod h1:+fEMWoPigw+M0lGNKY7AZ+g8FIgj+4sONjsinMmeL3k= -github.com/projectdiscovery/retryablehttp-go v1.3.23 h1:s5gsGP7sWJvss1Xeb06Y5TRux8pXvA/xXPCWdPBHOQ8= -github.com/projectdiscovery/retryablehttp-go v1.3.23/go.mod h1:dnzMyBzMmaTHYp5caYCd8mge23qhTGcBjIJ9SagdQ4Q= +github.com/projectdiscovery/rawhttp v0.1.92 h1:38GRsVLNllES/dZcEcA6s6P+rYBp34H9mDphMJMDvvg= +github.com/projectdiscovery/rawhttp v0.1.92/go.mod h1:spfXTPjxCj+bfWLynMVB7uOb+LS3PDNAI9HexMEV1Zg= +github.com/projectdiscovery/retryabledns v1.0.116 h1:tK28PJ4tLO7bgM89RpIFjkfTPMSbRegwvg9mN/q6z78= +github.com/projectdiscovery/retryabledns v1.0.116/go.mod h1:r3vH0AKYtJ9fculcqRFrdVYe2ABcHwZeR4C6j3rqCbY= +github.com/projectdiscovery/retryablehttp-go v1.3.25 h1:caWjhagt49zkSbaOZnsD+qi8UwfgbofezHOtYPHvhb0= +github.com/projectdiscovery/retryablehttp-go v1.3.25/go.mod h1:+lyetyXnmFCGzjfebzVCsszts9JHkKncal+nNwYeqCs= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= -github.com/projectdiscovery/tlsx v1.3.2 h1:1Lh2ith79o8R+rOOSBZiMq8EiZ8j4D3T/8ENO+gAqXA= -github.com/projectdiscovery/tlsx v1.3.2/go.mod h1:2wgTGC/sourHvoR+RX8cAok0Sa8YK6QbEP3xo5SDzPI= -github.com/projectdiscovery/useragent v0.0.108 h1:fb+uLuFJvC+MHZjCtxQJxtvp1X6A8n98CUGPyFcg3NE= -github.com/projectdiscovery/useragent v0.0.108/go.mod h1:XdNRrlvtDmYfVL1Oybat4uMe+W6cLwsK9S18ond17CI= -github.com/projectdiscovery/utils v0.11.1 h1:PWj1KjIASxt8icxommH72C0TQqNOvGkcSODRkiq0SQw= -github.com/projectdiscovery/utils v0.11.1/go.mod h1:yktGrHGk2CTjNiccXovnvGrLHX9sV2bqz9nSnbA3V8M= -github.com/projectdiscovery/wappalyzergo v0.2.95 h1:XHjbQVrrxusezFq7ciWmtznPKWhpr/srWp7Yk2noxqg= -github.com/projectdiscovery/wappalyzergo v0.2.95/go.mod h1:E2p8L90ysTTUkU5FUOgGoCQ7ucEUgqJ/tvYgiGGAlEM= +github.com/projectdiscovery/tlsx v1.4.0 h1:OU2mNK6TOB/YiFX3jhmgwOepFWxPoj5rn8npWiCCz74= +github.com/projectdiscovery/tlsx v1.4.0/go.mod h1:WCrUXAHjdKbDNjsu5EbVBS7qvzap3fxi/E85v/Flyx8= +github.com/projectdiscovery/useragent v0.0.109 h1:b86BPdZvmgbJousa04jAugDRY4Y+rAvEamE0TFCPmeE= +github.com/projectdiscovery/useragent v0.0.109/go.mod h1:sRyBqDAcD31RKYrO8fyXGowLd36fNwY1g0MNW9mp7zk= +github.com/projectdiscovery/utils v0.11.3 h1:TNBhSNJ8uFw7DWASUHBf1vfCx7mwMi+jtGizrizMM5s= +github.com/projectdiscovery/utils v0.11.3/go.mod h1:HMxhxLigsAr+M9Oa8n9Z0ROZcs8wAJHGsgR7UGb+oUE= +github.com/projectdiscovery/wappalyzergo v0.2.96 h1:/YHMiUA2kL2bap2vHDuZs1cJK3RgHZjxdVam5GxJ12k= +github.com/projectdiscovery/wappalyzergo v0.2.96/go.mod h1:5ZuunxbKPGvnJeEOMIqlfj5xg/XSEAoakZr/s9l2pNI= github.com/refraction-networking/utls v1.8.2 h1:j4Q1gJj0xngdeH+Ox/qND11aEfhpgoEvV+S9iJ2IdQo= github.com/refraction-networking/utls v1.8.2/go.mod h1:jkSOEkLqn+S/jtpEHPOsVv/4V4EVnelwbMQl4vCWXAM= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -418,8 +418,8 @@ github.com/zmap/zcrypto v0.0.0-20201211161100-e54a5822fb7e/go.mod h1:aPM7r+JOkfL github.com/zmap/zcrypto v0.0.0-20240803002437-3a861682ac77 h1:DCz0McWRVJNICkHdu2XpETqeLvPtZXs315OZyUs1BDk= github.com/zmap/zcrypto v0.0.0-20240803002437-3a861682ac77/go.mod h1:aSvf+uTU222mUYq/KQj3oiEU7ajhCZe8RRSLHIoM4EM= github.com/zmap/zlint/v3 v3.0.0/go.mod h1:paGwFySdHIBEMJ61YjoqT4h7Ge+fdYG4sUQhnTb1lJ8= -go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= -go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= +go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= +go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= go.mongodb.org/mongo-driver v1.17.9 h1:IexDdCuuNJ3BHrELgBlyaH9p60JXAvdzWR128q+U5tU= go.mongodb.org/mongo-driver v1.17.9/go.mod h1:LlOhpH5NUEfhxcAwG0UEkMqwYcc4JU18gtCdGudk/tQ= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -439,7 +439,6 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= -golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= golang.org/x/exp v0.0.0-20260112195511-716be5621a96 h1:Z/6YuSHTLOHfNFdb8zVZomZr7cqNgTJvA8+Qz75D8gU= @@ -449,8 +448,6 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= -golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -468,7 +465,6 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= -golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -482,7 +478,6 @@ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -511,7 +506,6 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= @@ -525,7 +519,6 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= -golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -542,7 +535,6 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= @@ -553,8 +545,6 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= -golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -564,9 +554,8 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= diff --git a/runner/banner.go b/runner/banner.go index 3f021e8c7..702af69d0 100644 --- a/runner/banner.go +++ b/runner/banner.go @@ -16,7 +16,7 @@ const banner = ` ` // Version is the current Version of httpx -const Version = `v1.11.0` +const Version = `v1.12.0` // showBanner is used to show the banner to the user func showBanner() { diff --git a/runner/cpe.go b/runner/cpe.go index 5d7c91f08..58b7243e8 100644 --- a/runner/cpe.go +++ b/runner/cpe.go @@ -183,21 +183,32 @@ var cpeProductSuffixes = []string{ "_policy_manager", } +// cpeProductAliases maps known dataset naming differences that cannot be +// resolved safely by the generic token fallback. +var cpeProductAliases = map[string][]string{ + "d3.js": {"d3"}, + "matomo": {"matomoanalytics"}, +} + +// primaryProductName returns the primary identifier from compound +// awesome-search-queries product names. +func primaryProductName(product string) string { + product = strings.TrimSpace(product) + if idx := strings.Index(product, ","); idx >= 0 { + product = strings.TrimSpace(product[:idx]) + } + return product +} + // productLookupKeys returns normalized lookup keys for joining a CPE product // name to wappalyzer technology names. Keys are ordered most-specific first; // the first key with a version match wins. func productLookupKeys(product string) []string { - product = strings.TrimSpace(product) + product = primaryProductName(product) if product == "" { return nil } - // Compound awesome-search-queries names list multiple products; the first is - // the primary identifier for version lookup purposes. - if idx := strings.Index(product, ","); idx >= 0 { - product = strings.TrimSpace(product[:idx]) - } - seen := make(map[string]struct{}) keys := make([]string, 0, 4) addKey := func(raw string) { @@ -215,6 +226,10 @@ func productLookupKeys(product string) []string { addKey(product) lower := strings.ToLower(product) + for _, alias := range cpeProductAliases[lower] { + addKey(alias) + } + suffixes := slices.Clone(cpeProductSuffixes) slices.SortFunc(suffixes, func(a, b string) int { return len(b) - len(a) @@ -236,15 +251,189 @@ func productLookupKeys(product string) []string { return keys } +// fallbackProductLookupKeys returns only aliases that preserve the complete +// product identity. Unlike productLookupKeys, it does not add underscore +// prefixes such as "tomcat" for "tomcat_jk_connector", which are useful for +// exact technology names but too broad after vendor-prefix removal. +func fallbackProductLookupKeys(product string) []string { + product = primaryProductName(product) + if product == "" { + return nil + } + + seen := make(map[string]struct{}) + keys := make([]string, 0, 3) + addKey := func(raw string) { + key := normalizeProductName(raw) + if key == "" { + return + } + if _, ok := seen[key]; ok { + return + } + seen[key] = struct{}{} + keys = append(keys, key) + } + + addKey(product) + lower := strings.ToLower(product) + for _, alias := range cpeProductAliases[lower] { + addKey(alias) + } + for _, suffix := range cpeProductSuffixes { + if strings.HasSuffix(lower, suffix) { + addKey(strings.TrimSuffix(lower, suffix)) + } + } + return keys +} + +// minTokenMatchLen guards vendor matching so short words cannot establish +// identity between unrelated technologies. +const minTokenMatchLen = 5 + +// genericMatchTokens are too common to establish vendor identity. +var genericMatchTokens = map[string]struct{}{ + "server": {}, "portal": {}, "software": {}, "platform": {}, "suite": {}, + "service": {}, "services": {}, "manager": {}, "panel": {}, "cms": {}, + "firmware": {}, "gateway": {}, "proxy": {}, "system": {}, "application": {}, + "framework": {}, "tower": {}, "commerce": {}, "ecommerce": {}, "network": {}, + "internet": {}, "captcha": {}, "cloud": {}, "web": {}, "app": {}, + "plugin": {}, "widget": {}, "tool": {}, "tools": {}, "analytics": {}, + "tag": {}, "api": {}, "project": {}, "builder": {}, +} + +// technologyTokens splits a name into lowercase alphanumeric words, breaking +// on every non-alphanumeric rune. Unlike normalizeProductName, words are kept +// separate rather than concatenated: "React" -> ["react"], "Apache Tomcat" -> +// ["apache", "tomcat"], "d3.js" -> ["d3", "js"]. This preserves word +// boundaries that a plain substring match would ignore - "react" is a +// character-for-character substring of "preact", but the two never share a +// token. +func technologyTokens(name string) []string { + var tokens []string + var b strings.Builder + flush := func() { + if b.Len() > 0 { + tokens = append(tokens, b.String()) + b.Reset() + } + } + for _, r := range name { + switch { + case r >= 'a' && r <= 'z', r >= '0' && r <= '9': + b.WriteRune(r) + case r >= 'A' && r <= 'Z': + b.WriteRune(r + ('a' - 'A')) + default: + flush() + } + } + flush() + return tokens +} + +// strongTokens returns technologyTokens filtered to words long and specific +// enough to be trusted for fallback matching. +func strongTokens(name string) []string { + tokens := technologyTokens(name) + filtered := tokens[:0] + for _, t := range tokens { + if len(t) < minTokenMatchLen { + continue + } + if _, generic := genericMatchTokens[t]; generic { + continue + } + filtered = append(filtered, t) + } + return filtered +} + +type technologyVersion struct { + version string + tokens []string +} + +// buildTechnologyVersions parses versioned technology names into candidates +// used by the conservative product/vendor fallback. +func buildTechnologyVersions(technologies []string) []technologyVersion { + candidates := make([]technologyVersion, 0, len(technologies)) + for _, tech := range technologies { + parts := strings.SplitN(tech, ":", 2) + if len(parts) != 2 { + continue + } + version := strings.TrimSpace(parts[1]) + if version == "" { + continue + } + tokens := technologyTokens(parts[0]) + if len(tokens) == 0 { + continue + } + candidates = append(candidates, technologyVersion{version: version, tokens: tokens}) + } + return candidates +} + // lookupTechVersion finds a wappalyzer version for a CPE product using exact // and alias keys derived from awesome-search-queries naming conventions. -func lookupTechVersion(product string, versions map[string]string) (string, bool) { - for _, key := range productLookupKeys(product) { +// +// If no exact key matches, the fallback removes CPE vendor words from each +// detected technology and requires the remaining name to equal a product +// lookup key. This catches "Apache Tomcat" versus vendor "apache", product +// "tomcat" without matching unrelated products on broad shared words. Every +// candidate must agree on the version. +func lookupTechVersion(product, vendor string, versions map[string]string, candidates []technologyVersion) (string, bool) { + keys := productLookupKeys(product) + for _, key := range keys { if version, ok := versions[key]; ok { return version, true } } - return "", false + + fallbackKeys := fallbackProductLookupKeys(product) + productKeys := make(map[string]struct{}, len(fallbackKeys)) + for _, key := range fallbackKeys { + productKeys[key] = struct{}{} + } + vendorTokens := make(map[string]struct{}) + for _, token := range strongTokens(vendor) { + vendorTokens[token] = struct{}{} + } + if len(productKeys) == 0 || len(vendorTokens) == 0 { + return "", false + } + + var version string + for _, candidate := range candidates { + var residual strings.Builder + vendorMatch := false + for _, token := range candidate.tokens { + if _, ok := vendorTokens[token]; ok { + vendorMatch = true + continue + } + residual.WriteString(token) + } + if !vendorMatch { + continue + } + if _, ok := productKeys[residual.String()]; !ok { + continue + } + + if version == "" { + version = candidate.version + continue + } + if version != candidate.version { + return "", false + } + } + + return version, version != "" } // buildTechVersionMap maps normalized technology name -> version, parsing @@ -286,15 +475,17 @@ func EnrichCPEVersions(matches []CPEInfo, technologies []string) []CPEInfo { return append([]CPEInfo(nil), matches...) } versions := buildTechVersionMap(technologies) + candidates := buildTechnologyVersions(technologies) enriched := make([]CPEInfo, len(matches)) for i, match := range matches { enriched[i] = match - if version, ok := lookupTechVersion(match.Product, versions); ok { + if version, ok := lookupTechVersion(match.Product, match.Vendor, versions, candidates); ok { enriched[i].CPE = setCPEVersion(match.CPE, version) } } return enriched + } func (d *CPEDetector) extractPattern(query string, info CPEInfo) { diff --git a/runner/cpe_test.go b/runner/cpe_test.go index 168fc2019..695e2f4cb 100644 --- a/runner/cpe_test.go +++ b/runner/cpe_test.go @@ -202,6 +202,16 @@ func TestProductLookupKeys(t *testing.T) { in: "next.js", want: []string{"nextjs"}, }, + { + name: "known short alias", + in: "d3.js", + want: []string{"d3js", "d3"}, + }, + { + name: "known display alias", + in: "matomo", + want: []string{"matomo", "matomoanalytics"}, + }, { name: "empty", in: "", @@ -223,16 +233,55 @@ func TestProductLookupKeys(t *testing.T) { } } +func TestFallbackProductLookupKeys(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + in string + want []string + }{ + {name: "bare product", in: "tomcat", want: []string{"tomcat"}}, + {name: "known suffix", in: "confluence_server", want: []string{"confluenceserver", "confluence"}}, + {name: "known short alias", in: "d3.js", want: []string{"d3js", "d3"}}, + {name: "known display alias", in: "matomo", want: []string{"matomo", "matomoanalytics"}}, + {name: "does not use arbitrary prefix", in: "tomcat_jk_connector", want: []string{"tomcatjkconnector"}}, + {name: "compound uses primary", in: "digital_experience_platform,liferay_portal", want: []string{"digitalexperienceplatform", "digitalexperience"}}, + {name: "empty", in: "", want: nil}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := fallbackProductLookupKeys(tt.in) + if len(got) != len(tt.want) { + t.Fatalf("fallbackProductLookupKeys(%q) = %v, want %v", tt.in, got, tt.want) + } + for i := range tt.want { + if got[i] != tt.want[i] { + t.Fatalf("fallbackProductLookupKeys(%q)[%d] = %q, want %q", tt.in, i, got[i], tt.want[i]) + } + } + }) + } +} + func TestLookupTechVersion(t *testing.T) { t.Parallel() - versions := buildTechVersionMap([]string{ + technologies := []string{ "Liferay:7.3.5", "Confluence:8.5.1", "Tableau:2023.1", "Apache HTTP Server:2.4.7", "Ansible:2.14.0", - }) + "Apache Tomcat:9.0.65", + "Preact:10.5.0", + "D3:7.8.5", + "Atlassian Jira:9.12.0", + "Matomo Analytics:5.0.0", + } + versions := buildTechVersionMap(technologies) + candidates := buildTechnologyVersions(technologies) tests := []struct { product string @@ -247,10 +296,26 @@ func TestLookupTechVersion(t *testing.T) { {"Apache HTTP Server", "2.4.7", true}, {"phpcollab", "", false}, {"unknown_product", "", false}, + // #2550: vendor-prefixed wappalyzer name ("Apache Tomcat") vs. the + // bare CPE product name ("tomcat") - resolved by the whole-word + // token fallback. + {"tomcat", "9.0.65", true}, + {"d3.js", "7.8.5", true}, + {"jira", "9.12.0", true}, + {"matomo", "5.0.0", true}, + // CodeRabbit review on #2569: "react" is a character-for-character + // substring of "preact", but they must never be treated as a match. + // Whole-word tokenization keeps them distinct ("react" != "preact" + // as tokens), unlike a raw substring check. + {"react", "", false}, } for _, tt := range tests { t.Run(tt.product, func(t *testing.T) { - got, ok := lookupTechVersion(tt.product, versions) + vendor := map[string]string{ + "tomcat": "apache", + "jira": "atlassian", + }[tt.product] + got, ok := lookupTechVersion(tt.product, vendor, versions, candidates) if ok != tt.wantFound { t.Fatalf("lookupTechVersion(%q) found = %v, want %v", tt.product, ok, tt.wantFound) } @@ -261,6 +326,208 @@ func TestLookupTechVersion(t *testing.T) { } } +func TestTechnologyTokens(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + in string + want []string + }{ + {name: "vendor prefix", in: "Apache Tomcat", want: []string{"apache", "tomcat"}}, + {name: "punctuation", in: "D3.js / Plugin", want: []string{"d3", "js", "plugin"}}, + {name: "underscore", in: "dashboard_console", want: []string{"dashboard", "console"}}, + {name: "case folding", in: "PreACT", want: []string{"preact"}}, + {name: "empty", in: "", want: nil}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := technologyTokens(tt.in) + if len(got) != len(tt.want) { + t.Fatalf("technologyTokens(%q) = %v, want %v", tt.in, got, tt.want) + } + for i := range tt.want { + if got[i] != tt.want[i] { + t.Fatalf("technologyTokens(%q)[%d] = %q, want %q", tt.in, i, got[i], tt.want[i]) + } + } + }) + } +} + +func TestStrongTokens(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + in string + want []string + }{ + {name: "keeps specific words", in: "Apache Tomcat Server", want: []string{"apache", "tomcat"}}, + {name: "drops generic words", in: "Cloud Web Application Framework", want: nil}, + {name: "drops short words", in: "D3.js API", want: nil}, + {name: "keeps token boundaries", in: "React Preact", want: []string{"react", "preact"}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := strongTokens(tt.in) + if len(got) != len(tt.want) { + t.Fatalf("strongTokens(%q) = %v, want %v", tt.in, got, tt.want) + } + for i := range tt.want { + if got[i] != tt.want[i] { + t.Fatalf("strongTokens(%q)[%d] = %q, want %q", tt.in, i, got[i], tt.want[i]) + } + } + }) + } +} + +func TestBuildTechnologyVersions(t *testing.T) { + t.Parallel() + + candidates := buildTechnologyVersions([]string{ + "Apache Tomcat:9.0.65", + "Oracle Tomcat:10.1.0", + "Vendor Dashboard:1.0", + "D3:7.8.5", + "Missing Version", + "Blank:", + "---:1.0", + }) + if len(candidates) != 4 { + t.Fatalf("candidates = %#v, want 4 versioned candidates", candidates) + } + if candidates[0].version != "9.0.65" { + t.Fatalf("first candidate version = %q, want 9.0.65", candidates[0].version) + } + for _, token := range []string{"apache", "tomcat"} { + if !sliceContains(candidates[0].tokens, token) { + t.Fatalf("first candidate tokens = %v, want %q", candidates[0].tokens, token) + } + } +} + +func TestLookupTechVersionRejectsAmbiguousFallback(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + product string + vendor string + technologies []string + want string + wantFound bool + }{ + { + name: "same fallback name has different versions", + product: "dashboard", + vendor: "acmecorp", + technologies: []string{"AcmeCorp Dashboard:1.0", "AcmeCorp Dashboard:2.0"}, + }, + { + name: "conflict is independent of input order", + product: "dashboard", + vendor: "acmecorp", + technologies: []string{"AcmeCorp Dashboard:2.0", "AcmeCorp Dashboard:1.0"}, + }, + { + name: "vendor disambiguates shared product token", + product: "tomcat", + vendor: "apache", + technologies: []string{"Apache Tomcat:9.0.65", "Oracle Tomcat:10.1.0"}, + want: "9.0.65", + wantFound: true, + }, + { + name: "matching fallback candidates agree", + product: "dashboard", + vendor: "acmecorp", + technologies: []string{"AcmeCorp Dashboard:1.0", "AcmeCorp Dashboard:1.0"}, + want: "1.0", + wantFound: true, + }, + { + name: "secondary compound product is ignored", + product: "digital_experience_platform,liferay_portal", + vendor: "acmecorp", + technologies: []string{"AcmeCorp Liferay:7.3.5"}, + }, + { + name: "exact match wins before ambiguous fallback", + product: "dashboard_console", + vendor: "acmecorp", + technologies: []string{"Dashboard Console:9.0", "AcmeCorp Dashboard:1.0", "AcmeCorp Dashboard:2.0"}, + want: "9.0", + wantFound: true, + }, + { + name: "shared brand is not product evidence", + product: "google_maps", + vendor: "google", + technologies: []string{"Google Analytics:4.0"}, + }, + { + name: "unrelated vendor rejects broad product token", + product: "wp-google-maps", + vendor: "wpgmaps", + technologies: []string{"Google Analytics:4.0"}, + }, + { + name: "extended vendor product is not shortened", + product: "experience_manager", + vendor: "adobe", + technologies: []string{"Adobe Experience Manager Edge Delivery Services:1.0"}, + }, + { + name: "connector is not parent product", + product: "tomcat_jk_connector", + vendor: "apache", + technologies: []string{"Apache Tomcat:9.0.65"}, + }, + { + name: "code editor is not visual studio", + product: "visual_studio_code", + vendor: "microsoft", + technologies: []string{"Microsoft Visual Studio:17.0"}, + }, + { + name: "service management is not jira", + product: "jira_service_management", + vendor: "atlassian", + technologies: []string{"Atlassian Jira:10.0"}, + }, + { + name: "jquery family name is not jquery", + product: "jquery-bbq", + vendor: "jquery-bbq_project", + technologies: []string{"jQuery:3.7.0"}, + }, + { + name: "wordpress is not microsoft word", + product: "wordpress", + vendor: "wordpress", + technologies: []string{"Microsoft Word:16.0"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, ok := lookupTechVersion( + tt.product, + tt.vendor, + buildTechVersionMap(tt.technologies), + buildTechnologyVersions(tt.technologies), + ) + if ok != tt.wantFound || got != tt.want { + t.Fatalf("lookupTechVersion(%q) = %q, %v; want %q, %v", tt.product, got, ok, tt.want, tt.wantFound) + } + }) + } +} + func TestEnrichCPEVersionsIssue2536(t *testing.T) { t.Parallel() @@ -321,6 +588,7 @@ func TestEnrichCPEVersionsIssue2536(t *testing.T) { wantCPE: "cpe:2.3:a:redhat:ansible_policy_manager:2.14.0:*:*:*:*:*:*:*", }, { + name: "conflicting tech versions leave cpe unchanged", product: "liferay_portal", vendor: "liferay", @@ -328,6 +596,28 @@ func TestEnrichCPEVersionsIssue2536(t *testing.T) { technologies: []string{"Liferay:7.3.5", "Liferay:7.4.0"}, wantCPE: "cpe:2.3:a:liferay:liferay_portal:*:*:*:*:*:*:*:*", }, + { + // #2550: wappalyzer's display name carries a vendor prefix + // ("Apache Tomcat") that awesome-search-queries' bare CPE + // product name ("tomcat") doesn't. Resolved by the whole-word + // token fallback in lookupTechVersion. + name: "vendor-prefixed wappalyzer name matches bare CPE product (#2550)", + product: "tomcat", + vendor: "apache", + cpe: "cpe:2.3:a:apache:tomcat:*:*:*:*:*:*:*:*", + technologies: []string{"Apache Tomcat:9.0.65"}, + wantCPE: "cpe:2.3:a:apache:tomcat:9.0.65:*:*:*:*:*:*:*", + }, + { + // Reverse case: a narrow alias bridges the CPE product "d3.js" to + // wappalyzer's short display name without weakening token guards. + name: "bare wappalyzer name matches suffixed CPE product (#2550)", + product: "d3.js", + vendor: "d3.js_project", + cpe: "cpe:2.3:a:d3.js_project:d3.js:*:*:*:*:*:*:*:*", + technologies: []string{"D3:7.8.5"}, + wantCPE: "cpe:2.3:a:d3.js_project:d3.js:7.8.5:*:*:*:*:*:*:*", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -424,6 +714,74 @@ func TestEnrichCPEVersionsWithRealWappalyzer(t *testing.T) { } } +func TestEnrichTomcatCPEWithRealDatasets(t *testing.T) { + wappalyze, err := wappalyzer.New() + if err != nil { + t.Fatalf("could not create wappalyzer: %s", err) + } + + info := wappalyze.FingerprintWithInfo(map[string][]string{ + "x-powered-by": {"Tomcat-9.0.65"}, + }, nil) + var technologies []string + for name := range info { + technologies = append(technologies, name) + } + if !sliceContains(technologies, "Apache Tomcat:9.0.65") { + t.Fatalf("expected real wappalyzer data to emit Apache Tomcat:9.0.65, got %v", technologies) + } + + detector, err := NewCPEDetector() + if err != nil { + t.Fatalf("could not create CPE detector: %s", err) + } + matches := detector.Detect("", "Apache Tomcat", "") + var found bool + for _, match := range EnrichCPEVersions(matches, technologies) { + if match.Product != "tomcat" { + continue + } + found = true + want := "cpe:2.3:a:apache:tomcat:9.0.65:*:*:*:*:*:*:*" + if match.CPE != want { + t.Fatalf("Tomcat CPE = %q, want %q", match.CPE, want) + } + } + if !found { + t.Fatalf("real awesome-search-queries data did not detect the Tomcat product; matches: %v", matches) + } +} + +func TestEnrichVendorPrefixedTechnologiesIndependently(t *testing.T) { + matches := []CPEInfo{ + { + Product: "tomcat", + Vendor: "apache", + CPE: "cpe:2.3:a:apache:tomcat:*:*:*:*:*:*:*:*", + }, + { + Product: "http_server", + Vendor: "apache", + CPE: "cpe:2.3:a:apache:http_server:*:*:*:*:*:*:*:*", + }, + } + technologies := []string{ + "Apache Tomcat:9.0.65", + "Apache HTTP Server:2.4.62", + } + + got := EnrichCPEVersions(matches, technologies) + want := []string{ + "cpe:2.3:a:apache:tomcat:9.0.65:*:*:*:*:*:*:*", + "cpe:2.3:a:apache:http_server:2.4.62:*:*:*:*:*:*:*", + } + for i := range want { + if got[i].CPE != want[i] { + t.Fatalf("CPE[%d] = %q, want %q", i, got[i].CPE, want[i]) + } + } +} + func sliceContains(s []string, v string) bool { for _, e := range s { if e == v { diff --git a/runner/runner.go b/runner/runner.go index 63dccb05d..ea0cac3f3 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1829,6 +1829,16 @@ func (r *Runner) analyze(hp *httpx.HTTPX, protocol string, target httpx.Target, protocol = determineMostLikelySchemeOrder(target.Host) } retried := false + tlsUpgraded := false + // The plaintext attempt is kept until an HTTPS one has actually succeeded. + // URL is cloned because the retry rewrites its scheme in place, while the + // restored value is used by downstream probes such as SupportHTTP2. + var ( + keptResp *httpx.Response + keptReq *retryablehttp.Request + keptURL *urlutil.URL + keptProtocol string + ) retry: if scanopts.VHostInput && target.CustomHost == "" { return Result{Input: origInput} @@ -1927,6 +1937,24 @@ retry: if r.options.ShowStatistics { r.stats.IncrementCounter("requests", 1) } + // Fall back to the response already in hand rather than asking again: a + // transient or one-shot service may not answer a second time. + if err != nil && keptResp != nil { + resp, err, req, URL, protocol = keptResp, nil, keptReq, keptURL, keptProtocol + keptResp, keptReq, keptURL = nil, nil, nil + } + // A 400 to a plaintext probe is a successful transaction, so the scheme + // retry below never fires and a TLS-only port is reported as plain http. + // Attempt HTTPS through the normal client path, which starts with the same + // handshake: a port that does not speak TLS fails there and the response + // kept above is restored. Unsafe mode bypasses the scheme retry entirely. + if err == nil && !tlsUpgraded && !scanopts.Unsafe && origProtocol == httpx.HTTPorHTTPS && + protocol == httpx.HTTP && resp != nil && resp.StatusCode == http.StatusBadRequest { + keptResp, keptReq, keptURL, keptProtocol = resp, req, URL.Clone(), protocol + protocol = httpx.HTTPS + tlsUpgraded = true + goto retry + } var requestDump []byte if scanopts.Unsafe { var errDump error diff --git a/runner/runner_test.go b/runner/runner_test.go index 5a847852e..3f9d5e559 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -1,10 +1,22 @@ package runner import ( + "bufio" + "context" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" "fmt" + "io" + "math/big" + "net" + "net/http" "os" "strings" "sync" + "sync/atomic" "testing" "time" @@ -529,8 +541,8 @@ func TestStoreResponse_withoutMatchersStoresAll(t *testing.T) { func TestStoreResponse_withMatcherSetsFlag(t *testing.T) { dir := t.TempDir() opts := &Options{ - StoreResponse: true, - StoreResponseDir: dir, + StoreResponse: true, + StoreResponseDir: dir, OutputMatchStatusCode: "200", } err := opts.ValidateOptions() @@ -768,3 +780,389 @@ func TestCreateNetworkpolicyInstance_AllowDenyFlags(t *testing.T) { }) } } + +// startTLSOnlyListener serves content over TLS and answers any plaintext +// request the way a real TLS listener does: a valid HTTP 400 saying TLS is +// required. The response is a successful HTTP transaction, which is what stops +// the transport-error scheme retry from firing. +func startTLSOnlyListener(t *testing.T) string { + t.Helper() + + const rejection = "HTTP/1.1 400 Bad Request\r\n" + + "Connection: close\r\n" + + "Content-Type: text/plain;charset=utf-8\r\n" + + "Content-Length: 62\r\n\r\n" + + "Bad Request\r\nThis combination of host and port requires TLS.\r\n" + + key, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + + template := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "tls-only.test"}, + DNSNames: []string{"localhost", "tls-only.test"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + der, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key) + require.NoError(t, err) + + tlsConfig := &tls.Config{Certificates: []tls.Certificate{{Certificate: [][]byte{der}, PrivateKey: key}}} + + mux := http.NewServeMux() + mux.HandleFunc("/", func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/html") + _, _ = io.WriteString(w, "Only Over TLSok") + }) + server := &http.Server{Handler: mux, ReadHeaderTimeout: 5 * time.Second} + + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = listener.Close() }) + + go func() { + for { + conn, err := listener.Accept() + if err != nil { + return + } + go func(conn net.Conn) { + buffered := bufio.NewReader(conn) + first, err := buffered.Peek(1) + if err != nil { + _ = conn.Close() + return + } + // 0x16 is a TLS handshake record; anything else is plaintext. + if first[0] != 0x16 { + drainRequest(buffered) + _, _ = io.WriteString(conn, rejection) + _ = conn.Close() + return + } + // Always returns io.EOF: the listener yields this one connection. + _ = server.Serve(oneShot(tls.Server(&peeked{Conn: conn, reader: buffered}, tlsConfig))) + }(conn) + } + }() + + return listener.Addr().String() +} + +// drainRequest reads the request head before a reply is written. Answering an +// HTTP client that has not finished asking is an unsolicited response, and it +// discards the reply instead of parsing it. +func drainRequest(r io.Reader) { + if conn, ok := r.(net.Conn); ok { + _ = conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + } + scanner := bufio.NewScanner(r) + for scanner.Scan() { + if scanner.Text() == "" { + return + } + } +} + +type peeked struct { + net.Conn + reader *bufio.Reader +} + +func (c *peeked) Read(b []byte) (int, error) { return c.reader.Read(b) } + +type oneShotListener struct { + conn net.Conn + used bool +} + +func oneShot(conn net.Conn) net.Listener { return &oneShotListener{conn: conn} } + +func (l *oneShotListener) Accept() (net.Conn, error) { + if l.used { + return nil, io.EOF + } + l.used = true + return l.conn, nil +} +func (l *oneShotListener) Close() error { return nil } +func (l *oneShotListener) Addr() net.Addr { return l.conn.LocalAddr() } + +// TestTLSOnlyPortIsProbedOverHTTPS covers a TLS-only service on a port above +// 1024, which the scheme heuristic probes as plain HTTP first. Without the +// retry on a TLS-required response the service is reported as plain http. +func TestTLSOnlyPortIsProbedOverHTTPS(t *testing.T) { + target := startTLSOnlyListener(t) + + var ( + mu sync.Mutex + results []Result + ) + + options := &Options{ + Threads: 1, + RateLimit: 10, + Retries: 0, + Timeout: 5, + Methods: http.MethodGet, + Delay: -1, + OnResult: func(r Result) { + if r.Err != nil || r.URL == "" { + return + } + mu.Lock() + results = append(results, r) + mu.Unlock() + }, + InputTargetHost: []string{target}, + } + + // The heuristic must pick http first, otherwise this test proves nothing. + require.Equal(t, "http", determineMostLikelySchemeOrder(target)) + + r, err := New(options) + require.NoError(t, err) + defer r.Close() + r.RunEnumeration() + + mu.Lock() + defer mu.Unlock() + + require.Len(t, results, 1) + require.Equal(t, "https", results[0].Scheme) + require.Equal(t, "https://"+target, results[0].URL) + require.Equal(t, http.StatusOK, results[0].StatusCode) +} + +// TestPlainHTTPPortStaysHTTP is the no-regression half of the TLS upgrade: a +// genuine cleartext service that answers 400 must not be relabelled https just +// because the upgrade was attempted. +func TestPlainHTTPPortStaysHTTP(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = listener.Close() }) + + go func() { + for { + conn, err := listener.Accept() + if err != nil { + return + } + go func(conn net.Conn) { + drainRequest(conn) + _, _ = io.WriteString(conn, "HTTP/1.1 400 Bad Request\r\nConnection: close\r\n"+ + "Content-Type: text/plain\r\nContent-Length: 11\r\n\r\nBad Request") + _ = conn.Close() + }(conn) + } + }() + + target := listener.Addr().String() + + var ( + mu sync.Mutex + results []Result + ) + options := &Options{ + Threads: 1, + RateLimit: 10, + Retries: 0, + Timeout: 5, + Methods: http.MethodGet, + Delay: -1, + OnResult: func(r Result) { + if r.Err != nil || r.URL == "" { + return + } + mu.Lock() + results = append(results, r) + mu.Unlock() + }, + InputTargetHost: []string{target}, + } + + r, err := New(options) + require.NoError(t, err) + defer r.Close() + r.RunEnumeration() + + mu.Lock() + defer mu.Unlock() + + require.Len(t, results, 1) + require.Equal(t, "http", results[0].Scheme, "a cleartext 400 must stay http") + require.Equal(t, http.StatusBadRequest, results[0].StatusCode) +} + +// TestOneShotPlainHTTPKeepsItsResult covers a cleartext service that answers +// once and then refuses: the scheme decision must not cost it a second +// request, or a reachable service disappears from the output. +func TestOneShotPlainHTTPKeepsItsResult(t *testing.T) { + var served int64 + + listener, err := (&net.ListenConfig{}).Listen(context.Background(), "tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = listener.Close() }) + + go func() { + for { + conn, err := listener.Accept() + if err != nil { + return + } + go func(conn net.Conn) { + defer func() { _ = conn.Close() }() + if atomic.AddInt64(&served, 1) != 1 { + return // refuse every later connection + } + _ = conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + drainRequest(conn) + _, _ = io.WriteString(conn, "HTTP/1.1 400 Bad Request\r\nConnection: close\r\n"+ + "Content-Length: 11\r\n\r\nBad Request") + }(conn) + } + }() + + var ( + mu sync.Mutex + results []Result + ) + options := &Options{ + Threads: 1, RateLimit: 10, Retries: 0, Timeout: 5, + Methods: http.MethodGet, Delay: -1, + InputTargetHost: []string{listener.Addr().String()}, + OnResult: func(r Result) { + if r.Err != nil || r.URL == "" { + return + } + mu.Lock() + results = append(results, r) + mu.Unlock() + }, + } + + runner, err := New(options) + require.NoError(t, err) + defer runner.Close() + runner.RunEnumeration() + + mu.Lock() + defer mu.Unlock() + + require.Len(t, results, 1, "a service that answered once must still be reported") + require.Equal(t, "http", results[0].Scheme) + require.Equal(t, http.StatusBadRequest, results[0].StatusCode) +} + +// TestHandshakeThenCloseKeepsPlainResult covers a port whose TLS handshake +// succeeds and which then closes without answering: the upgrade must fall back +// to the plaintext response rather than request it again, so a cleartext +// service that answers only once still appears in the output. +func TestHandshakeThenCloseKeepsPlainResult(t *testing.T) { + var plainServed, h2cServed, tlsHandshakes int64 + + key, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + template := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "handshake.test"}, + DNSNames: []string{"localhost"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + der, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key) + require.NoError(t, err) + tlsConfig := &tls.Config{Certificates: []tls.Certificate{{Certificate: [][]byte{der}, PrivateKey: key}}} + + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = listener.Close() }) + + go func() { + for { + conn, err := listener.Accept() + if err != nil { + return + } + go func(conn net.Conn) { + defer func() { _ = conn.Close() }() + buffered := bufio.NewReader(conn) + first, err := buffered.Peek(1) + if err != nil { + return + } + // 0x16 is a TLS ClientHello: complete the handshake, then close + // without ever sending an HTTP response. + if first[0] == 0x16 { + tlsConn := tls.Server(&peeked{Conn: conn, reader: buffered}, tlsConfig) + if err := tlsConn.HandshakeContext(context.Background()); err != nil { + return + } + atomic.AddInt64(&tlsHandshakes, 1) + _ = tlsConn.Close() + return + } + _ = conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + request, err := http.ReadRequest(buffered) + if err != nil { + return + } + _ = request.Body.Close() + if strings.EqualFold(request.Header.Get("Upgrade"), "h2c") { + atomic.AddInt64(&h2cServed, 1) + _, _ = io.WriteString(conn, "HTTP/1.1 101 Switching Protocols\r\n"+ + "Connection: Upgrade\r\nUpgrade: h2c\r\n\r\n") + return + } + if atomic.AddInt64(&plainServed, 1) != 1 { + return // the cleartext application answers exactly once + } + _, _ = io.WriteString(conn, "HTTP/1.1 400 Bad Request\r\nConnection: close\r\n"+ + "Content-Length: 11\r\n\r\nBad Request") + }(conn) + } + }() + + var ( + mu sync.Mutex + results []Result + ) + options := &Options{ + Threads: 1, RateLimit: 10, Retries: 0, Timeout: 5, + Methods: http.MethodGet, Delay: -1, + HTTP2Probe: true, + InputTargetHost: []string{listener.Addr().String()}, + OnResult: func(r Result) { + if r.Err != nil || r.URL == "" { + return + } + mu.Lock() + results = append(results, r) + mu.Unlock() + }, + } + + runner, err := New(options) + require.NoError(t, err) + defer runner.Close() + runner.RunEnumeration() + + mu.Lock() + defer mu.Unlock() + + require.Len(t, results, 1, "the plaintext result must survive a failed HTTPS attempt") + require.Equal(t, "http", results[0].Scheme) + require.Equal(t, http.StatusBadRequest, results[0].StatusCode) + require.True(t, results[0].HTTP2, + "the h2c probe must use the restored plaintext URL") + require.EqualValues(t, 1, atomic.LoadInt64(&plainServed), + "the plaintext service must not be asked a second time") + require.EqualValues(t, 1, atomic.LoadInt64(&h2cServed), + "exactly one plaintext HTTP/2 probe must be observed") + require.EqualValues(t, 1, atomic.LoadInt64(&tlsHandshakes), + "the failed HTTPS request must complete its TLS handshake first") +}