diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..2c88175 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,18 @@ +# Keep credentials and local state out of the build context entirely: the +# image is built by CI and published, so anything reachable here can leak. +provider-config.yaml +provider-config.yml +.env +.env.* +# Anchored: an unanchored 'secrets/' also matches internal/secrets/. +/secrets/ +*.pem +*.key + +.git/ +.github/ +dist/ +bin/ +results/ +*.md +coverage.out diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..4052da9 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,114 @@ +name: CI + +on: + push: + branches: ["**"] + tags: ["v*"] + pull_request: + +permissions: + contents: read + +jobs: + test: + name: Lint and test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: go vet + run: go vet ./... + + - name: golangci-lint + uses: golangci/golangci-lint-action@v8 + with: + version: v2.12.2 + + - name: Test + run: go test -race -coverprofile=coverage.out ./... + + # The runtime image is distroless/static, which has no dynamic loader. A + # dependency that reaches libc through dlopen makes the binary + # dynamically linked, and it then fails at exec time with a message that + # says nothing about the cause. Catch it here instead. + - name: Binary must stay statically linked + run: | + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /tmp/fastrecon ./cmd/fastrecon + if ! file /tmp/fastrecon | grep -q "statically linked"; then + file /tmp/fastrecon + echo "::error::binary is not statically linked; the distroless static image cannot exec it" + exit 1 + fi + + secrets: + name: Secret scan + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + # The published image is built from this repository, so a credential + # committed here is a credential shipped. A hit fails the build. + # The scan covers the full history, and .gitleaks.toml carries the + # exemptions for known-fake test fixtures — one literal at a time, never + # a path-wide hole. + - name: gitleaks + run: | + docker run --rm -v "$PWD:/repo" zricethezav/gitleaks:latest \ + detect --source=/repo --no-banner --redact --verbose + + image: + name: Build image + runs-on: ubuntu-latest + needs: [test, secrets] + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v4 + + - uses: docker/setup-buildx-action@v3 + + - name: Metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + tags: | + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=sha,format=long + + # Pull requests build the image to prove the Dockerfile still works, + # but never publish it. + - name: Log in to GHCR + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + platforms: linux/amd64 + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + # Build args carry build identity only — never a credential, which + # would persist in the image history. + build-args: | + VERSION=${{ steps.meta.outputs.version }} + COMMIT=${{ github.sha }} + DATE=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.created'] }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..95cbe48 --- /dev/null +++ b/.gitignore @@ -0,0 +1,15 @@ +/dist/ +/bin/ +/results/ +# Anchored: an unanchored 'fastrecon' would also ignore cmd/fastrecon/. +/fastrecon +*.test +coverage.out + +# Never commit source credentials. +provider-config.yaml +provider-config.yml +.env +.env.* +# Anchored: an unanchored 'secrets/' also matches internal/secrets/. +/secrets/ diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 100644 index 0000000..9bbec28 --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1,24 @@ +title = "FastRecon" + +# Start from the shipped rules; this file only ever subtracts from them. +[extend] +useDefault = true + +# Known-fake values used as test fixtures. +# +# The scan covers the whole history, so annotating the current line would not +# clear a finding recorded against an earlier commit — the exemption has to +# live here. +# +# Entries are listed one literal at a time, on purpose. A path-wide exemption +# for test files would be a hole: a test file can hold a real credential just +# as easily as any other, and the point of this scan is that the published +# image is built from this repository. +[[allowlists]] +description = "Fixture credential in the redaction tests, which need a value shaped like a real key to be worth anything" +regexTargets = ["secret"] +regexes = [ + # internal/secrets/secrets_test.go — asserts the redactor scrubs a key out + # of a request URL, the way c99 sends one. + '''^abcdef1234567890$''', +] diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..5c8ea0b --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,24 @@ +version: "2" + +linters: + enable: + - bodyclose + - errorlint + - misspell + - revive + - unconvert + - usestdlibvars + settings: + revive: + rules: + - name: error-strings + - name: context-as-argument + exclusions: + rules: + # Table-driven tests read better without a comment on every helper. + - path: _test\.go + linters: [revive] + +formatters: + enable: + - gofmt diff --git a/Dockerfile b/Dockerfile index d6e5106..8af0e48 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,36 +1,45 @@ -# Build stage -FROM golang:alpine3.21 as builder - -RUN apk add make gcc g++ zlib zlib-dev git wget - -WORKDIR /app -COPY main.go . -RUN go build main.go - -RUN git clone https://github.com/blechschmidt/massdns && \ - cd massdns && \ - make - -RUN wget https://raw.githubusercontent.com/trickest/resolvers/main/resolvers.txt && \ - wget https://raw.githubusercontent.com/trickest/resolvers/main/resolvers-trusted.txt - -RUN go install -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@latest -RUN go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest -RUN go install github.com/d3mondev/puredns/v2@latest - -# Run stage -FROM alpine:latest - -# Create app directory -WORKDIR /app -COPY --from=builder /go/bin/subfinder /usr/local/bin/subfinder -COPY --from=builder /go/bin/httpx /usr/local/bin/httpx -COPY --from=builder /go/bin/puredns /usr/local/bin/puredns -COPY --from=builder /app/massdns/bin/massdns /usr/local/bin/massdns -COPY --from=builder /app/resolvers.txt /app/resolvers.txt -COPY --from=builder /app/resolvers-trusted.txt /app/resolvers-trusted.txt -COPY --from=builder /app/main /app/main -COPY subfinder.yaml . - -# Run the binary -CMD ["sh", "-c", "./main"] +# syntax=docker/dockerfile:1 + +# Build stage. Nothing here may take a credential: build args are recorded in +# the image history, and this image is built by CI and published. +# The builder runs natively on the build machine and cross-compiles, which is +# far faster than emulating the target platform. +FROM --platform=$BUILDPLATFORM golang:1.26-alpine AS build + +WORKDIR /src + +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . + +ARG VERSION=dev +ARG COMMIT="" +ARG DATE="" + +# Provided by buildx. The runtime stage below resolves to the same platform, +# so the image manifest and the binary inside it can never disagree. +ARG TARGETOS +ARG TARGETARCH + +# Static build: no libc at runtime, so the final image can be distroless. +RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH:-amd64} go build \ + -trimpath \ + -ldflags="-s -w \ + -X github.com/JoshuaMart/FastRecon/internal/version.Version=${VERSION} \ + -X github.com/JoshuaMart/FastRecon/internal/version.Commit=${COMMIT} \ + -X github.com/JoshuaMart/FastRecon/internal/version.Date=${DATE}" \ + -o /out/fastrecon ./cmd/fastrecon + +# Runtime stage: static distroless, non-root, no shell, no package manager. +# The binary needs CA certificates to reach the enumeration sources; the +# static image ships them. +FROM gcr.io/distroless/static-debian12:nonroot + +COPY --from=build /out/fastrecon /usr/local/bin/fastrecon + +USER nonroot:nonroot + +# Configuration arrives as environment variables and arguments, so the same +# image serves a local `docker run` and a serverless job definition. +ENTRYPOINT ["/usr/local/bin/fastrecon"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..6aa2528 --- /dev/null +++ b/Makefile @@ -0,0 +1,48 @@ +BINARY := fastrecon +PKG := github.com/JoshuaMart/FastRecon +VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) +COMMIT ?= $(shell git rev-parse HEAD 2>/dev/null) +DATE ?= $(shell date -u +%Y-%m-%dT%H:%M:%SZ) +LDFLAGS := -s -w \ + -X $(PKG)/internal/version.Version=$(VERSION) \ + -X $(PKG)/internal/version.Commit=$(COMMIT) \ + -X $(PKG)/internal/version.Date=$(DATE) + +.PHONY: build test lint fmt vet cover static docker clean + +build: + go build -trimpath -ldflags="$(LDFLAGS)" -o bin/$(BINARY) ./cmd/fastrecon + +test: + go test -race ./... + +cover: + go test -race -coverprofile=coverage.out ./... + go tool cover -func=coverage.out | tail -1 + +lint: + golangci-lint run ./... + +fmt: + gofmt -w . + +vet: + go vet ./... + +# The distroless runtime image has no dynamic loader, so a dependency that +# dlopens libc would produce a binary that cannot start in it. +static: + @CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /tmp/$(BINARY)-static ./cmd/fastrecon + @file /tmp/$(BINARY)-static | grep -q "statically linked" \ + && echo "static: ok" \ + || { file /tmp/$(BINARY)-static; echo "static: FAILED"; exit 1; } + +docker: + docker build \ + --build-arg VERSION=$(VERSION) \ + --build-arg COMMIT=$(COMMIT) \ + --build-arg DATE=$(DATE) \ + -t $(BINARY):$(VERSION) . + +clean: + rm -rf bin coverage.out diff --git a/README.md b/README.md index bd22c1f..b860711 100644 --- a/README.md +++ b/README.md @@ -1,91 +1,150 @@ -# ⚡FastRecon - -FastRecon is a fast and simple tool for discovering subdomains of a target domain. It is designed to be non-exhaustive and is not intended to be the most complete solution, but it is ideal for quickly identifying subdomains. +# FastRecon + +Attack-surface discovery for a domain: passive subdomain enumeration, exclusion +filtering, live/dead separation, port scanning, and HTTP probing — as a single +static binary that runs the same way locally, in Docker, in a serverless job, +and behind a serverless function. + +The design is specified in **[SPECIFICATIONS.md](SPECIFICATIONS.md)**. Read that +first; this file is only the quick start. + +## Status + +Phase 6 of 8. The whole pipeline runs, and its output reaches all three +sinks. + +- passive subdomain enumeration from multiple sources, with per-source + accounting in the report, +- exclusion patterns — exact, wildcard and regex — applied before any network + activity touches a host, +- DNS resolution splitting live from dead hosts, with per-parent wildcard + detection so a `*.example.com` record cannot flood the live set, +- resolver pools from a file or an https URL, health-checked before the run, +- unprivileged TCP connect port scanning, rate-limited, with CDN and WAF + determination so a narrowed port list is never mistaken for an exhaustive + one, +- HTTP probing of the discovered ports, HTTPS-first so the recorded scheme is + the one that actually worked, with titles, technologies and certificates, +- delivery to stdout, a file, and a webhook, with retries that distinguish + "not now" from "not like this", +- the CLI, with the full option surface and its precedence rules + (flag > environment > config file > default), +- the run report model and its `json` / `jsonl` / `text` renderings, +- the stdout and file sinks, +- pipeline orchestration: the stage ladder, deadline budgeting, truncation + handling, and exit codes, +- the container image and CI. + +What is left is the Scaleway job and function deployments (phases 7–8). Asking for a +wider scope walks the ladder as far as it can, then reports the stage that has +no implementation and exits 2 — it does not pretend to have found nothing. + +## Quick start + +```sh +make build +./bin/fastrecon -d example.com --stages enum + +# With exclusions, as text. +./bin/fastrecon -d example.com --stages enum --format text \ + --exclude '*.dev.example.com' --exclude 're:^staging[0-9]*\.' + +# Which sources exist, and which need a key. +./bin/fastrecon sources + +# Enumerate, then split live hosts from dead ones. +./bin/fastrecon -d example.com --stages resolve --format text + +# Bring your own resolver pool, from a file or a URL. +./bin/fastrecon -d example.com --stages resolve --resolvers-file ./resolvers.txt + +# Enumerate, resolve, then scan the web ports of the live hosts. +./bin/fastrecon -d example.com --stages ports --ports web --format text + +# The whole pipeline, ending with HTTP service detection. +./bin/fastrecon -d example.com --stages full --format text + +# POST the report to an internal API instead of writing it anywhere. +./bin/fastrecon -d example.com --output "" \ + --webhook-url https://internal.example.net/hooks/recon \ + --webhook-header "Authorization: Bearer $TOKEN" +``` - * Fast and efficient subdomain discovery - * Compatible with Go Serverless functions - * Uses popular open source tools such as [Subfinder](https://github.com/projectdiscovery/subfinder), [PureDNS](https://github.com/d3mondev/puredns), [MassDNS](https://github.com/blechschmidt/massdns) & [HTTPX](https://github.com/projectdiscovery/httpx) - * Returns results in JSON format for easy integration with other tools - * Supports raw output mode for simple domain lists +The default resolver pool is small and deliberate: Cloudflare, Google and +Quad9's unfiltered endpoints. Large public lists are supported but are the +wrong tool for this workload — see `SPECIFICATIONS.md` §8, which has the +measurements. -When used in a serverless function, the binaries must also be joined with the Go code. +```sh +# Same thing, containerised. +make docker +docker run --rm -e CHAOS_API_KEY fastrecon:dev -d example.com --stages enum +``` -> [!IMPORTANT] -> When Fastrecon is run in a Serverless function, the tool is not designed to be run on targets containing a large number of sub-domains (such as Google or Apple). +## Configuration -The Docker image produced is very light despite the many embedded binaries, making it perfect for Serverless use. +Every option is settable three ways, and the names are mechanically related: -![Docker image](https://zupimages.net/up/24/07/evjx.png) +| | | +|---|---| +| flag | `--scan-mode connect` | +| environment | `FASTRECON_SCAN_MODE=connect` | +| config file | `scan-mode: connect` | -## Build, launch the container (Go version) +Precedence is `flag > environment > config file > default`. Run +`fastrecon --help` for the full list, which is generated from the flag +definitions and is therefore always current. -> [!NOTE] -> Fill in the `subfinder.yaml` file first with your API keys for best results. +The scope of a run is one value: -``` -docker build . -t fastrecon -docker run -p 8080:8080 fastrecon +```sh +fastrecon -d example.com --stages enum # enumeration only +fastrecon -d example.com --stages resolve # + live/dead separation +fastrecon -d example.com --stages ports # + port scan +fastrecon -d example.com --stages full # + HTTP probe (default) ``` -## Usage +## Credentials -Make HTTP requests to `/?domain=[target_domain]` with optional parameters. +API keys are never baked into the image. Provide them at runtime, in this order +of precedence: -### Parameters +1. `FASTRECON_KEY_` — the namespaced environment variable, +2. `_API_KEY` — the upstream spelling (`CHAOS_API_KEY`, …), +3. a provider config file mounted into the container, pointed at with + `--provider-config` (subfinder/subfaster format), +4. `FASTRECON_KEY__FILE` — a path to a file holding the key, for + Docker and Kubernetes secret mounts. -- `domain` (required): The target domain to scan -- `raw` (optional): Set to `true` to return only the list of discovered subdomains without additional metadata +The default sources are `chaos`, `securitytrails`, `c99` (all key-required), +plus `submd` and `crt` which work without one. A run with no credentials at all +still returns data from the last two; the others are reported as +`skipped_no_key` rather than silently dropped. -### Examples +Credential values never appear in the logs or the report. Error messages are +scrubbed, including request URLs — some sources put the key in the query +string. -**Full scan with detailed JSON output:** -```bash -curl "http://localhost:8080/?domain=example.com" -``` +## Exit codes -**Raw output (domains only):** -```bash -curl "http://localhost:8080/?domain=example.com&raw=true" -``` +| Code | Meaning | +|---|---| +| 0 | run completed, report emitted | +| 1 | invalid configuration or usage | +| 2 | report emitted, but the run did not finish its scope | +| 3 | report produced, at least one destination failed | +| 4 | fatal error, no report produced | -### Output Formats - -#### Full JSON Output (default) -Returns a JSON array with detailed information about each subdomain: - -```json -[ - { - "url": "https://example.com", - "status_code": 200, - "content_length": 1234, - "content_type": "text/html", - "title": "Example Domain", - "a": ["93.184.216.34"], - "cname": null, - "cdn": false, - "tech": ["Apache HTTP Server:2.4.41"], - "header": { - "content_type": "text/html; charset=UTF-8", - "server": "Apache/2.4.41 (Ubuntu)" - } - } -] -``` - -#### Raw Output (raw=true) -Returns a simple list of discovered subdomains: +## Development +```sh +make test # go test -race ./... +make lint # golangci-lint +make cover # coverage summary +make static # assert the binary is still statically linked ``` -https://example.com -https://www.example.com -https://api.example.com -https://mail.example.com -``` - -## Performance - -Example of resources consumption in a Serverless Container with 560mVCPU & 512MB RAM: - * 220 seconds with a cold start for a recon on a domain with about 500 subdomains -![Resources Consumption](https://zupimages.net/up/24/07/7lsp.png) +`make static` is not optional busywork: the runtime image is +`distroless/static`, which has no dynamic loader. A dependency that reaches +libc through `dlopen` produces a binary that builds fine, passes every test, +and then fails at `exec` inside the container. CI runs the same check. diff --git a/SPECIFICATIONS.md b/SPECIFICATIONS.md new file mode 100644 index 0000000..0b29f7d --- /dev/null +++ b/SPECIFICATIONS.md @@ -0,0 +1,852 @@ +# FastRecon — Specifications + +> Status: draft v0.1 — design document, no implementation yet. + +## 1. Overview + +FastRecon is a single-binary attack-surface discovery tool. Given a root domain and an +exclusion list, it enumerates subdomains from multiple passive sources, filters out +excluded hosts, separates live hosts from dead ones, optionally scans ports, and probes +the discovered ports for HTTP services. + +It is designed to run identically in four environments: + +- as a local CLI binary, +- inside a Docker container, +- as a serverless job (primary target: Scaleway Serverless Jobs), +- as a serverless function (container-based, primary target: Scaleway Serverless Functions). + +The same artifact — one container image, one binary — serves all four. There is no separate +"serverless build". The binary exposes two entrypoints: the default one-shot CLI run, and a +`serve` subcommand that turns the same pipeline into an HTTP handler for the function +deployment. + +## 2. Goals + +- One static binary with no runtime dependency on external tools. This is the target, not a + hard constraint: shelling out to an external binary is acceptable where it buys something + real, as long as the default container image stays self-contained. +- Runs unprivileged: no root, no `CAP_NET_RAW`, no raw sockets required on the default path. +- Fully configurable via CLI flags **and** environment variables (serverless jobs configure + through env vars and args). +- Zero secrets baked into the container image — the image is built by CI and is public-safe. +- Selectable pipeline stages: enumeration only, enumeration + ports, or full. +- Machine-readable output (JSON) usable by downstream tooling. +- Stateless: every run is self-contained and produces a complete report. No database, no + cross-run diffing inside the tool. + +## 3. Stack decision + +**Go (1.23+).** + +Rationale — this is not a preference, it is what makes the rest of the spec cheap: + +- Every tool in the target pipeline is Go and is importable as a library: + `subfaster` (enumeration), `dnsx` (resolution), `naabu` (port scan), `httpx` (HTTP probe). + Using them as packages removes the need to ship and exec external binaries, which is the + main source of pain in serverless (no package manager, read-only or ephemeral filesystem, + binary size, PATH assumptions). +- Cross-compiles to a fully static binary → `FROM scratch`/distroless image, small pull, + fast job cold start. +- Native concurrency model fits a fan-out pipeline (thousands of DNS resolutions and TCP + connects). + +Consequence: the deliverable is a Go module exposing both a `cmd/fastrecon` CLI and an +internal pipeline package, so the pipeline can later be driven from an HTTP handler or a +queue consumer without restructuring. + +## 4. Execution environments + +| | Local CLI | Docker (local) | Serverless Job | Serverless Function | +|---|---|---|---|---| +| Invocation | `fastrecon -d example.com` | `docker run ... fastrecon -d ...` | job definition: image + args + env | HTTP request to `fastrecon serve` | +| Config source | flags, env, config file | flags, env, mounted config file | env vars + startup args | env vars + per-request JSON body | +| Secrets | env / config file / keychain | env / mounted file | job env vars | function env vars / Secret Manager | +| stdout JSON | read directly | read directly | captured as logs (Cockpit) — readable, but not machine-retrievable | logs only; the report goes in the HTTP response | +| Result delivery | stdout / file | stdout / mounted volume file | **webhook** (primary), stdout for inspection | **HTTP response** (primary), webhook optional | +| Privileges | can be root if the user wants | can add `--cap-add=NET_RAW` | assume unprivileged, no capabilities | assume unprivileged, no capabilities | +| Time budget | unbounded | unbounded | the job's configured timeout | the function's hard timeout — the tightest of the four | +| Realistic scope | any | any | any, up to the job timeout | `enum` / `resolve` on bounded scopes | + +### 4.1 Serverless constraints that shape the design + +- **No root, no raw sockets.** `masscan` and `naabu`'s SYN mode are unavailable. The default + port-scan mode is TCP **connect** scanning, which works unprivileged. SYN mode is an opt-in + flag that is only expected to work locally with `--cap-add=NET_RAW`; the tool must detect + the missing capability and fail with a clear message rather than silently returning zero ports. +- **stdout is logs, not a return value.** A serverless job's stdout goes to the platform log + pipeline. It is fine for humans and for debugging, and it is the default output because it + is the one thing that works everywhere — but a job that must hand results to another system + uses the webhook sink. This is why the sinks are independent and combinable. +- **Ephemeral, possibly read-only filesystem.** Nothing is written outside the configured + output path and `$TMPDIR`. No implicit writes to `$HOME` (relevant: subfinder/subfaster + default to `$HOME/.config/...` — FastRecon must pass provider config explicitly instead of + relying on that default). +- **Bounded run time.** The tool takes a global deadline (`--timeout`) and must return a + partial, well-formed report when it expires rather than being killed mid-flight. Each stage + gets its own budget derived from the global one. +- **No inbound network (job).** A job is push-only; nothing listens. A function is the + opposite — it exists to be called — which is why the HTTP entrypoint is a separate + subcommand rather than always-on behaviour. + +### 4.2 Serverless function specifics + +The function deployment is **container-based**: the same image, started with `serve` instead +of a one-shot run. This keeps a single artifact and a single code path — the handler builds a +run configuration and calls the same pipeline the CLI calls. + +- **The function's timeout is the binding constraint.** It is materially shorter than a job's, + and a full five-stage run over a large scope will not fit. The function is intended for + `enum` and `resolve` scopes; `ports` and `full` are permitted but the caller owns the risk. + The tool does not silently downgrade the requested stages — it runs what was asked and + returns a truncated report if the deadline hits. +- **The deadline is derived, not assumed.** On startup the handler resolves its budget from, + in order: an explicit `timeout` in the request body, `FASTRECON_TIMEOUT`, or a platform- + provided remaining-time value when one is available. It reserves a margin (default 10%) to + serialize and return the report, so the caller always receives a well-formed document + instead of a platform-level timeout error. +- **No work after the response.** On most FaaS platforms the instance is frozen or reclaimed + once the handler returns, so "respond 202 immediately, finish in the background, POST to a + webhook later" is not reliable. The function therefore runs **synchronously** and returns + the report in the response body. Work that cannot fit in a function timeout belongs in a + job — that is the split between the two deployments, and the documentation must say so + plainly rather than offering a fire-and-forget mode that intermittently loses runs. +- **Concurrency and reuse.** See §4.3 — the engine makes this the hardest constraint of the + deployment, and it is resolved by freezing how the engine is wired and serializing runs. +- **Authentication.** The handler requires a shared token (`--api-token` / + `FASTRECON_API_TOKEN`) compared in constant time, in addition to whatever the platform + provides. `serve` refuses to start without one: an unauthenticated subdomain-enumeration + endpoint is free reconnaissance for whoever finds it, charged to your API quotas. + +#### Request and response + +``` +POST /run +Authorization: Bearer + +{ + "domain": "example.com", + "exclude": ["*.dev.example.com", "re:^staging[0-9]*\\."], + "stages": "enum", + "ports": "top-100", + "timeout": "300s", + "webhook_url": "https://internal.example.net/hooks/recon" +} +``` + +The response body is the same report document defined in §13.2. `GET /healthz` returns +readiness for the platform's probe, without a token. Any field omitted from the body falls +back to the environment-variable configuration, so a function can be deployed fully +pre-configured and called with `{"domain": "..."}` alone. + +| Status | Meaning | +|---|---| +| `200` | report produced — **including a partial one**, which says so in `completed` and `truncated_by_timeout` | +| `400` | malformed body, or a configuration the run rejected | +| `401` | missing or wrong token | +| `429` | a run is already in progress on this instance; `Retry-After` set | +| `500` | a transient failure of ours: no report produced | + +The `400`/`500` split mirrors the exit codes exactly: a preparation failure marked transient +is ours, anything else is the caller's. Not every option can be checked before a stage is +built — a port expression is parsed by the scanner — so some caller mistakes surface once the +run starts and still return `400`, with the offending part named. + +An **unknown field in the body is rejected**, not ignored. Most of them name an option that +is deliberately not caller-settable, and silently dropping it would run a scan the caller did +not ask for while reporting success. + +### 4.3 What the request may not set, and why + +The request says **what to scan**, never how the deployment is wired. Credentials, the source +selection, the resolver pool and the webhook destination come from the function's +environment. + +This is not tidiness. The enumeration engine keeps API keys on **globally shared source +instances** and overwrites them on each configuration, and the per-source counters that become +the report's `sources` block live on those same instances. Two runs in one process would +overwrite each other's keys and report each other's statistics. Verified in the engine's +source, not assumed — and note that repetition is harmless there, since the keys are replaced +rather than appended; it is concurrency that breaks it. + +Two consequences, and one rejected option: + +- **Credentials and sources are settled once, at startup**, before any stage exists. A request + cannot supply them, so no request can disturb another's. +- **One run at a time per instance.** A second concurrent request is refused with `429` and a + `Retry-After` rather than queued: a queued request spends its own deadline waiting and then + reports a timeout that explains nothing. Deploy the function with a per-instance concurrency + of 1 and this never fires; it is a guard, not a mode. +- **No caller-supplied webhook URL.** The caller already receives the report in the response; + letting them name a destination would turn the function into a request-forwarding gadget + onto its own network, reachable by anyone holding the token. + +Rejected: forking the engine so keys live per run. It is a fork of forty-seven source +integrations to solve what freezing plus a mutex solves for nothing — and not maintaining +those integrations is the whole reason the engine is a dependency. + +## 5. Pipeline + +``` +input (domain + exclusions) + │ + ▼ + [1] ENUMERATE ──► raw subdomains (multi-source, deduplicated) + │ + ▼ + [2] EXCLUDE ────► in-scope subdomains + │ + ▼ + [3] RESOLVE ────► live hosts (with A/AAAA/CNAME) + dead hosts + │ + ▼ + [4] PORTSCAN ───► open ports per live host + │ + ▼ + [5] HTTP PROBE ─► HTTP(S) services with correct scheme, status, title, tech + │ + ▼ + report (JSON) ──► stdout | file | webhook +``` + +### 5.1 Stage selection + +Controlled by a single option, `--stages` / `FASTRECON_STAGES`: + +| Value | Stages run | Meaning | +|---|---|---| +| `enum` | 1, 2 | Subdomain enumeration only. | +| `resolve` | 1, 2, 3 | Enumeration + live/dead separation. | +| `ports` | 1, 2, 3, 4 | Enumeration + resolution + port scan. | +| `full` | 1–5 | Everything, including HTTP probing. (default) | + +Stages form a **strict ladder**: each depends on the one before it, and only the four values +above are accepted. Arbitrary combinations (`enumerate,httpprobe`, skipping the port scan) are +deliberately not supported — probing without a port scan would mean inventing a default port +set, which produces results that look like discovery but are really an assumption. Selecting +`enum` must not perform a single DNS query beyond what the passive sources do themselves. + +The report always states which stages ran, so a consumer can tell "no open ports found" +apart from "port scanning did not run". + +## 6. Stage 1 — Subdomain enumeration + +### 6.1 Engine + +Primary engine: **`subfaster` used as a Go library** (`github.com/melvinsh/subfaster`). +It is a fork of ProjectDiscovery's subfinder, so it keeps subfinder's provider-config +format while shipping fast keyless sources (`submd`, `crt`, `thc`, `rapiddns`, `hackertarget`, +`shodanct`, `sitedossier`) and the full keyed source set behind its "all sources" mode. + +Only the library's **passive agent** is used, not its CLI runner. The runner reads a provider +config from the user's home directory when none is given, and a container run must not depend +on what happens to be in `$HOME` — nor attempt that read on a read-only filesystem. Driving the +agent directly also removes the runner's resolver initialisation and update check, neither of +which this stage needs. + +The engine sits behind an interface: + +```go +type Enumerator interface { + Enumerate(ctx context.Context, domain string) (<-chan Subdomain, error) + Name() string +} +``` + +so that `subfinder` (library or binary) or a hand-written source can be substituted without +touching the rest of the pipeline. v1 ships the subfaster-backed implementation and no way to +select another: an option offering a choice of one would be a flag that does nothing. + +### 6.2 Required sources + +These must be supported and enabled by default when credentials are present: + +| Source | Engine name | Key | Environment variable | +|---|---|---|---| +| ProjectDiscovery Chaos | `chaos` | required | `CHAOS_API_KEY` | +| SecurityTrails | `securitytrails` | required | `SECURITYTRAILS_API_KEY` | +| c99.nl | `c99` | required | `C99_API_KEY` | +| sub.md | `submd` | optional | `SUBMD_API_KEY` | +| crt.name | `crt` | optional | `CRT_API_KEY` | + +The last two work without a credential, which is what makes a run with no keys at all still +return data; a key only improves their results. The three keyed sources report themselves as +`skipped_no_key` rather than being silently dropped from the selection. + +This set is the default selection. `--sources` replaces it, `--exclude-sources` subtracts from +it, and `--all-sources` queries everything the engine knows. `fastrecon sources` lists the +available names with their key requirement — an unknown name is rejected at startup rather +than silently ignored, because a misspelled source is a source that never ran. + +### 6.3 Behaviour + +- Source names are normalized to lower case before they reach the engine, whose lookup is + case-sensitive and which calls `os.Exit` on an empty selection. An unknown name is still + rejected outright. +- Sources run concurrently, each with its own timeout and its own error handling. +- **A failing source is a warning, not a fatal error** — a missing API key, a rate limit, or + a 5xx degrades the run instead of aborting it. Every source's status + (`ok` / `skipped_no_key` / `error` / `timeout`) plus the count it contributed is recorded + in the report, so a silently empty source is visible. +- Results are deduplicated, lowercased, and normalized (trailing dot stripped, IDNA/punycode + normalized, wildcard entries such as `*.example.com` dropped). +- Out-of-scope results (not equal to the root domain and not a subdomain of it) are dropped. + +### 6.4 Rate limits and time ceilings + +The intent is **bounded waiting, then abandonment of the source**. Failing fast on the first +429 discards a source that would have answered a couple of seconds later; waiting without a +ceiling lets one throttled source consume the whole run deadline. + +What is enforced today, through the engine: + +- **A time ceiling per source** — `--source-timeout` (default 30s), which bounds a source's + whole session, retries and backoff included. When it expires the source is abandoned. +- **A ceiling on the stage** — the enumeration's share of the run deadline, computed by the + pipeline. Sources run concurrently, so a throttled source never blocks the others; the + stage ends when every source has finished or when its budget runs out. +- **Results already collected are kept.** A paginated source that returned three pages of + five contributes those three; it is recorded with `partial: true` rather than discarded. +- **Throttling is distinguished from failure.** A source whose errors carry a 429, a rate-limit + message or a quota message is recorded as `rate_limited`, not `error` — "refused to answer" + and "had nothing to say" are different findings. + +What is **not** implemented: FastRecon does not schedule the retries itself. The engine owns +its HTTP layer and exposes no retry hook, so honouring `Retry-After` and applying proactive +client-side rate limiting per source would require a hand-written source layer. Until then the +timeout above is the ceiling, and the report says which sources hit it. + +## 7. Stage 2 — Exclusions + +Exclusions are supplied as a repeatable flag, a file, or an environment variable, and are +applied to the enumeration output before any network activity. + +Supported pattern forms: + +| Form | Example | Matches | +|---|---|---| +| Exact host | `admin.example.com` | that host only | +| Wildcard suffix | `*.dev.example.com` | any host under `dev.example.com` (and `dev.example.com` itself unless `--exclude-strict-wildcard`) | +| Regex | `re:^(staging\|preprod)[0-9]*\.` | hosts matching the RE2 pattern | + +Rules: + +- Matching is case-insensitive. +- The root domain itself can be excluded. +- Every exclusion decision is counted; the report contains the number of hosts removed and, + under `--report-excluded`, the list of removed hosts with the pattern that matched — needed + to debug an over-broad exclusion. +- An exclusion pattern that matches nothing is reported as a warning (typo detection). + +Sources of exclusions, merged together: + +- `--exclude ` (repeatable) +- `--exclude-file ` (one pattern per line, `#` comments allowed) +- `FASTRECON_EXCLUDE` — the serverless path + +The environment form splits on **lines** first. A line beginning with `re:` is one pattern, +kept whole; any other line is a comma-separated list of hosts and wildcards. Regexps have to +be exempt from comma splitting because a repeat count contains one — and cutting +`re:^a{1,3}\.example\.com$` yields two halves that **both still compile**, so the run would +quietly resolve, scan and probe hosts the operator had excluded. Splitting the compact host +form on commas is what keeps it usable in a job's environment variable. + +## 8. Stage 3 — Live/dead separation + +Resolution splits in-scope hosts into **live** and **dead**. + +- Engine: **`dnsx` as a Go library** — pure Go, unprivileged, does its own concurrency and + retries. No `massdns`/`puredns` binary is required in the image. +- Configurable: concurrency, extra attempts per query, per-query timeout. Records queried: + A, AAAA, CNAME. +- A resolver given without a port gets `:53`, so `1.1.1.1` and `1.1.1.1:53` both work. + +#### Resolver pool + +Three sources, merged and deduplicated: `--resolvers` (repeatable), `--resolvers-file` (one +per line, `#` comments), and `--resolvers-url` (fetched at startup over https, for the +serverless deployments which have no volume to mount a file from). Entries must be literal IP +addresses — a resolver given as a hostname would have to be resolved by some other resolver +first, a dependency this stage must not have. Unparseable entries are counted and sampled in +the log rather than silently skipped. + +The default is a small bundled set: Cloudflare, Google, and Quad9's **unfiltered** endpoints +(`9.9.9.10`, `149.112.112.10`, not `9.9.9.9`). Non-filtering matters more here than resolver +count: a filtering resolver returns a block-page address for a name it dislikes, and a +resolver that redirects NXDOMAIN turns every dead host into a live one — corrupting exactly +the live/dead split this stage exists to produce. + +**On large published resolver lists.** Lists like `trickest/resolvers` (~12,900 entries) exist +for brute-force workloads: millions of queries that need spreading across many resolvers. +That is not this stage's workload, which resolves the passive enumeration output — thousands +of names at most. Measured on the same 30 hosts, from one network: + +| Pool | Resolution time | +|---|---| +| bundled default (6) | 0.8s | +| `resolvers-trusted.txt`, 17 usable after validation | 23s | +| `resolvers.txt` (12,925), unvalidated | 49s | + +The large pool is slower and less reliable for this workload, not faster. It remains available +because it is the right tool if brute-force enumeration is ever added. + +#### Resolver health check + +`--validate-resolvers` (default on) probes each resolver twice before the run: + +- a **known name with a known answer**, so a resolver that lies is caught rather than merely + one that is unreachable, +- a **random name that must not exist**, catching NXDOMAIN hijacking. + +A published list is validated by whoever publishes it, from wherever their validator runs, +which says nothing about reachability from inside this job's network. Measured against +`resolvers-trusted.txt`: **14 of its 31 entries were unusable** from one network, independently +confirmed with `dig`. + +The pass is bounded by `--resolver-health-budget` (default 30s). Resolvers it did not reach are +**kept and counted** — dropping them would silently shrink the pool, and claiming they passed +would be a lie. Both the dropped count and the unchecked count reach the report as warnings. A +pool where every resolver fails is an error, not a run with no resolvers. + +The check verifies correctness, not speed. A resolver that answers correctly but slowly stays +in the pool, which is why the table above matters when choosing one. + +A host is **live** if it resolves to at least one address and is not a wildcard artifact. +Everything else is **dead**, with a reason (`nxdomain`, `no_answer`, `timeout`). Dead hosts +stay in the report — a dangling CNAME is a finding, not noise, so the alias target is kept +alongside the verdict. + +`nxdomain` and `no_answer` are deliberately distinct: a name that exists but has no address +(MX or TXT records only) is not the same as a name that does not exist. + +Hosts the stage never reached before its deadline are reported as `discovered`, not as dead. +Inventing a verdict for a host that was never queried would be worse than admitting the gap. + +### 8.1 Wildcard DNS detection + +**Mandatory, and per parent domain.** A domain resolving `*.example.com` to a single address +would otherwise flood the live set with junk, and a wildcard on `*.dev.example.com` is just as +capable of it as one on the apex — only the parent it sits on can reveal it. + +- Every parent domain appearing in the host list is probed, plus the root, with random names. +- A parent is a wildcard only when a **majority** of its probes answer, so one flaky lookup + cannot condemn a whole branch. The answers of all probes are unioned, which covers wildcards + that round-robin between addresses. +- A host is a wildcard artifact when **all** of its addresses belong to a parent's wildcard + set, or its CNAME target matches the wildcard's. A host answering with the wildcard address + *and* one of its own is a real host that shares infrastructure. +- Wildcard hosts are marked `wildcard`, excluded from the live set, and **kept in the report** + with their answers. + +Known limitation: a genuine host that resolves to exactly the wildcard's addresses is +indistinguishable from an artifact at the DNS level — `pages.github.io` behind `*.github.io` +is a real example. This is why such hosts are reported rather than dropped: the classification +is recoverable by whatever consumes the report, but deleted data is not. + +### 8.2 Not implemented + +An external resolver engine (`--resolver-engine=external` shelling out to `puredns`/`massdns` +for their throughput) is not built. dnsx covers the requirement in-process, and shelling out +would mean parsing another tool's output format and shipping its binary — neither of which the +default container image should carry. If throughput ever becomes the constraint, this is where +it gets revisited. + +## 9. Stage 4 — Port scanning + +- Engine: a **built-in TCP connect scanner**. Connect scanning requires no privileges, which + is what makes it work in a serverless job. Probes are ordered port-major — every address on + one port before moving to the next — so a single host is never hammered with the whole port + list back to back, and they are rate-limited by `--scan-rate`. +- A refusal is a definitive answer and is never retried; only timeouts and local file + descriptor exhaustion are, since those leave the port's state unknown. +- `--scan-mode=syn` is **refused**, not silently downgraded: a SYN scan the process cannot + perform finds no open ports at all, which reads exactly like a host with nothing listening. + +#### Why not naabu + +naabu was the intended engine and its API fits well — `OnResult` callback, stdout disabled, no +`$HOME` writes in library mode. It was implemented, and then removed after the container was +actually run rather than assumed to work: + +naabu reaches `sendmmsg` through `purego.Dlopen("libc.so.6")` to batch raw sends. On Linux +that path is unconditional, and `//go:cgo_import_dynamic` makes the binary **dynamically +linked even with `CGO_ENABLED=0`**. The `distroless/static` runtime image has no dynamic +loader, so the binary failed at exec with `no such file or directory` — a message that names +nothing about the cause. + +The trade was: keep naabu and give up the static image that every deployment here is built +on, or keep the static image and write the connect scanner. The only capability naabu offered +beyond it is SYN mode, which needs privileges the target environments do not grant. CI now +asserts the binary stays statically linked, because this failure is invisible until the image +is executed. +- Port selection (`--ports` / `FASTRECON_PORTS`): + - `top-100` (default), `top-1000`, `full`, `web` (a curated HTTP-oriented set), + - explicit lists and ranges: `80,443,8000-8100`, + - `--exclude-ports` to subtract. + A malformed expression is rejected at startup with the offending part named, rather than + inside the engine. +- Tunables: concurrency, rate limit, connect timeout, retries. Both limits are global to the + run and shared by the two scan passes — one limiter per pass would hand the full configured + rate to each, so `--scan-rate` would not describe the run. +- Probes are executed by a **fixed pool of `--scan-concurrency` workers** reading a stream of + targets. One goroutine per probe would allocate a stack for every address-port pair up + front: a full sweep of ten addresses measured 5.7 GB peak resident, against 65 MB for the + pool. In a memory-capped job that is an OOM kill before any report is delivered — worse + than the truncated report the budget design exists to guarantee. +- **Only live hosts are scanned.** A dead host has no address to connect to, and a wildcard + artifact is not a host — scanning either spends the budget proving something already known. +- Scanning targets resolved addresses, deduplicated: several subdomains pointing at the same + address are scanned once and the result is mapped back onto every host sharing it. Ports + found across a host's several addresses are merged and deduplicated. + +### 9.1 CDN and WAF determination + +Before a single port is touched, every target address is checked against the known CDN, WAF +and cloud-provider ranges. This is a **determination step, not a filter**: it runs on every +run, and its outcome is recorded whether or not it changes what gets scanned. + +The two halves are independent, and FastRecon does both: + +| Half | Role | Governed by | +|---|---|---| +| restriction | scan CDN/WAF addresses for ports 80 and 443 only | `--skip-cdn` (default on) | +| determination | record which provider was matched | always on | + +The determination uses `cdncheck`, the same library naabu uses for its own `-exclude-cdn`. +Running it before the scan is what lets the port list be decided per address: the full sweep +for origin addresses, ports 80 and 443 for the edges. + +- **Why restrict.** A CDN edge answers for thousands of unrelated customers. Its open ports + describe the provider's infrastructure, not the target's attack surface, so scanning the + full range burns the budget on results that mean nothing — and looks, from the provider's + side, like an attack on their edge. +- **Why record it regardless.** "Only 80 and 443 are open" is indistinguishable from a + genuinely minimal host unless the report states that the scan was deliberately narrowed. + Every host behind a CDN therefore carries the provider name and a `scan_limited` marker, so + no consumer mistakes a truncated port list for an exhaustive one. +- `--skip-cdn=false` scans CDN addresses in full. Detection still runs and the provider is + still recorded; only the restriction is lifted. +- Detection is recorded per provider, each entry naming the addresses it matched, so a host + with both a CDN address and an origin address shows which is which. The restriction applies + to the matched addresses only. + +## 10. Stage 5 — HTTP probing + +- Engine: **httpx's client**, used at the request level rather than through its CLI runner. + The runner calls `gologger.Fatal` — and therefore `os.Exit` — on several ordinary paths, + including "no input provided", and its enumeration entry point takes no context, so a run + could neither be cancelled nor survive a bad input. Technology detection uses + `wappalyzergo` directly. +- Probing targets the **discovered open ports only**. Assuming 80 and 443 would report + services that were never observed and miss the ones on unusual ports, which is the entire + reason the scan runs first. +- Collected per service: probed URL, scheme, status code, page title, content length, + redirect chain, final URL, response time, server header, detected technologies, and — for + TLS connections — subject CN, issuer, expiry and SANs. +- Tunables: concurrency, **rate limit**, request timeout, retries, follow-redirects toggle and + hop limit, custom User-Agent and headers. The sweep is rate-limited like the port scan: an + HTTP request costs a target far more than a TCP handshake, so if a ceiling belongs anywhere + it belongs here. Probes run on a fixed worker pool, for the same reason as the scan. +- **Redirects are not followed by default.** The `Location` target is recorded in `final_url` + either way, so following them buys the final page's title and status at the cost of a + request per hop to a host that may be out of scope entirely. `--probe-follow-redirects` + turns it on. +- TLS SANs discovered here may reveal additional hostnames; v1 **records** them in the report + but does not feed them back into the pipeline. (Candidate for v2: a re-enumeration loop.) + +### 10.1 Scheme detection + +**HTTPS is tried first on every port**, with plain HTTP as the fallback. There is no +port-number heuristic, because the TLS handshake is the only reliable discriminator. + +Trying HTTP first would misclassify TLS ports: a plain request to an HTTPS port commonly +returns a genuine HTTP `400 The plain HTTP request was sent to HTTPS port`, which is +indistinguishable from a working HTTP service. A handshake, by contrast, either succeeds or +fails. The cost is one fast-failing handshake on plain ports, which is worth one rule with no +exceptions to get wrong. + +Three consequences the report makes explicit: + +- **`url` always matches the probed scheme and port.** Where the redirects landed goes in + `final_url`. Overwriting `url` with the redirect target produced records reading + `"scheme": "http"` alongside `"url": "https://…"`, which is not a description of anything. +- **A certificate is recorded only for a connection that was itself TLS.** A plain-HTTP probe + that follows a redirect to an HTTPS host would otherwise attach that other endpoint's + certificate to this port. +- **A broken redirect chain still reports its first hop**, marked `redirect_unfollowed`. A + port answering `301` toward a host whose handshake fails is a finding; failing the whole + request would report the port as having no service at all. Observed in practice on a + Cloudflare DNS address: port 80 answers `301` to an HTTPS endpoint that refuses the + handshake. + +## 11. Configuration + +### 11.1 Precedence + +`CLI flag` > `environment variable` > `config file` > `built-in default`. + +Every option is settable through all three mechanisms. The env var name for an option is its +flag name uppercased with `-` → `_`, prefixed `FASTRECON_` (e.g. `--http-timeout` → +`FASTRECON_HTTP_TIMEOUT`). This mechanical mapping is what makes the serverless job +configurable without a config file. + +### 11.2 Config file + +Optional YAML, selected by `--config` / `FASTRECON_CONFIG`. Never read from an implicit +`$HOME` path unless `--config auto` is passed, so container runs stay deterministic. + +### 11.3 Core options (indicative) + +| Flag | Env | Default | Purpose | +|---|---|---|---| +| `-d, --domain` | `FASTRECON_DOMAIN` | — | root domain (required) | +| `--exclude`, `--exclude-file` | `FASTRECON_EXCLUDE` | — | exclusion patterns | +| `--stages` | `FASTRECON_STAGES` | `full` | pipeline scope | +| `--ports` | `FASTRECON_PORTS` | `top-100` | port selection | +| `--scan-mode` | `FASTRECON_SCAN_MODE` | `connect` | `connect` \| `syn` | +| `--output`, `-o` | `FASTRECON_OUTPUT` | `-` (stdout) | file sink path | +| `--format` | `FASTRECON_FORMAT` | `json` | `json` \| `jsonl` \| `text` | +| `--webhook-url` | `FASTRECON_WEBHOOK_URL` | — | webhook sink | +| `--timeout` | `FASTRECON_TIMEOUT` | `30m` | global deadline | +| `--concurrency` | `FASTRECON_CONCURRENCY` | tuned per stage | worker counts | +| `--log-level` | `FASTRECON_LOG_LEVEL` | `info` | stderr verbosity | +| `--provider-config` | `FASTRECON_PROVIDER_CONFIG` | — | source credentials file | +| `--sources` | `FASTRECON_SOURCES` | the five above | enumeration sources to query | +| `--exclude-sources` | `FASTRECON_EXCLUDE_SOURCES` | — | sources to subtract from the selection | +| `--all-sources` | `FASTRECON_ALL_SOURCES` | `false` | query every source the engine knows | +| `--source-timeout` | `FASTRECON_SOURCE_TIMEOUT` | `30s` | time ceiling for a single source (whole seconds) | +| `--probe-rate` | `FASTRECON_PROBE_RATE` | `200` | HTTP probes per second | +| `--resolvers` | `FASTRECON_RESOLVERS` | bundled set | DNS resolver IPs to use | +| `--resolvers-file` | `FASTRECON_RESOLVERS_FILE` | — | file of resolver IPs | +| `--resolvers-url` | `FASTRECON_RESOLVERS_URL` | — | https URL of a resolver list | +| `--validate-resolvers` | `FASTRECON_VALIDATE_RESOLVERS` | `true` | health-check the pool before the run | +| `--resolver-health-budget` | `FASTRECON_RESOLVER_HEALTH_BUDGET` | `30s` | ceiling on the health check | +| `--wildcard-probes` | `FASTRECON_WILDCARD_PROBES` | `3` | random names probed per parent domain | +| `--scan-retries` | `FASTRECON_SCAN_RETRIES` | `2` | retries per port | +| `--probe-retries` | `FASTRECON_PROBE_RETRIES` | `1` | retries per HTTP probe | +| `--listen` (serve) | `FASTRECON_LISTEN` | `:8080` | HTTP bind address in function mode | +| `--api-token` (serve) | `FASTRECON_API_TOKEN` | — | shared token required by the handler | + +## 12. Secrets and API keys + +Hard requirement: **the container image contains no credentials**, because CI builds and +publishes it. + +Accepted credential sources, in precedence order: + +1. **Environment variables** — the primary path for both Docker and Scaleway Jobs. + Two accepted spellings: the upstream names the enumeration engine already understands + (`CHAOS_API_KEY`, `SECURITYTRAILS_API_KEY`, `C99_API_KEY`, …) and a namespaced form + (`FASTRECON_KEY_CHAOS`, …). The namespaced form wins on conflict. +2. **A provider config file** mounted at runtime, path given by `--provider-config`. + Subfinder/subfaster-compatible YAML. Never copied into the image. +3. **A secret file per key** — `FASTRECON_KEY_CHAOS_FILE=/run/secrets/chaos`, for Docker + secrets and mounted-volume setups. + +Enforcement and hygiene: + +- `.dockerignore` excludes `*.yaml` provider configs, `.env*`, and any `secrets/` directory. +- The build must never accept credentials as `--build-arg` (they persist in image history). +- CI runs a secret scanner (gitleaks or equivalent) on the repository and fails the build on + a hit. +- The tool **redacts credentials in all output and logs**. Redaction happens where a value + enters a message — a source error carrying a request URL, for instance, since c99 puts the + key in the query string. The rendered report is then scrubbed once more before it leaves + the process, as a last line of defence over whatever was missed. +- Webhook header **values are never logged**; only their names are, since a webhook header is + exactly where a bearer token goes. +- Startup emits a credential inventory at `info` level: which sources have a key, which do + not. Values are never logged, not even truncated. + +## 13. Output + +### 13.1 Sinks + +Sinks are independent and can be combined in a single run: + +- **stdout** (default) — the full JSON document. When stdout is a sink, *all* logging goes to + stderr so stdout stays parseable. This is what makes `fastrecon ... | jq` work locally and + keeps the job's logs readable in Cockpit. +- **file** — `--output ./results/report.json`; parent directories created as needed. Written + atomically (temp file + rename) so a consumer never reads a half-written report. +- **webhook** — `--webhook-url`; an HTTP POST of the full report document as raw JSON + (`Content-Type: application/json`), unchanged from what the other sinks emit. The target is + an internal API, not a chat destination, so there is no message formatting and no summary + variant: one shape, defined in §13.2, for every consumer. Options: `--webhook-method` + (default `POST`), `--webhook-header` (repeatable, for auth), `--webhook-timeout`, and + `--webhook-retries`. A delivery failure sets a non-zero exit code but does not discard the + other sinks' output — the report must still reach stdout or the file sink. + + Retries cover 5xx, 429 and transport errors, with exponential backoff plus jitter so + several jobs retrying do not synchronise. A `Retry-After` header is honoured over the + computed wait — the endpoint knows better than the schedule — and the whole wait is capped + so a long chain cannot outlive its budget. **Every other 4xx is not retried**: the request + itself is wrong, most often the credentials or the URL, and repeating it only delays the + error. + + The response body is drained but never logged: a webhook target may echo the payload back, + and re-logging it would undo the redaction applied upstream. + +### 13.4 Delivery is detached from the run + +Delivery runs on its own context, derived from `context.WithoutCancel` and bounded by the +share of the budget reserved by `--output-margin`. + +A stopped job arrives as a signal that cancels the run's context. The entire point of +catching that signal is that the partial report still reaches its destinations — which +delivering on the cancelled context would prevent. The sinks therefore get a fresh deadline +of their own. + +The file sink writes atomically, except to destinations that are not regular files: +`/dev/stdout`, `/dev/null` and named pipes cannot be replaced by a rename, and there is +nothing to make atomic. Those are written through directly. + +Formats: `json` (single document, default), `jsonl` (one host per line, stream-friendly for +large scopes), `text` (human-readable summary). + +### 13.2 Report shape (indicative) + +```json +{ + "schema_version": "1.0", + "run": { + "id": "01J...", + "domain": "example.com", + "stages": ["enumerate", "exclude", "resolve", "portscan", "httpprobe"], + "started_at": "2026-01-01T00:00:00Z", + "finished_at": "2026-01-01T00:07:12Z", + "duration_ms": 432000, + "completed": true, + "truncated_by_timeout": false, + "version": "1.2.3", + "environment": "serverless-job" + }, + "sources": [ + {"name": "chaos", "status": "ok", "found": 812, "duration_ms": 1400}, + {"name": "securitytrails", "status": "skipped_no_key", "found": 0}, + {"name": "c99", "status": "error", "found": 0, "error": "rate limited"} + ], + "stats": { + "enumerated": 1204, "excluded": 37, "in_scope": 1167, + "live": 480, "dead": 671, "wildcard": 16, + "open_ports": 913, "http_services": 604 + }, + "hosts": [ + { + "host": "api.example.com", + "status": "live", + "addresses": ["93.184.216.34"], + "cname": ["edge.example.net"], + "cdn": [{"name": "cloudflare", "type": "waf", "addresses": ["93.184.216.34"], "scan_limited": true}], + "ports": [ + {"port": 443, "protocol": "tcp", "state": "open", + "http": { + "url": "https://api.example.com", + "scheme": "https", + "status_code": 200, + "title": "API", + "content_length": 1533, + "tech": ["nginx"], + "tls": {"subject_cn": "*.example.com", "issuer": "R3", + "not_after": "2026-06-01T00:00:00Z", + "sans": ["api.example.com", "www.example.com"]} + }} + ] + }, + {"host": "old.example.com", "status": "dead", "reason": "nxdomain"} + ], + "excluded": [{"host": "admin.example.com", "pattern": "admin.*"}], + "warnings": ["source c99 rate limited", "exclusion pattern '*.qa.example.com' matched nothing"] +} +``` + +`completed: false` plus `truncated_by_timeout: true` is how a deadline-truncated run is +reported — the report is still emitted and still valid. + +An operator stopping the job is reported differently: `completed: false` with +`truncated_by_timeout` left **false**. A consumer may reasonably retry a run that ran out of +time, and must not retry one somebody stopped on purpose. + +A host carries the status of the furthest stage that reached it. In an `enum` scope nothing is +resolved, so every surviving host is `discovered`: the enumeration result is data in its own +right and must appear in the report, not merely be counted in `stats`. + +### 13.3 Exit codes + +| Code | Meaning | +|---|---| +| 0 | run completed, report emitted | +| 1 | invalid configuration / usage | +| 2 | report emitted, but the run did not finish its scope — the deadline was reached, or a stage failed or is unavailable | +| 3 | run produced a report, but a sink failed (e.g. webhook delivery) | +| 4 | fatal error, no report produced — including a transient preparation failure such as an unreachable resolver list, or a health check that left no usable resolver | + +Distinct codes matter because a job scheduler's only signal is the exit status, and it keys +its retry on them. A network blip reported as `1` — invalid configuration — is something no +scheduler will ever retry. + +## 14. Logging and observability + +- All logs go to **stderr**, structured JSON by default (`--log-format=text` for humans), + keeping stdout clean for the report. +- Per-stage progress lines with counts and durations at `info`; per-host detail at `debug`. +- A final summary line with the same counters as `stats`, so the run outcome is visible in a + log-only environment (Cockpit) even when the report itself went to a webhook. +- No credential, no full request URL containing a key, ever appears in logs. + +## 15. Packaging and CI + +### 15.1 Container image + +- Multi-stage build: Go builder → `gcr.io/distroless/static` (or `scratch`) final stage. +- Static build (`CGO_ENABLED=0`), with CA certificates and `/etc/passwd` from distroless. +- Runs as a non-root user; no capabilities added; no privileged requirement. +- Entrypoint is the binary, so job args map straight onto CLI flags. +- `linux/amd64` only for now — it is the target job and function runtime. The build is set up + so a second architecture is a matrix entry, not a rewrite. +- Version, commit, and build date injected via `-ldflags`, surfaced by `fastrecon version` + and in the report's `run.version`. + +### 15.2 CI + +GitHub Actions, triggered on push, PR, and tag: + +1. lint (`golangci-lint`) + `go vet` +2. unit tests with race detector; integration tests behind a build tag (they need network) +3. secret scan (gitleaks) — build fails on any hit +4. build the `linux/amd64` image, push to the registry (GHCR and/or Scaleway Container Registry) +5. tags: `latest` on default branch, semver on git tags, plus the commit SHA +6. no secret is ever passed as a build arg; registry credentials come from CI secrets only + +## 16. Project layout (planned) + +``` +cmd/fastrecon/ CLI entrypoint, flag/env binding +internal/config/ config resolution + validation + precedence +internal/pipeline/ stage orchestration, deadline budgeting +internal/enumerate/ Enumerator interface + subfaster implementation +internal/exclude/ pattern parsing and matching +internal/resolve/ dnsx implementation, wildcard detection +internal/portscan/ naabu implementation, connect/syn modes +internal/probe/ httpx implementation +internal/report/ report model, JSON schema, formatters +internal/sink/ stdout, file, webhook +internal/secrets/ credential resolution + redaction +internal/httpapi/ `serve` handler for the serverless function deployment +deploy/scaleway/ job + function definition examples, CLI/Terraform snippets +Dockerfile +.github/workflows/ +``` + +## 17. Delivery phases + +1. **Skeleton** — module, CLI, config precedence, report model, stdout/file sinks, Dockerfile, CI. +2. **Enumeration + exclusions** — stage 1 and 2 end to end, `--stages enum` fully usable. +3. **Resolution** — stage 3 with wildcard detection. +4. **Port scan** — stage 4, connect mode, CDN handling. +5. **HTTP probe** — stage 5, `--stages full`. +6. **Webhook sink, hardening** — retries, redaction, timeout truncation, exit codes. +7. **Scaleway job deployment** — job definition, env-var configuration, documented run. +8. **Serverless function deployment** — `serve` handler, auth, derived deadline, documented + container-based function deployment. + diff --git a/cmd/fastrecon/main.go b/cmd/fastrecon/main.go new file mode 100644 index 0000000..e77a444 --- /dev/null +++ b/cmd/fastrecon/main.go @@ -0,0 +1,299 @@ +// Command fastrecon discovers the attack surface of a domain. +// +// The same binary serves every deployment: a local CLI run, a container, a +// serverless job, and — from the `serve` subcommand — a serverless function. +package main + +import ( + "context" + "errors" + "fmt" + "log/slog" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "github.com/spf13/pflag" + + "github.com/JoshuaMart/FastRecon/internal/app" + "github.com/JoshuaMart/FastRecon/internal/config" + "github.com/JoshuaMart/FastRecon/internal/enumerate" + "github.com/JoshuaMart/FastRecon/internal/logging" + "github.com/JoshuaMart/FastRecon/internal/serve" + "github.com/JoshuaMart/FastRecon/internal/sink" + "github.com/JoshuaMart/FastRecon/internal/version" +) + +// Exit codes. A job scheduler's only signal is the exit status, so an +// incomplete run, a failed delivery and a fatal error must not look alike. +const ( + exitOK = 0 // run completed, report emitted + exitUsage = 1 // invalid configuration or usage + exitIncomplete = 2 // report emitted, but the run did not finish its scope + exitSinkFailed = 3 // report produced, at least one destination failed + exitFatal = 4 // no report produced +) + +func main() { + os.Exit(dispatch(os.Args[1:])) +} + +func dispatch(args []string) int { + if len(args) > 0 { + switch args[0] { + case "version": + fmt.Println("fastrecon", version.String()) + return exitOK + case "serve": + return serveMode(args[1:]) + case "run": + args = args[1:] + case "sources": + listSources() + return exitOK + case "help": + usage(newFlagSet()) + return exitOK + } + } + return run(args) +} + +// serveMode answers run requests over HTTP, for the function deployment. +func serveMode(args []string) int { + fs := newFlagSet() + if err := fs.Parse(args); err != nil { + if errors.Is(err, pflag.ErrHelp) { + return exitOK + } + fmt.Fprintf(os.Stderr, "fastrecon: %v\n", err) + return exitUsage + } + + // The domain is not known at startup here: it arrives with each request. + cfg, err := config.LoadServe(fs) + if err != nil { + fmt.Fprintf(os.Stderr, "fastrecon: invalid configuration:\n%v\n", err) + return exitUsage + } + // Labelled explicitly rather than guessed: nothing inside the process + // distinguishes a function from any other container. + if cfg.Environment == "" { + cfg.Environment = config.EnvServerlessFunction + } + + log, err := logging.New(cfg.LogLevel, cfg.LogFormat) + if err != nil { + fmt.Fprintf(os.Stderr, "fastrecon: %v\n", err) + return exitUsage + } + for _, w := range cfg.Warnings { + log.Warn("configuration", "warning", w) + } + + application, err := app.New(cfg, log) + if err != nil { + fmt.Fprintf(os.Stderr, "fastrecon: %v\n", err) + return exitUsage + } + server, err := serve.New(application, cfg, log) + if err != nil { + fmt.Fprintf(os.Stderr, "fastrecon: %v\n", err) + return exitUsage + } + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + if err := server.ListenAndServe(ctx, cfg.Listen); err != nil { + log.Error("serve failed", "error", err) + return exitFatal + } + return exitOK +} + +func newFlagSet() *pflag.FlagSet { + fs := pflag.NewFlagSet("fastrecon", pflag.ContinueOnError) + fs.SortFlags = false + fs.Usage = func() { usage(fs) } + config.RegisterFlags(fs) + return fs +} + +func usage(fs *pflag.FlagSet) { + fmt.Fprintf(os.Stderr, `fastrecon %s — attack-surface discovery + +Usage: + fastrecon [run] -d [options] + fastrecon serve [options] + fastrecon sources + fastrecon version + +Every option below is also settable as an environment variable (--http-timeout +becomes FASTRECON_HTTP_TIMEOUT) or as a config-file key of the same name. +Precedence: flag > environment > config file > default. + +Options: +%s`, version.Version, fs.FlagUsages()) +} + +func run(args []string) int { + fs := newFlagSet() + if err := fs.Parse(args); err != nil { + if errors.Is(err, pflag.ErrHelp) { + return exitOK + } + fmt.Fprintf(os.Stderr, "fastrecon: %v\n", err) + return exitUsage + } + if extra := fs.Args(); len(extra) > 0 { + fmt.Fprintf(os.Stderr, "fastrecon: unexpected argument %q\n", extra[0]) + return exitUsage + } + + cfg, err := config.Load(fs) + if err != nil { + // Configuration errors are reported all at once, one per line. + fmt.Fprintf(os.Stderr, "fastrecon: invalid configuration:\n%v\n", err) + return exitUsage + } + + log, err := logging.New(cfg.LogLevel, cfg.LogFormat) + if err != nil { + fmt.Fprintf(os.Stderr, "fastrecon: %v\n", err) + return exitUsage + } + for _, w := range cfg.Warnings { + log.Warn("configuration", "warning", w) + } + if cfg.ConfigFile != "" { + log.Debug("config file loaded", "path", cfg.ConfigFile) + } + + // A stopped job or a container shutdown arrives as a signal. Cancelling + // the run rather than dying on the spot means the partial report still + // reaches its destinations. It is created before the stages are built so + // that resolver loading and health checking are cancellable too. + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + application, err := app.New(cfg, log) + if err != nil { + fmt.Fprintf(os.Stderr, "fastrecon: %v\n", err) + return exitUsage + } + + rep, err := application.Run(ctx, cfg) + if err != nil { + fmt.Fprintf(os.Stderr, "fastrecon: %v\n", err) + // A scheduler keys its retry on the exit status. Reporting a network + // blip as a configuration error means it never retries something that + // would have worked on the next run. + if errors.Is(err, app.ErrRuntime) { + return exitFatal + } + // Everything else that stops a run before it starts is a mistake in + // the configuration: an unparseable exclusion, an impossible scan + // mode, a port list that is not one. + return exitUsage + } + + data, err := rep.Render(cfg.Format) + if err != nil { + log.Error("render report", "error", err) + return exitFatal + } + data = application.Redactor().RedactBytes(data) + + sinks, err := buildSinks(cfg, log) + if err != nil { + fmt.Fprintf(os.Stderr, "fastrecon: %v\n", err) + return exitUsage + } + + // Delivery runs on its own context, detached from the run's. A stopped + // job arrives as a signal that cancels ctx, and the whole point of + // catching it is that the partial report still reaches its destinations — + // which a cancelled context would prevent. + deliverCtx, cancelDelivery := deliveryContext(cfg) + defer cancelDelivery() + + results := sink.DeliverAll(deliverCtx, data, sinks) + for _, r := range results { + if r.Err != nil { + log.Error("delivery failed", "sink", r.Sink, "error", r.Err) + continue + } + log.Info("report delivered", "sink", r.Sink) + } + + switch { + case sink.Errs(results) != nil: + return exitSinkFailed + case !rep.Run.Completed: + return exitIncomplete + default: + return exitOK + } +} + +// deliveryContext bounds the report delivery, using the slice of the run +// budget that was reserved for exactly this. +func deliveryContext(cfg *config.Config) (context.Context, context.CancelFunc) { + budget := time.Duration(float64(cfg.Timeout) * cfg.OutputMargin) + if budget <= 0 { + budget = 30 * time.Second + } + return context.WithTimeout(context.WithoutCancel(context.Background()), budget) +} + +func listSources() { + fmt.Printf("%-18s %-10s %s\n", "SOURCE", "KEY", "DEFAULT") + for _, s := range enumerate.Available() { + def := "" + if s.Default { + def = "yes" + } + fmt.Printf("%-18s %-10s %s\n", s.Name, s.Key, def) + } +} + +func buildSinks(cfg *config.Config, log *slog.Logger) ([]sink.Sink, error) { + stdout, file, webhook := cfg.Sinks() + var sinks []sink.Sink + if stdout { + sinks = append(sinks, sink.NewStdout()) + } + if file { + sinks = append(sinks, sink.NewFile(cfg.Output)) + } + if webhook { + w, err := sink.NewWebhook(sink.WebhookOptions{ + URL: cfg.WebhookURL, + Method: cfg.WebhookMethod, + Headers: cfg.WebhookHeaders, + Timeout: cfg.WebhookTimeout, + Retries: cfg.WebhookRetries, + Logger: log, + }) + if err != nil { + return nil, err + } + sinks = append(sinks, w) + // Header names only: a webhook header is where the bearer token goes. + log.Debug("webhook configured", "url", cfg.WebhookURL, "method", cfg.WebhookMethod, "headers", headerNames(cfg.WebhookHeaders)) + } + return sinks, nil +} + +// headerNames lists the configured header names without their values. +func headerNames(headers []string) []string { + out := make([]string, 0, len(headers)) + for _, h := range headers { + name, _, _ := strings.Cut(h, ":") + out = append(out, strings.TrimSpace(name)) + } + return out +} diff --git a/deploy/scaleway/README.md b/deploy/scaleway/README.md new file mode 100644 index 0000000..5789e73 --- /dev/null +++ b/deploy/scaleway/README.md @@ -0,0 +1,111 @@ +# Scaleway deployment + +Both deployments run the **same image** published by CI. Nothing here is +FastRecon-specific configuration baked into an artifact — it is all job or +function definition. + +## Serverless Job + +A job is the right target for a full run: its timeout is generous enough for +the whole ladder, and it needs no inbound network. + +```sh +IMAGE=ghcr.io/joshuamart/fastrecon:latest + +scw jobs definition create \ + name=fastrecon \ + image-uri="$IMAGE" \ + cpu-limit=1000 \ + memory-limit=2048 \ + timeout=30m \ + command="-d example.com --stages full --output /dev/stdout" \ + environment-variables.FASTRECON_ENVIRONMENT=serverless-job \ + environment-variables.FASTRECON_EXCLUDE='*.dev.example.com,admin.example.com' \ + environment-variables.FASTRECON_LOG_FORMAT=json +``` + +Credentials are set as job environment variables, ideally backed by Secret +Manager, never as build arguments: + +```sh +scw jobs definition update \ + environment-variables.CHAOS_API_KEY=... \ + environment-variables.SECURITYTRAILS_API_KEY=... \ + environment-variables.C99_API_KEY=... +``` + +### Getting the results out + +The job's stdout is collected as logs (Cockpit). That is fine for reading a run +and for `--format text`, but it is a log stream, not an artifact another system +can fetch. A job that has to hand its results to something else posts them: + +```sh +scw jobs definition update \ + environment-variables.FASTRECON_OUTPUT="" \ + environment-variables.FASTRECON_WEBHOOK_URL=https://internal.example.net/hooks/recon \ + environment-variables.FASTRECON_WEBHOOK_HEADER='Authorization: Bearer ...' +``` + +`FASTRECON_OUTPUT=""` turns off the stdout sink so the report exists only where +it was sent. Leave it set to `-` to keep a copy in the logs as well. + +Stopping a job sends SIGTERM. FastRecon cancels the run, marks the report +incomplete, and still delivers it: delivery runs on its own deadline, taken +from the share of the budget reserved by `--output-margin`. + +### Scheduling + +```sh +scw jobs definition update cron.schedule="0 3 * * *" cron.timezone=Europe/Paris +``` + +## Serverless Function + +The function deployment is container-based: the same image, started with the +`serve` subcommand. It exists for short, bounded runs — `--stages enum` or +`resolve` on a known scope. Anything that needs the full ladder belongs in a +job, because a function's timeout is the tightest of the four environments and +work does not survive the response being returned. + +```sh +scw function create \ + name=fastrecon \ + runtime=docker \ + registry-image="$IMAGE" \ + memory-limit=2048 \ + timeout=900s \ + environment-variables.FASTRECON_ENVIRONMENT=serverless-function \ + environment-variables.FASTRECON_SOURCES=chaos,securitytrails,c99,submd,crt \ + secret-environment-variables.FASTRECON_API_TOKEN=... +``` + +The container is started with the `serve` argument. Deploy it with a +**per-instance concurrency of 1**: the enumeration engine keeps per-run state on +globally shared instances, so a second concurrent request on one instance is +refused with `429` rather than corrupting both runs. Scale by adding instances, +not by sharing one. + +Calling it: + +```sh +curl -X POST "$FUNCTION_URL/run" \ + -H "Authorization: Bearer $FASTRECON_API_TOKEN" \ + -d '{"domain":"example.com","stages":"enum"}' +``` + +The request says what to scan, never how the function is wired: credentials, +sources, resolvers and any webhook destination come from the function's +environment. An unknown field in the body is rejected rather than ignored, so a +caller who believes they configured something is told they did not. + +`fastrecon serve` refuses to start without `FASTRECON_API_TOKEN`. + +## Notes + +- Both deployments run unprivileged. The port scan therefore uses connect mode; + `--scan-mode syn` needs `CAP_NET_RAW` and will refuse to run here rather than + silently reporting zero open ports. +- Set `FASTRECON_ENVIRONMENT` explicitly. A serverless job and a plain + container are indistinguishable from inside the process, so the label in the + report comes from the deployment, not from a guess. diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..20bea7a --- /dev/null +++ b/go.mod @@ -0,0 +1,164 @@ +module github.com/JoshuaMart/FastRecon + +go 1.26 + +require ( + github.com/projectdiscovery/cdncheck v1.2.42 + github.com/projectdiscovery/dnsx v1.2.3 + github.com/projectdiscovery/gologger v1.1.71 + github.com/spf13/pflag v1.0.10 + golang.org/x/net v0.56.0 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + github.com/PuerkitoBio/goquery v1.12.0 // indirect + github.com/andybalholm/cascadia v1.3.3 // indirect + github.com/brianvoe/gofakeit/v7 v7.2.1 // indirect + github.com/cloudflare/cfssl v1.6.4 // indirect + github.com/cnf/structhash v0.0.0-20250313080605-df4c6cc74a9a // indirect + github.com/corona10/goimagehash v1.1.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-faker/faker/v4 v4.9.0 // indirect + github.com/go-rod/rod v0.116.2 // indirect + github.com/go-viper/mapstructure/v2 v2.5.0 // indirect + github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 // indirect + github.com/google/certificate-transparency-go v1.3.2 // indirect + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect + github.com/gosimple/slug v1.15.0 // indirect + github.com/gosimple/unidecode v1.0.1 // indirect + github.com/happyhackingspace/dit v0.0.28 // indirect + github.com/hashicorp/go-version v1.8.0 // indirect + github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf // indirect + github.com/hdm/jarm-go v0.0.8 // indirect + github.com/iangcarroll/cookiemonster v1.6.0 // indirect + github.com/kataras/jwt v0.1.10 // indirect + github.com/logrusorgru/aurora v2.0.3+incompatible // indirect + github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 // indirect + github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect + github.com/projectdiscovery/asnmap v1.1.1 // indirect + github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30f7193 // indirect + github.com/projectdiscovery/clistats v0.1.4 // indirect + github.com/projectdiscovery/dsl v0.8.20 // indirect + github.com/projectdiscovery/fdmax v0.0.4 // indirect + github.com/projectdiscovery/freeport v0.0.7 // indirect + github.com/projectdiscovery/goconfig v0.0.1 // indirect + github.com/projectdiscovery/goflags v0.1.74 // indirect + github.com/projectdiscovery/gostruct v0.0.2 // indirect + github.com/projectdiscovery/govaluate v0.0.0-20260504230327-80320480bb6e // indirect + github.com/projectdiscovery/httpx v1.10.0 + github.com/projectdiscovery/mapcidr v1.1.97 // indirect + github.com/projectdiscovery/rawhttp v0.1.90 // indirect + github.com/projectdiscovery/tlsx v1.2.2 // indirect + github.com/projectdiscovery/useragent v0.0.108 // indirect + github.com/projectdiscovery/wappalyzergo v0.2.87 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/sashabaranov/go-openai v1.37.0 // indirect + github.com/seh-msft/burpxml v1.0.1 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/spf13/cast v1.10.0 // indirect + github.com/vulncheck-oss/go-exploit v1.51.0 // indirect + github.com/xdg-go/pbkdf2 v1.0.0 // indirect + github.com/ysmood/fetchup v0.2.3 // indirect + github.com/ysmood/goob v0.4.0 // indirect + github.com/ysmood/got v0.40.0 // indirect + github.com/ysmood/gson v0.7.3 // indirect + github.com/ysmood/leakless v0.9.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect +) + +require ( + aead.dev/minisign v0.2.0 // indirect + github.com/Masterminds/semver/v3 v3.4.0 // indirect + github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 // indirect + github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect + github.com/VividCortex/ewma v1.2.0 // indirect + github.com/akrylysov/pogreb v0.10.2 // indirect + github.com/alecthomas/chroma/v2 v2.14.0 // indirect + github.com/andybalholm/brotli v1.2.0 // indirect + github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect + github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/aymerick/douceur v0.2.0 // indirect + github.com/charmbracelet/glamour v0.8.0 // indirect + github.com/charmbracelet/lipgloss v0.13.0 // indirect + github.com/charmbracelet/x/ansi v0.3.2 // indirect + github.com/cheggaaa/pb/v3 v3.1.6 // indirect + github.com/dimchansky/utfbom v1.1.1 // indirect + github.com/djherbis/times v1.6.0 // indirect + github.com/dlclark/regexp2 v1.11.5 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect + github.com/ebitengine/purego v0.10.0 // indirect + github.com/fatih/color v1.18.0 // indirect + github.com/gaissmai/bart v0.28.0 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/google/go-github/v30 v30.1.0 // indirect + github.com/google/go-querystring v1.1.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/gorilla/css v1.0.1 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.18.2 // indirect + github.com/logrusorgru/aurora/v4 v4.0.0 // indirect + github.com/lucasb-eyer/go-colorful v1.3.0 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.16 // indirect + github.com/melvinsh/subfaster/v2 v2.18.0 + github.com/microcosm-cc/bluemonday v1.0.27 // indirect + github.com/miekg/dns v1.1.68 + github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/muesli/reflow v0.3.0 // indirect + github.com/muesli/termenv v0.16.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect + github.com/projectdiscovery/blackrock v0.0.1 // indirect + github.com/projectdiscovery/chaos-client v0.5.2 // indirect + github.com/projectdiscovery/fastdialer v0.5.11 // indirect + github.com/projectdiscovery/hmap v0.0.101 // indirect + github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582 // indirect + github.com/projectdiscovery/networkpolicy v0.1.41 // indirect + github.com/projectdiscovery/ratelimit v0.0.88 // indirect + github.com/projectdiscovery/retryabledns v1.0.115 + github.com/projectdiscovery/retryablehttp-go v1.3.16 // indirect + github.com/projectdiscovery/utils v0.11.1 // indirect + github.com/refraction-networking/utls v1.8.2 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/rogpeppe/go-internal v1.16.0 // indirect + github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect + github.com/shirou/gopsutil/v4 v4.26.3 // indirect + github.com/syndtr/goleveldb v1.0.0 // indirect + github.com/tidwall/btree v1.7.0 // indirect + github.com/tidwall/buntdb v1.3.1 // indirect + github.com/tidwall/gjson v1.18.0 // indirect + github.com/tidwall/grect v0.1.4 // indirect + github.com/tidwall/match v1.2.0 // indirect + github.com/tidwall/pretty v1.2.1 // indirect + github.com/tidwall/rtred v0.1.2 // indirect + github.com/tidwall/tinyqueue v0.1.1 // indirect + github.com/tklauser/go-sysconf v0.3.16 // indirect + github.com/tklauser/numcpus v0.11.0 // indirect + github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 // indirect + github.com/weppos/publicsuffix-go v0.50.3 // indirect + github.com/yuin/goldmark v1.7.13 // indirect + github.com/yuin/goldmark-emoji v1.0.3 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + github.com/zcalusic/sysinfo v1.0.2 // indirect + github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect + github.com/zmap/zcrypto v0.0.0-20240803002437-3a861682ac77 // indirect + go.etcd.io/bbolt v1.4.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/crypto v0.53.0 // indirect + golang.org/x/exp v0.0.0-20250911091902-df9299821621 // indirect + golang.org/x/mod v0.36.0 // indirect + golang.org/x/oauth2 v0.34.0 // indirect + golang.org/x/sync v0.21.0 // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/term v0.44.0 // indirect + golang.org/x/text v0.38.0 // indirect + golang.org/x/time v0.14.0 // indirect + golang.org/x/tools v0.45.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..854c503 --- /dev/null +++ b/go.sum @@ -0,0 +1,547 @@ +aead.dev/minisign v0.2.0 h1:kAWrq/hBRu4AARY6AlciO83xhNnW9UaC8YipS2uhLPk= +aead.dev/minisign v0.2.0/go.mod h1:zdq6LdSd9TbuSxchxwhpA9zEb9YXcVGoE8JakuiGaIQ= +cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 h1:KFac3SiGbId8ub47e7kd2PLZeACxc1LkiiNoDOFRClE= +github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057/go.mod h1:iLB2pivrPICvLOuROKmlqURtFIEsoJZaMidQfCG1+D4= +github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 h1:ZbFL+BDfBqegi+/Ssh7im5+aQfBRx6it+kHnC7jaDU8= +github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809/go.mod h1:upgc3Zs45jBDnBT4tVRgRcgm26ABpaP7MoTSdgysca4= +github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g= +github.com/PuerkitoBio/goquery v1.12.0 h1:pAcL4g3WRXekcB9AU/y1mbKez2dbY2AajVhtkO8RIBo= +github.com/PuerkitoBio/goquery v1.12.0/go.mod h1:802ej+gV2y7bbIhOIoPY5sT183ZW0YFofScC4q/hIpQ= +github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= +github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= +github.com/akrylysov/pogreb v0.10.2 h1:e6PxmeyEhWyi2AKOBIJzAEi4HkiC+lKyCocRGlnDi78= +github.com/akrylysov/pogreb v0.10.2/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= +github.com/alecthomas/assert/v2 v2.7.0 h1:QtqSACNS3tF7oasA8CU6A6sXZSBDqnm7RfpLl9bZqbE= +github.com/alecthomas/assert/v2 v2.7.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= +github.com/alecthomas/chroma/v2 v2.14.0 h1:R3+wzpnUArGcQz7fCETQBzO5n9IMNi13iIs46aU4V9E= +github.com/alecthomas/chroma/v2 v2.14.0/go.mod h1:QolEbTfmUHIMVpBqxeDnNBj2uoeI4EbYP4i6n68SG4I= +github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= +github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= +github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= +github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= +github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= +github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= +github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8= +github.com/aymanbagabas/go-udiff v0.2.0/go.mod h1:RE4Ex0qsGkTAJoQdQQCA0uG+nAzJO/pI/QwceO5fgrA= +github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= +github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= +github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE= +github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bits-and-blooms/bloom/v3 v3.5.0 h1:AKDvi1V3xJCmSR6QhcBfHbCN4Vf8FfxeWkMNQfmAGhY= +github.com/bits-and-blooms/bloom/v3 v3.5.0/go.mod h1:Y8vrn7nk1tPIlmLtW2ZPV+W7StdVMor6bC1xgpjMZFs= +github.com/brianvoe/gofakeit/v7 v7.2.1 h1:AGojgaaCdgq4Adzrd2uWdbGNDyX6MWNhHdQBraNfOHI= +github.com/brianvoe/gofakeit/v7 v7.2.1/go.mod h1:QXuPeBw164PJCzCUZVmgpgHJ3Llj49jSLVkKPMtxtxA= +github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/charmbracelet/glamour v0.8.0 h1:tPrjL3aRcQbn++7t18wOpgLyl8wrOHUEDS7IZ68QtZs= +github.com/charmbracelet/glamour v0.8.0/go.mod h1:ViRgmKkf3u5S7uakt2czJ272WSg2ZenlYEZXT2x7Bjw= +github.com/charmbracelet/lipgloss v0.13.0 h1:4X3PPeoWEDCMvzDvGmTajSyYPcZM4+y8sCA/SsA3cjw= +github.com/charmbracelet/lipgloss v0.13.0/go.mod h1:nw4zy0SBX/F/eAO1cWdcvy6qnkDUxr8Lw7dvFrAIbbY= +github.com/charmbracelet/x/ansi v0.3.2 h1:wsEwgAN+C9U06l9dCVMX0/L3x7ptvY1qmjMwyfE6USY= +github.com/charmbracelet/x/ansi v0.3.2/go.mod h1:dk73KoMTT5AX5BsX0KrqhsTqAnhZZoCBjs7dGWp4Ktw= +github.com/charmbracelet/x/exp/golden v0.0.0-20240806155701-69247e0abc2a h1:G99klV19u0QnhiizODirwVksQB91TJKV/UaTnACcG30= +github.com/charmbracelet/x/exp/golden v0.0.0-20240806155701-69247e0abc2a/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U= +github.com/cheggaaa/pb/v3 v3.1.6 h1:h0x+vd7EiUohAJ29DJtJy+SNAc55t/elW3jCD086EXk= +github.com/cheggaaa/pb/v3 v3.1.6/go.mod h1:urxmfVtaxT+9aWk92DbsvXFZtNSWQSO5TRAp+MJ3l1s= +github.com/cloudflare/cfssl v1.6.4 h1:NMOvfrEjFfC63K3SGXgAnFdsgkmiq4kATme5BfcqrO8= +github.com/cloudflare/cfssl v1.6.4/go.mod h1:8b3CQMxfWPAeom3zBnGJ6sd+G1NkL5TXqmDXacb+1J0= +github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= +github.com/cnf/structhash v0.0.0-20250313080605-df4c6cc74a9a h1:Ohw57yVY2dBTt+gsC6aZdteyxwlxfbtgkFEMTEkwgSw= +github.com/cnf/structhash v0.0.0-20250313080605-df4c6cc74a9a/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4= +github.com/corona10/goimagehash v1.1.0 h1:teNMX/1e+Wn/AYSbLHX8mj+mF9r60R1kBeqE9MkoYwI= +github.com/corona10/goimagehash v1.1.0/go.mod h1:VkvE0mLn84L4aF8vCb6mafVajEb6QYMHl2ZJLn0mOGI= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= +github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= +github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c= +github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0= +github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= +github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 h1:2tV76y6Q9BB+NEBasnqvs7e49aEBFI8ejC89PSnWH+4= +github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= +github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU= +github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/gaissmai/bart v0.28.0 h1:89yZLo8NmyqD0RYgJ3QO9HhqqGGw+oWhf90cZm69Lko= +github.com/gaissmai/bart v0.28.0/go.mod h1:GREWQfTLRWz/c5FTOsIw+KkscuFkIV5t8Rp7Nd1Td5c= +github.com/go-faker/faker/v4 v4.9.0 h1:a4HXLwueuTCtgF93VpUsl8Zd2nG1VH2SgNWDPVEBg5U= +github.com/go-faker/faker/v4 v4.9.0/go.mod h1:u1dIRP5neLB6kTzgyVjdBOV5R1uP7BdxkcWk7tiKQXk= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-rod/rod v0.116.2 h1:A5t2Ky2A+5eD/ZJQr1EfsQSe5rms5Xof/qj296e+ZqA= +github.com/go-rod/rod v0.116.2/go.mod h1:H+CMO9SCNc2TJ2WfrG+pKhITz57uGNYU43qYHh438Mg= +github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= +github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 h1:FWNFq4fM1wPfcK40yHE5UO3RUdSNPaBC+j3PokzA6OQ= +github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1/go.mod h1:5YoVOkjYAQumqlV356Hj3xeYh4BdZuLE0/nRkf2NKkI= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/certificate-transparency-go v1.3.2 h1:9ahSNZF2o7SYMaKaXhAumVEzXB2QaayzII9C8rv7v+A= +github.com/google/certificate-transparency-go v1.3.2/go.mod h1:H5FpMUaGa5Ab2+KCYsxg6sELw3Flkl7pGZzWdBoYLXs= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-github/v30 v30.1.0 h1:VLDx+UolQICEOKu2m4uAoMti1SxuEBAl7RSEG16L+Oo= +github.com/google/go-github/v30 v30.1.0/go.mod h1:n8jBpHl45a/rlBUtRJMOG4GhNADUQFEufcolZ95JfU8= +github.com/google/go-github/v50 v50.2.0/go.mod h1:VBY8FB6yPIjrtKhozXv4FQupxKLS6H4m6xFZlT43q8Q= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= +github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= +github.com/gosimple/slug v1.15.0 h1:wRZHsRrRcs6b0XnxMUBM6WK1U1Vg5B0R7VkIf1Xzobo= +github.com/gosimple/slug v1.15.0/go.mod h1:UiRaFH+GEilHstLUmcBgWcI42viBN7mAb818JrYOeFQ= +github.com/gosimple/unidecode v1.0.1 h1:hZzFTMMqSswvf0LBJZCZgThIZrpDHFXux9KeGmn6T/o= +github.com/gosimple/unidecode v1.0.1/go.mod h1:CP0Cr1Y1kogOtx0bJblKzsVWrqYaqfNOnHzpgWw4Awc= +github.com/happyhackingspace/dit v0.0.28 h1:iuX6NjGGEiPNXXltmCrj+LObSKxAy4pwKIEImfMm6pc= +github.com/happyhackingspace/dit v0.0.28/go.mod h1:TFhaJk9bARUpwcycEUwtLK8ZRC0XKInXivg3j/QmNcY= +github.com/hashicorp/go-version v1.8.0 h1:KAkNb1HAiZd1ukkxDFGmokVZe1Xy9HG6NUp+bPle2i4= +github.com/hashicorp/go-version v1.8.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf h1:umfGUaWdFP2s6457fz1+xXYIWDxdGc7HdkLS9aJ1skk= +github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf/go.mod h1:V99KdStnMHZsvVOwIvhfcUzYgYkRZeQWUtumtL+SKxA= +github.com/hdm/jarm-go v0.0.8 h1:8yDIGrmhEJs+QW1voR3JU0Zaz+AUqs3CJzMAQKzGFIc= +github.com/hdm/jarm-go v0.0.8/go.mod h1:memdt/YMMW3811nvANDJsrtc8Fy2e1685aDWdI7Gwto= +github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/iangcarroll/cookiemonster v1.6.0 h1:NPFkn/ZZYZgzXhJ1awRnYhZ3fJK3hKWgbctfTW21kew= +github.com/iangcarroll/cookiemonster v1.6.0/go.mod h1:n3MvoAq56NkNyCEyhcYs3ZJMzTc9rL3w7IaITI0apMg= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kataras/jwt v0.1.10 h1:GBXOF9RVInDPhCFBiDumRG9Tt27l7ugLeLo8HL5SeKQ= +github.com/kataras/jwt v0.1.10/go.mod h1:xkimAtDhU/aGlQqjwvgtg+VyuPwMiyZHaY8LJRh0mYo= +github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= +github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/logrusorgru/aurora/v4 v4.0.0 h1:sRjfPpun/63iADiSvGGjgA1cAYegEWMPCJdUpJYn9JA= +github.com/logrusorgru/aurora/v4 v4.0.0/go.mod h1:lP0iIa2nrnT/qoFXcOZSrZQpJ1o6n2CUf/hyHi2Q4ZQ= +github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag= +github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/melvinsh/subfaster/v2 v2.18.0 h1:sIy0ClfXCcYC8bd6tH8xyfUYm7fFyxrXQy/Zam0+Y1A= +github.com/melvinsh/subfaster/v2 v2.18.0/go.mod h1:VXsynLlJkCKDsUB+JouY+TTAqzBVeTrlZ+9Yy/mTeH8= +github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 h1:bjfMeqxWEJ6IRUvGkiTkSwx0a6UdQJsbirRSoXogteY= +github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6/go.mod h1:WVJJvUw/pIOcwu2O8ZzHEhmigq2jzwRNfJVRMJB7bR8= +github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= +github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= +github.com/miekg/dns v1.1.68 h1:jsSRkNozw7G/mnmXULynzMNIsgY2dHC8LO6U6Ij2JEA= +github.com/miekg/dns v1.1.68/go.mod h1:fujopn7TB3Pu3JM69XaawiU0wqjpL9/8xGop5UrTPps= +github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 h1:yRZGarbxsRytL6EGgbqK2mCY+Lk5MWKQYKJT2gEglhc= +github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= +github.com/mreiferson/go-httpclient v0.0.0-20201222173833-5e475fde3a4d/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= +github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= +github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= +github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= +github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= +github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= +github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= +github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= +github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= +github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= +github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/projectdiscovery/asnmap v1.1.1 h1:ImJiKIaACOT7HPx4Pabb5dksolzaFYsD1kID2iwsDqI= +github.com/projectdiscovery/asnmap v1.1.1/go.mod h1:QT7jt9nQanj+Ucjr9BqGr1Q2veCCKSAVyUzLXfEcQ60= +github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30f7193 h1:UCZRqs1BP1wsvhCwQxfIQc7NJcXGBhQvAnEw3awhsng= +github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30f7193/go.mod h1:nSovPcipgSx/EzAefF+iCfORolkKAuodiRWL3RCGHOM= +github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= +github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= +github.com/projectdiscovery/cdncheck v1.2.42 h1:Y1Q9MPq7uuv25+aGlgjA5nToOcsk+9gNEKjicyhIwQI= +github.com/projectdiscovery/cdncheck v1.2.42/go.mod h1:9oE9KKxCSHNvUf0UaMeqqUwWpC38FkNaTll0ScIBT3w= +github.com/projectdiscovery/chaos-client v0.5.2 h1:dN+7GXEypsJAbCD//dBcUxzAEAEH1fjc/7Rf4F/RiNU= +github.com/projectdiscovery/chaos-client v0.5.2/go.mod h1:KnoJ/NJPhll42uaqlDga6oafFfNw5l2XI2ajRijtDuU= +github.com/projectdiscovery/clistats v0.1.4 h1:kDnXoNxIdOvQElOF7k2Mt6XosGa5GbMKPtRXdPHMVzU= +github.com/projectdiscovery/clistats v0.1.4/go.mod h1:hjJYNcUubk9T3cuFvA+JkLhZGjzYW50fkC48dqUAtbU= +github.com/projectdiscovery/dnsx v1.2.3 h1:S87U9kYuuqqvMFyen8mZQy1FMuR5EGCsXHqfHPQAeuc= +github.com/projectdiscovery/dnsx v1.2.3/go.mod h1:NjAEyJt6+meNqZqnYHL4ZPxXfysuva+et56Eq/e1cVE= +github.com/projectdiscovery/dsl v0.8.20 h1:CxWcKuoHFpOSS1kzqnbJuK5No/6qoRG8IzNDMnZ6c/M= +github.com/projectdiscovery/dsl v0.8.20/go.mod h1:e1oHi7mxAxF+UhBhD5gOk90Ga6LQqvFea2voMO1E5D0= +github.com/projectdiscovery/fastdialer v0.5.11 h1:eI7jfwz0i73Ot1cowIBezQLxbg0i6INdAsFGJjfwPa0= +github.com/projectdiscovery/fastdialer v0.5.11/go.mod h1:W1ZkULr9mMR6i0oRFTztANnpVyEEzPUovK8sUM4eAw8= +github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= +github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= +github.com/projectdiscovery/freeport v0.0.7 h1:Q6uXo/j8SaV/GlAHkEYQi8WQoPXyJWxyspx+aFmz9Qk= +github.com/projectdiscovery/freeport v0.0.7/go.mod h1:cOhWKvNBe9xM6dFJ3RrrLvJ5vXx2NQ36SecuwjenV2k= +github.com/projectdiscovery/goconfig v0.0.1 h1:36m3QjohZvemqh9bkJAakaHsm9iEZ2AcQSS18+0QX/s= +github.com/projectdiscovery/goconfig v0.0.1/go.mod h1:CPO25zR+mzTtyBrsygqsHse0sp/4vB/PjaHi9upXlDw= +github.com/projectdiscovery/goflags v0.1.74 h1:n85uTRj5qMosm0PFBfsvOL24I7TdWRcWq/1GynhXS7c= +github.com/projectdiscovery/goflags v0.1.74/go.mod h1:UMc9/7dFz2oln+10tv6cy+7WZKTHf9UGhaNkF95emh4= +github.com/projectdiscovery/gologger v1.1.71 h1:IYU4mw9viKdSzMTIGVpYuw1Gtg7QIHIStqAQgeNXcBQ= +github.com/projectdiscovery/gologger v1.1.71/go.mod h1:mJwODZcFDg70ihINpOvZevmBtgvpP8H9/l8Y+OPhZPY= +github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= +github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= +github.com/projectdiscovery/govaluate v0.0.0-20260504230327-80320480bb6e h1:o+ulEIaC2+9V2Ezr6mI5xEhKWsf0V/+FUQIS723Aj6U= +github.com/projectdiscovery/govaluate v0.0.0-20260504230327-80320480bb6e/go.mod h1:xH7bPwHxUlz1yx9UlVeTF+UVCUaKhTnZgaxHb5z362E= +github.com/projectdiscovery/hmap v0.0.101 h1:zXM6YtLmsn8Q0CUUw8QavhqWmiQYwaw+/U679Rr00pc= +github.com/projectdiscovery/hmap v0.0.101/go.mod h1:w6N9/a5H8kvyx53AhtPDUWe5Qq3D6NBDPA23glHpa/Q= +github.com/projectdiscovery/httpx v1.10.0 h1:sZvfeJDeX4eJfB5Oh29My4gOHmLfvpCt2wCqhargJrE= +github.com/projectdiscovery/httpx v1.10.0/go.mod h1:gfdNnFznLLlAgAgSQlYjUeEVvYzKrNV0jO5Qyo5w+FI= +github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582 h1:eR+0HE//Ciyfwy3HC7fjRyKShSJHYoX2Pv7pPshjK/Q= +github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= +github.com/projectdiscovery/mapcidr v1.1.97 h1:7FkxNNVXp+m1rIu5Nv/2SrF9k4+LwP8QuWs2puwy+2w= +github.com/projectdiscovery/mapcidr v1.1.97/go.mod h1:9dgTJh1SP02gYZdpzMjm6vtYFkEHQHoTyaVNvaeJ7lA= +github.com/projectdiscovery/networkpolicy v0.1.41 h1:6daf4A8Vj1+iQ7nH2FR4+sOYd7q1WH1qfN61EyQU74c= +github.com/projectdiscovery/networkpolicy v0.1.41/go.mod h1:9ULLaMbdv9UnT0C5rmuK4nIwYs0o776xMnkPUb8TtaE= +github.com/projectdiscovery/ratelimit v0.0.88 h1:AcurW9aLRzlEyPe9kSjnOpr3XzLMWTpiWAlW/w73ALU= +github.com/projectdiscovery/ratelimit v0.0.88/go.mod h1:CU1s+68UUG2mctSl2wi32/DHLJA6TMg+4rxgP59LfVk= +github.com/projectdiscovery/rawhttp v0.1.90 h1:LOSZ6PUH08tnKmWsIwvwv1Z/4zkiYKYOSZ6n+8RFKtw= +github.com/projectdiscovery/rawhttp v0.1.90/go.mod h1:VZYAM25UI/wVB3URZ95ZaftgOnsbphxyAw/XnQRRz4Y= +github.com/projectdiscovery/retryabledns v1.0.115 h1:RKV63FNIznFHUoawg/1hs53pVH3wqPtFhwstCuxVSoA= +github.com/projectdiscovery/retryabledns v1.0.115/go.mod h1:+fEMWoPigw+M0lGNKY7AZ+g8FIgj+4sONjsinMmeL3k= +github.com/projectdiscovery/retryablehttp-go v1.3.16 h1:M/xICwSaiSHhd5OIarU3+5JoU7VmbiSAAwYUCK7CTjw= +github.com/projectdiscovery/retryablehttp-go v1.3.16/go.mod h1:s0azLAqAbcVCjHI9t0ezPhamevYGM1eoOvFkn4QmpZ8= +github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= +github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= +github.com/projectdiscovery/tlsx v1.2.2 h1:Y96QBqeD2anpzEtBl4kqNbwzXh2TrzJuXfgiBLvK+SE= +github.com/projectdiscovery/tlsx v1.2.2/go.mod h1:ZJl9F1sSl0sdwE+lR0yuNHVX4Zx6tCSTqnNxnHCFZB4= +github.com/projectdiscovery/useragent v0.0.108 h1:fb+uLuFJvC+MHZjCtxQJxtvp1X6A8n98CUGPyFcg3NE= +github.com/projectdiscovery/useragent v0.0.108/go.mod h1:XdNRrlvtDmYfVL1Oybat4uMe+W6cLwsK9S18ond17CI= +github.com/projectdiscovery/utils v0.11.1 h1:PWj1KjIASxt8icxommH72C0TQqNOvGkcSODRkiq0SQw= +github.com/projectdiscovery/utils v0.11.1/go.mod h1:yktGrHGk2CTjNiccXovnvGrLHX9sV2bqz9nSnbA3V8M= +github.com/projectdiscovery/wappalyzergo v0.2.87 h1:KuUpeRSQ80L6tx9YaAPhfYWTF47bpEURYbAymr76zto= +github.com/projectdiscovery/wappalyzergo v0.2.87/go.mod h1:gMH0o5lBp65sKMwHx/tuUdOtW2RjodC6Ti+9QDsYMkY= +github.com/refraction-networking/utls v1.8.2 h1:j4Q1gJj0xngdeH+Ox/qND11aEfhpgoEvV+S9iJ2IdQo= +github.com/refraction-networking/utls v1.8.2/go.mod h1:jkSOEkLqn+S/jtpEHPOsVv/4V4EVnelwbMQl4vCWXAM= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/go-internal v1.16.0 h1:O9DK+vNMDVGLr2BeZqmpLeMjiMNkuXfcqntWbZV6S5g= +github.com/rogpeppe/go-internal v1.16.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= +github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= +github.com/sashabaranov/go-openai v1.37.0 h1:hQQowgYm4OXJ1Z/wTrE+XZaO20BYsL0R3uRPSpfNZkY= +github.com/sashabaranov/go-openai v1.37.0/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= +github.com/seh-msft/burpxml v1.0.1 h1:5G3QPSzvfA1WcX7LkxmKBmK2RnNyGviGWnJPumE0nwg= +github.com/seh-msft/burpxml v1.0.1/go.mod h1:lTViCHPtGGS0scK0B4krm6Ld1kVZLWzQccwUomRc58I= +github.com/shirou/gopsutil/v4 v4.26.3 h1:2ESdQt90yU3oXF/CdOlRCJxrP+Am1aBYubTMTfxJ1qc= +github.com/shirou/gopsutil/v4 v4.26.3/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= +github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= +github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= +github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= +github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI= +github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8= +github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI= +github.com/tidwall/btree v1.7.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY= +github.com/tidwall/buntdb v1.3.1 h1:HKoDF01/aBhl9RjYtbaLnvX9/OuenwvQiC3OP1CcL4o= +github.com/tidwall/buntdb v1.3.1/go.mod h1:lZZrZUWzlyDJKlLQ6DKAy53LnG7m5kHyrEHvvcDmBpU= +github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/grect v0.1.4 h1:dA3oIgNgWdSspFzn1kS4S/RDpZFLrIxAZOdJKjYapOg= +github.com/tidwall/grect v0.1.4/go.mod h1:9FBsaYRaR0Tcy4UwefBX/UDcDcDy9V5jUcxHzv2jd5Q= +github.com/tidwall/lotsa v1.0.2 h1:dNVBH5MErdaQ/xd9s769R31/n2dXavsQ0Yf4TMEHHw8= +github.com/tidwall/lotsa v1.0.2/go.mod h1:X6NiU+4yHA3fE3Puvpnn1XMDrFZrE9JO2/w+UMuqgR8= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/match v1.2.0 h1:0pt8FlkOwjN2fPt4bIl4BoNxb98gGHN2ObFEDkrfZnM= +github.com/tidwall/match v1.2.0/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/rtred v0.1.2 h1:exmoQtOLvDoO8ud++6LwVsAMTu0KPzLTUrMln8u1yu8= +github.com/tidwall/rtred v0.1.2/go.mod h1:hd69WNXQ5RP9vHd7dqekAz+RIdtfBogmglkZSRxCHFQ= +github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaymYE= +github.com/tidwall/tinyqueue v0.1.1/go.mod h1:O/QNHwrnjqr6IHItYrzoHAKYhBkLI67Q096fQP5zMYw= +github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA= +github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI= +github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw= +github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ= +github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y= +github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE= +github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/vulncheck-oss/go-exploit v1.51.0 h1:HTmJ4Q94tbEDPb35mQZn6qMg4rT+Sw9n+L7g3Pjr+3o= +github.com/vulncheck-oss/go-exploit v1.51.0/go.mod h1:J28w0dLnA6DnCrnBm9Sbt6smX8lvztnnN2wCXy7No6c= +github.com/weppos/publicsuffix-go v0.13.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= +github.com/weppos/publicsuffix-go v0.40.2/go.mod h1:XsLZnULC3EJ1Gvk9GVjuCTZ8QUu9ufE4TZpOizDShko= +github.com/weppos/publicsuffix-go v0.50.3 h1:eT5dcjHQcVDNc0igpFEsGHKIip30feuB2zuuI9eJxiE= +github.com/weppos/publicsuffix-go v0.50.3/go.mod h1:/rOa781xBykZhHK/I3QeHo92qdDKVmKZKF7s8qAEM/4= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU= +github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g= +github.com/ysmood/fetchup v0.2.3 h1:ulX+SonA0Vma5zUFXtv52Kzip/xe7aj4vqT5AJwQ+ZQ= +github.com/ysmood/fetchup v0.2.3/go.mod h1:xhibcRKziSvol0H1/pj33dnKrYyI2ebIvz5cOOkYGns= +github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ= +github.com/ysmood/goob v0.4.0/go.mod h1:u6yx7ZhS4Exf2MwciFr6nIM8knHQIE22lFpWHnfql18= +github.com/ysmood/gop v0.2.0 h1:+tFrG0TWPxT6p9ZaZs+VY+opCvHU8/3Fk6BaNv6kqKg= +github.com/ysmood/gop v0.2.0/go.mod h1:rr5z2z27oGEbyB787hpEcx4ab8cCiPnKxn0SUHt6xzk= +github.com/ysmood/got v0.40.0 h1:ZQk1B55zIvS7zflRrkGfPDrPG3d7+JOza1ZkNxcc74Q= +github.com/ysmood/got v0.40.0/go.mod h1:W7DdpuX6skL3NszLmAsC5hT7JAhuLZhByVzHTq874Qg= +github.com/ysmood/gotrace v0.6.0 h1:SyI1d4jclswLhg7SWTL6os3L1WOKeNn/ZtzVQF8QmdY= +github.com/ysmood/gotrace v0.6.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM= +github.com/ysmood/gson v0.7.3 h1:QFkWbTH8MxyUTKPkVWAENJhxqdBa4lYTQWqZCiLG6kE= +github.com/ysmood/gson v0.7.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg= +github.com/ysmood/leakless v0.9.0 h1:qxCG5VirSBvmi3uynXFkcnLMzkphdh3xx5FtrORwDCU= +github.com/ysmood/leakless v0.9.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yuin/goldmark v1.7.1/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E= +github.com/yuin/goldmark v1.7.13 h1:GPddIs617DnBLFFVJFgpo1aBfe/4xcvMc3SB5t/D0pA= +github.com/yuin/goldmark v1.7.13/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= +github.com/yuin/goldmark-emoji v1.0.3 h1:aLRkLHOuBR2czCY4R8olwMjID+tENfhyFDMCRhbIQY4= +github.com/yuin/goldmark-emoji v1.0.3/go.mod h1:tTkZEbwu5wkPmgTcitqddVxY9osFZiavD+r4AzQrh1U= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/zcalusic/sysinfo v1.0.2 h1:nwTTo2a+WQ0NXwo0BGRojOJvJ/5XKvQih+2RrtWqfxc= +github.com/zcalusic/sysinfo v1.0.2/go.mod h1:kluzTYflRWo6/tXVMJPdEjShsbPpsFRyy+p1mBQPC30= +github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= +github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 h1:Nzukz5fNOBIHOsnP+6I79kPx3QhLv8nBy2mfFhBRq30= +github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= +github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54tB79AMBcySS0R2XIyZBAVmeHranShAFELYx7is= +github.com/zmap/zcertificate v0.0.1/go.mod h1:q0dlN54Jm4NVSSuzisusQY0hqDWvu92C+TWveAxiVWk= +github.com/zmap/zcrypto v0.0.0-20201128221613-3719af1573cf/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ= +github.com/zmap/zcrypto v0.0.0-20201211161100-e54a5822fb7e/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ= +github.com/zmap/zcrypto v0.0.0-20240803002437-3a861682ac77 h1:DCz0McWRVJNICkHdu2XpETqeLvPtZXs315OZyUs1BDk= +github.com/zmap/zcrypto v0.0.0-20240803002437-3a861682ac77/go.mod h1:aSvf+uTU222mUYq/KQj3oiEU7ajhCZe8RRSLHIoM4EM= +github.com/zmap/zlint/v3 v3.0.0/go.mod h1:paGwFySdHIBEMJ61YjoqT4h7Ge+fdYG4sUQhnTb1lJ8= +go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= +go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= +golang.org/x/exp v0.0.0-20250911091902-df9299821621 h1:2id6c1/gto0kaHYyrixvknJ8tUK/Qs5IsmBtrc+FtgU= +golang.org/x/exp v0.0.0-20250911091902-df9299821621/go.mod h1:TwQYMMnGpvZyc+JpB/UAuTNIsVJifOlSkrZkhcvpVUk= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= +golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= +golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= +golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201126233918-771906719818/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210228012217-479acdf4ea46/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= +golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= +golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/app/app.go b/internal/app/app.go new file mode 100644 index 0000000..329aa50 --- /dev/null +++ b/internal/app/app.go @@ -0,0 +1,257 @@ +// Package app assembles a run from a configuration. +// +// It exists so the one-shot CLI and the HTTP handler share one construction +// path: a request served by `fastrecon serve` must be wired exactly like a +// command line invocation, or the two would drift and only one of them would +// be the tested one. +package app + +import ( + "context" + "errors" + "fmt" + "log/slog" + "slices" + "sync" + + "github.com/JoshuaMart/FastRecon/internal/config" + "github.com/JoshuaMart/FastRecon/internal/enumerate" + "github.com/JoshuaMart/FastRecon/internal/exclude" + "github.com/JoshuaMart/FastRecon/internal/pipeline" + "github.com/JoshuaMart/FastRecon/internal/portscan" + "github.com/JoshuaMart/FastRecon/internal/probe" + "github.com/JoshuaMart/FastRecon/internal/report" + "github.com/JoshuaMart/FastRecon/internal/resolve" + "github.com/JoshuaMart/FastRecon/internal/secrets" + "github.com/JoshuaMart/FastRecon/internal/stage" +) + +// ErrRuntime marks a preparation failure that is transient rather than a +// mistake in the configuration: an unreachable resolver list, a health check +// that found nothing usable. Callers map it to an exit code or a status code, +// which is the only signal a scheduler or a caller has. +var ErrRuntime = errors.New("runtime failure") + +// App holds what is settled once per process and reused by every run. +// +// The split is not an optimisation. The enumeration engine keys its API keys +// into globally shared source instances, so credentials cannot vary per run +// without one run overwriting another's; resolving them here, once, is what +// makes serving several requests from one process safe. +type App struct { + cfg *config.Config + log *slog.Logger + creds map[string]secrets.Credential + redactor *secrets.Redactor + + // The resolver pool is loaded and health-checked on first use, then kept: + // re-checking a public resolver list on every request would cost more than + // the run it serves. A failure is not cached — it is usually the network. + poolMu sync.Mutex + pool []string + poolWarnings []string +} + +// New settles the process-level configuration. +func New(cfg *config.Config, log *slog.Logger) (*App, error) { + if cfg == nil || log == nil { + return nil, errors.New("app: configuration and logger are required") + } + + a := &App{cfg: cfg, log: log} + a.creds = a.resolveCredentials() + a.redactor = secrets.NewRedactor(a.creds) + + // Published to the environment here, once: it is the only channel the + // enumeration engine reads, and it is process-global either way. + if err := secrets.Export(a.creds); err != nil { + return nil, err + } + return a, nil +} + +// Redactor returns the process redactor, so a caller can scrub a rendered +// report before it leaves the process. +func (a *App) Redactor() *secrets.Redactor { return a.redactor } + +// Config returns the process-level configuration. +func (a *App) Config() *config.Config { return a.cfg } + +// Run executes one pipeline against runCfg, which may be the process +// configuration or a per-request copy of it. +func (a *App) Run(ctx context.Context, runCfg *config.Config) (*report.Report, error) { + stages, err := a.buildStages(ctx, runCfg) + if err != nil { + return nil, err + } + return pipeline.New(runCfg, stages, a.log).Run(ctx) +} + +// resolveCredentials resolves the source keys for every source that may be +// queried. +func (a *App) resolveCredentials() map[string]secrets.Credential { + resolver, err := secrets.NewResolver(a.cfg.ProviderConfig) + if err != nil { + // A provider config that cannot be read is reported by the stage that + // needs it; a run without keys is still a run. + a.log.Warn("provider config unusable", "error", err) + return nil + } + + // Cloned: appending to cfg.Sources would write into its backing array + // whenever it has spare capacity. + wanted := slices.Clone(a.cfg.Sources) + if a.cfg.AllSources { + for _, s := range enumerate.Available() { + wanted = append(wanted, s.Name) + } + slices.Sort(wanted) + wanted = slices.Compact(wanted) + } + + creds := resolver.Resolve(wanted) + // Reported against every source that will be queried, not just the + // default five: under --all-sources the two differ completely. + inv := secrets.Take(wanted, creds) + for source, origin := range inv.Configured { + a.log.Debug("source credential", "source", source, "origin", origin) + } + a.log.Info("credentials resolved", "configured", len(inv.Configured), "missing", inv.Missing) + return creds +} + +// buildStages wires the stage implementations the scope calls for. +func (a *App) buildStages(ctx context.Context, cfg *config.Config) (pipeline.Stages, error) { + enumerator, err := enumerate.NewSubfaster(enumerate.Options{ + Sources: a.cfg.Sources, + ExcludeSources: a.cfg.ExcludeSources, + All: a.cfg.AllSources, + SourceTimeout: a.cfg.SourceTimeout, + Credentials: a.creds, + Redactor: a.redactor, + Logger: a.log, + }) + if err != nil { + return pipeline.Stages{}, err + } + + excluder, err := exclude.New(cfg.Exclude, cfg.ExcludeStrictWildcard) + if err != nil { + return pipeline.Stages{}, fmt.Errorf("invalid exclusions:\n%w", err) + } + + stages := pipeline.Stages{Enumerator: enumerator, Excluder: excluder} + + // Built only when the scope reaches it: a resolver constructed for an + // enumeration-only run would open sockets nothing asked for. + if cfg.Scope.Includes(stage.Resolve) { + resolvers, warnings, err := a.resolverPool(ctx) + if err != nil { + return pipeline.Stages{}, err + } + cfg.Warnings = append(cfg.Warnings, warnings...) + + resolver, err := resolve.New(resolve.Options{ + Domain: cfg.Domain, + Resolvers: resolvers, + Concurrency: a.cfg.ResolverConcurrency, + Retries: a.cfg.ResolverRetries, + Timeout: a.cfg.ResolverTimeout, + WildcardProbes: a.cfg.WildcardProbes, + Logger: a.log, + }) + if err != nil { + return pipeline.Stages{}, err + } + stages.Resolver = resolver + } + + if cfg.Scope.Includes(stage.PortScan) { + scanner, err := portscan.New(portscan.Options{ + Mode: a.cfg.ScanMode, + Ports: cfg.Ports, + ExcludePorts: cfg.ExcludePorts, + SkipCDN: cfg.SkipCDN, + Concurrency: a.cfg.ScanConcurrency, + Rate: a.cfg.ScanRate, + Retries: a.cfg.ScanRetries, + Timeout: a.cfg.ScanTimeout, + Logger: a.log, + }) + if err != nil { + return pipeline.Stages{}, err + } + stages.PortScanner = scanner + } + + if cfg.Scope.Includes(stage.HTTPProbe) { + prober, err := probe.New(probe.Options{ + Concurrency: a.cfg.ProbeConcurrency, + Rate: a.cfg.ProbeRate, + Timeout: a.cfg.ProbeTimeout, + Retries: a.cfg.ProbeRetries, + FollowRedirects: a.cfg.ProbeFollowRedirects, + MaxRedirects: a.cfg.ProbeMaxRedirects, + UserAgent: a.cfg.ProbeUserAgent, + Headers: a.cfg.ProbeHeaders, + Logger: a.log, + }) + if err != nil { + return pipeline.Stages{}, err + } + stages.Prober = prober + } + + return stages, nil +} + +// resolverPool assembles the resolver list once and, unless told otherwise, +// removes the resolvers that cannot be trusted to answer correctly. +func (a *App) resolverPool(ctx context.Context) (resolvers, warnings []string, err error) { + a.poolMu.Lock() + defer a.poolMu.Unlock() + + if a.pool != nil { + return a.pool, a.poolWarnings, nil + } + + resolvers, err = resolve.LoadResolvers(ctx, resolve.LoadOptions{ + Inline: a.cfg.Resolvers, + File: a.cfg.ResolversFile, + URL: a.cfg.ResolversURL, + Logger: a.log, + }) + if err != nil { + // Fetching a list over the network can fail for reasons that have + // nothing to do with the configuration being wrong. + if a.cfg.ResolversURL != "" { + return nil, nil, fmt.Errorf("%w: %w", ErrRuntime, err) + } + return nil, nil, err + } + a.log.Info("resolvers loaded", "count", len(resolvers)) + + if a.cfg.ValidateResolvers { + health := resolve.CheckResolvers(ctx, resolvers, resolve.HealthOptions{ + Budget: a.cfg.ResolverHealthBudget, + Timeout: a.cfg.ResolverTimeout, + Concurrency: a.cfg.ResolverConcurrency, + Logger: a.log, + }) + if len(health.Good) == 0 { + return nil, nil, fmt.Errorf("%w: every one of the %d configured resolvers failed the health check", ErrRuntime, len(resolvers)) + } + // These reach the report, not just the log: a resolution done through + // a pool that lost half its members is a result worth qualifying. + if len(health.Dropped) > 0 { + warnings = append(warnings, fmt.Sprintf("%d of %d resolvers dropped by the health check", len(health.Dropped), len(resolvers))) + } + if health.Unchecked > 0 { + warnings = append(warnings, fmt.Sprintf("%d of %d resolvers were kept unchecked: the health budget ran out", health.Unchecked, len(resolvers))) + } + resolvers = health.Good + } + + a.pool, a.poolWarnings = resolvers, warnings + return resolvers, warnings, nil +} diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..65bd2fe --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,148 @@ +// Package config resolves the run configuration. +// +// Every option is settable three ways and the precedence is always the same: +// +// CLI flag > environment variable > config file > built-in default +// +// The names are mechanically related: a flag named "http-timeout" reads the +// environment variable FASTRECON_HTTP_TIMEOUT and the config-file key +// "http-timeout". That rule is what lets a serverless job be configured +// entirely through environment variables without a config file. +package config + +import ( + "fmt" + "slices" + "time" + + "github.com/JoshuaMart/FastRecon/internal/report" + "github.com/JoshuaMart/FastRecon/internal/stage" +) + +// EnvPrefix is prepended to every derived environment-variable name. +const EnvPrefix = "FASTRECON_" + +// Config is the fully resolved configuration of a single run. +type Config struct { + // Target + Domain string + Exclude []string + ExcludeFile string + ExcludeStrictWildcard bool + ReportExcluded bool + + // Pipeline + Scope stage.Scope + Timeout time.Duration + // OutputMargin is the fraction of the total budget reserved for building + // and delivering the report, so a deadline yields a truncated report + // instead of a killed process. + OutputMargin float64 + + // Enumeration + ProviderConfig string + Sources []string + ExcludeSources []string + AllSources bool + SourceTimeout time.Duration + + // Resolution + Resolvers []string + ResolversFile string + ResolversURL string + ValidateResolvers bool + ResolverHealthBudget time.Duration + ResolverConcurrency int + ResolverRetries int + ResolverTimeout time.Duration + WildcardProbes int + + // Port scan + ScanMode string + Ports string + ExcludePorts string + SkipCDN bool + ScanConcurrency int + ScanRate int + ScanTimeout time.Duration + ScanRetries int + + // HTTP probe + ProbeConcurrency int + ProbeRate int + ProbeRetries int + ProbeTimeout time.Duration + ProbeFollowRedirects bool + ProbeMaxRedirects int + ProbeUserAgent string + ProbeHeaders []string + + // Output + Output string + Format report.Format + WebhookURL string + WebhookMethod string + WebhookHeaders []string + WebhookTimeout time.Duration + WebhookRetries int + + // Process + LogLevel string + LogFormat string + Environment string + ConfigFile string + + // Serve mode + Listen string + APIToken string + + // Warnings collected while resolving, surfaced in the report so a typo in + // a config file is visible rather than silently ignored. + Warnings []string +} + +// RequiredSources is the default source selection: the sources a run is +// expected to use. The keyed ones report themselves as skipped when no +// credential is configured, rather than being silently dropped. +// +// chaos ProjectDiscovery Chaos key required +// securitytrails SecurityTrails key required +// c99 c99.nl key required +// submd sub.md key optional +// crt crt.name key optional +var RequiredSources = []string{"chaos", "securitytrails", "c99", "submd", "crt"} + +// Scan modes. +const ( + ScanModeConnect = "connect" + ScanModeSYN = "syn" +) + +// StdoutPath is the Output value that selects the stdout sink. +const StdoutPath = "-" + +// Clone deep-copies the configuration. +// +// A served request overlays its own fields onto the process configuration, so +// the copy has to be independent: sharing a slice would let one request's +// exclusions leak into the next. +func (c *Config) Clone() *Config { + out := *c + out.Exclude = slices.Clone(c.Exclude) + out.Sources = slices.Clone(c.Sources) + out.ExcludeSources = slices.Clone(c.ExcludeSources) + out.Resolvers = slices.Clone(c.Resolvers) + out.ProbeHeaders = slices.Clone(c.ProbeHeaders) + out.WebhookHeaders = slices.Clone(c.WebhookHeaders) + out.Warnings = slices.Clone(c.Warnings) + return &out +} + +// Sinks reports which destinations are configured. +func (c *Config) Sinks() (stdout, file, webhook bool) { + return c.Output == StdoutPath, c.Output != "" && c.Output != StdoutPath, c.WebhookURL != "" +} + +func (c *Config) warnf(format string, args ...any) { + c.Warnings = append(c.Warnings, fmt.Sprintf(format, args...)) +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 0000000..b2423e9 --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,227 @@ +package config + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/spf13/pflag" +) + +func load(t *testing.T, args ...string) (*Config, error) { + t.Helper() + fs := pflag.NewFlagSet("test", pflag.ContinueOnError) + RegisterFlags(fs) + if err := fs.Parse(args); err != nil { + t.Fatalf("parse args: %v", err) + } + return Load(fs) +} + +func mustLoad(t *testing.T, args ...string) *Config { + t.Helper() + cfg, err := load(t, args...) + if err != nil { + t.Fatalf("Load: %v", err) + } + return cfg +} + +func TestEnvName(t *testing.T) { + for flag, want := range map[string]string{ + "domain": "FASTRECON_DOMAIN", + "http-timeout": "FASTRECON_HTTP_TIMEOUT", + "scan-mode": "FASTRECON_SCAN_MODE", + } { + if got := EnvName(flag); got != want { + t.Errorf("EnvName(%q) = %q, want %q", flag, got, want) + } + } +} + +func TestPrecedenceFlagOverEnvOverFileOverDefault(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + if err := os.WriteFile(path, []byte("domain: file.example.com\nports: top-1000\nscan-mode: syn\n"), 0o600); err != nil { + t.Fatal(err) + } + + // Default wins when nothing else is set. + t.Setenv("FASTRECON_DOMAIN", "env.example.com") + cfg := mustLoad(t, "--config", path) + if cfg.ScanMode != ScanModeSYN { + t.Errorf("scan-mode = %q, want the file value %q", cfg.ScanMode, ScanModeSYN) + } + if cfg.Ports != "top-1000" { + t.Errorf("ports = %q, want the file value", cfg.Ports) + } + // Env beats the file. + if cfg.Domain != "env.example.com" { + t.Errorf("domain = %q, want the environment value", cfg.Domain) + } + // Flag beats the environment. + cfg = mustLoad(t, "--config", path, "-d", "flag.example.com") + if cfg.Domain != "flag.example.com" { + t.Errorf("domain = %q, want the flag value", cfg.Domain) + } + // Untouched options keep their default. + if cfg.Timeout != 30*time.Minute { + t.Errorf("timeout = %s, want the 30m default", cfg.Timeout) + } +} + +func TestEnvListSplitting(t *testing.T) { + t.Setenv("FASTRECON_EXCLUDE", "a.example.com, *.dev.example.com\nre:^staging") + cfg := mustLoad(t, "-d", "example.com") + want := []string{"a.example.com", "*.dev.example.com", "re:^staging"} + if len(cfg.Exclude) != len(want) { + t.Fatalf("exclude = %v, want %v", cfg.Exclude, want) + } + for i := range want { + if cfg.Exclude[i] != want[i] { + t.Errorf("exclude[%d] = %q, want %q", i, cfg.Exclude[i], want[i]) + } + } +} + +// A regexp repeat count contains a comma. Splitting on it yields two halves +// that both still compile, so the run would quietly scan hosts the operator +// excluded — the worst possible failure for an exclusion. +func TestRegexExclusionsSurviveEnvSplitting(t *testing.T) { + t.Setenv("FASTRECON_EXCLUDE", `re:^a{1,3}\.example\.com$`+"\nadmin.example.com, www.example.com") + cfg := mustLoad(t, "-d", "example.com") + + want := []string{`re:^a{1,3}\.example\.com$`, "admin.example.com", "www.example.com"} + if len(cfg.Exclude) != len(want) { + t.Fatalf("exclude = %#v, want %#v", cfg.Exclude, want) + } + for i := range want { + if cfg.Exclude[i] != want[i] { + t.Errorf("exclude[%d] = %q, want %q", i, cfg.Exclude[i], want[i]) + } + } +} + +// The engine matches source names case-sensitively and calls os.Exit on an +// empty selection, so names are normalized before they can get there. +func TestSourceNamesAreNormalized(t *testing.T) { + cfg := mustLoad(t, "-d", "example.com", "--sources", "Crt", "--sources", " SUBMD ", "--exclude-sources", "Chaos") + if len(cfg.Sources) != 2 || cfg.Sources[0] != "crt" || cfg.Sources[1] != "submd" { + t.Errorf("sources = %v, want them lowercased and trimmed", cfg.Sources) + } + if len(cfg.ExcludeSources) != 1 || cfg.ExcludeSources[0] != "chaos" { + t.Errorf("exclude-sources = %v, want them lowercased", cfg.ExcludeSources) + } +} + +// The engine takes whole seconds and truncates, so anything under a second +// would silently become no ceiling at all. +func TestSubSecondSourceTimeoutRejected(t *testing.T) { + if _, err := load(t, "-d", "example.com", "--source-timeout", "500ms"); err == nil { + t.Error("a sub-second source timeout was accepted") + } + if _, err := load(t, "-d", "example.com", "--source-timeout", "1s"); err != nil { + t.Errorf("a one-second source timeout was rejected: %v", err) + } +} + +// Header values may legitimately contain a comma, so they split on newlines +// only — otherwise "Accept: a,b" would silently become two broken headers. +func TestHeaderEnvSplitsOnNewlinesOnly(t *testing.T) { + t.Setenv("FASTRECON_PROBE_HEADER", "Accept: text/html,application/json\nX-Trace: 1") + cfg := mustLoad(t, "-d", "example.com") + if len(cfg.ProbeHeaders) != 2 { + t.Fatalf("probe headers = %v, want 2 entries", cfg.ProbeHeaders) + } + if cfg.ProbeHeaders[0] != "Accept: text/html,application/json" { + t.Errorf("header[0] = %q, comma was split", cfg.ProbeHeaders[0]) + } +} + +func TestUnknownConfigKeyWarns(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + if err := os.WriteFile(path, []byte("domain: example.com\nporst: top-100\n"), 0o600); err != nil { + t.Fatal(err) + } + cfg := mustLoad(t, "--config", path) + if len(cfg.Warnings) != 1 || !strings.Contains(cfg.Warnings[0], "porst") { + t.Errorf("warnings = %v, want one warning naming the unknown key", cfg.Warnings) + } +} + +func TestExcludeFileMergesWithInlinePatterns(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "exclusions.txt") + body := "# comment\n\n*.dev.example.com\nadmin.example.com\n" + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + cfg := mustLoad(t, "-d", "example.com", "--exclude", "inline.example.com", "--exclude-file", path) + want := []string{"inline.example.com", "*.dev.example.com", "admin.example.com"} + if len(cfg.Exclude) != len(want) { + t.Fatalf("exclude = %v, want %v", cfg.Exclude, want) + } +} + +func TestInvalidValuesAreReportedTogether(t *testing.T) { + _, err := load(t, "-d", "example.com", "--scan-mode", "sneaky", "--timeout", "0s", "--probe-concurrency", "0") + if err == nil { + t.Fatal("Load succeeded on an invalid configuration") + } + msg := err.Error() + for _, want := range []string{"scan-mode", "timeout", "probe-concurrency"} { + if !strings.Contains(msg, want) { + t.Errorf("error %q does not mention %q; every problem must surface at once", msg, want) + } + } +} + +func TestNormalizeDomain(t *testing.T) { + ok := map[string]string{ + "Example.COM": "example.com", + "example.com.": "example.com", + "*.example.com": "example.com", + " example.com ": "example.com", + "sub.example.co.uk": "sub.example.co.uk", + } + for in, want := range ok { + got, err := NormalizeDomain(in) + if err != nil { + t.Errorf("NormalizeDomain(%q) failed: %v", in, err) + continue + } + if got != want { + t.Errorf("NormalizeDomain(%q) = %q, want %q", in, got, want) + } + } + + for _, in := range []string{"", "localhost", "https://example.com", "example.com/path", "exa mple.com", "-bad.example.com", "café.fr"} { + if got, err := NormalizeDomain(in); err == nil { + t.Errorf("NormalizeDomain(%q) = %q, want an error", in, got) + } + } +} + +func TestWebhookHeadersWithoutURLRejected(t *testing.T) { + if _, err := load(t, "-d", "example.com", "--webhook-header", "X-Token: abc"); err == nil { + t.Error("webhook-header accepted without webhook-url") + } +} + +func TestMalformedHeaderRejected(t *testing.T) { + if _, err := load(t, "-d", "example.com", "--probe-header", "NoColon"); err == nil { + t.Error("malformed probe header accepted") + } +} + +func TestDetectEnvironmentPrefersExplicitLabel(t *testing.T) { + if got := DetectEnvironment("serverless-job", false); got != EnvServerlessJob { + t.Errorf("DetectEnvironment = %q, want the explicit label", got) + } + if got := DetectEnvironment("", true); got != EnvServerlessFunction { + t.Errorf("DetectEnvironment in serve mode = %q, want %q", got, EnvServerlessFunction) + } +} diff --git a/internal/config/env.go b/internal/config/env.go new file mode 100644 index 0000000..aa66826 --- /dev/null +++ b/internal/config/env.go @@ -0,0 +1,23 @@ +package config + +import ( + "os" + "strings" +) + +// EnvName derives the environment variable for a flag: "http-timeout" becomes +// FASTRECON_HTTP_TIMEOUT. The mapping is mechanical on purpose — there is no +// per-option table to fall out of sync with the flag set. +func EnvName(flag string) string { + return EnvPrefix + strings.ToUpper(strings.ReplaceAll(flag, "-", "_")) +} + +func lookupEnv(flag string) (string, bool) { + v, ok := os.LookupEnv(EnvName(flag)) + if !ok { + return "", false + } + // An explicitly empty variable is a value, not an absence: it is how a + // deployment unsets an option inherited from a config file. + return v, true +} diff --git a/internal/config/environment.go b/internal/config/environment.go new file mode 100644 index 0000000..e19f858 --- /dev/null +++ b/internal/config/environment.go @@ -0,0 +1,47 @@ +package config + +import ( + "os" + "strings" +) + +// Environment labels recorded in the report. +const ( + EnvLocal = "local" + EnvContainer = "container" + EnvServerlessJob = "serverless-job" + EnvServerlessFunction = "serverless-function" +) + +// DetectEnvironment labels the runtime for the report. +// +// Only local and container are detectable from inside the process. A +// serverless job and a plain container look the same to the runtime, so the +// job and function labels come from the deployment setting --environment +// (FASTRECON_ENVIRONMENT); the deploy manifests in deploy/ do that. Guessing +// from undocumented platform variables would silently mislabel runs the day +// the platform renames one. +func DetectEnvironment(explicit string, serveMode bool) string { + if e := strings.TrimSpace(explicit); e != "" { + return e + } + if serveMode { + return EnvServerlessFunction + } + if inContainer() { + return EnvContainer + } + return EnvLocal +} + +func inContainer() bool { + if _, err := os.Stat("/.dockerenv"); err == nil { + return true + } + data, err := os.ReadFile("/proc/1/cgroup") + if err != nil { + return false + } + s := string(data) + return strings.Contains(s, "docker") || strings.Contains(s, "containerd") || strings.Contains(s, "kubepods") +} diff --git a/internal/config/file.go b/internal/config/file.go new file mode 100644 index 0000000..fd59cbd --- /dev/null +++ b/internal/config/file.go @@ -0,0 +1,48 @@ +package config + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + + "gopkg.in/yaml.v3" +) + +// AutoConfig is the --config value that opts into the user config directory. +// Without it nothing outside the explicitly given paths is read, so a +// container run never depends on what happens to be in $HOME. +const AutoConfig = "auto" + +// loadFile reads a YAML config file whose keys are flag long names. It returns +// a nil map when no file applies. +func loadFile(path string) (map[string]any, string, error) { + if path == "" { + return nil, "", nil + } + + resolved := path + if path == AutoConfig { + dir, err := os.UserConfigDir() + if err != nil { + return nil, "", fmt.Errorf("config auto: %w", err) + } + resolved = filepath.Join(dir, "fastrecon", "config.yaml") + if _, err := os.Stat(resolved); errors.Is(err, fs.ErrNotExist) { + // An absent auto config is the normal case, not a failure. + return nil, "", nil + } + } + + data, err := os.ReadFile(resolved) + if err != nil { + return nil, "", fmt.Errorf("read config %s: %w", resolved, err) + } + + var raw map[string]any + if err := yaml.Unmarshal(data, &raw); err != nil { + return nil, "", fmt.Errorf("parse config %s: %w", resolved, err) + } + return raw, resolved, nil +} diff --git a/internal/config/flags.go b/internal/config/flags.go new file mode 100644 index 0000000..1160a82 --- /dev/null +++ b/internal/config/flags.go @@ -0,0 +1,97 @@ +package config + +import ( + "fmt" + "strings" + "time" + + "github.com/spf13/pflag" + + "github.com/JoshuaMart/FastRecon/internal/stage" +) + +// newlineOnlyFlags hold values that may legitimately contain a comma, so +// their environment form is split on newlines only. +var newlineOnlyFlags = map[string]bool{ + "probe-header": true, + "webhook-header": true, +} + +// patternFlags hold exclusion patterns, which need a rule of their own: a +// regexp's repeat count contains a comma, and splitting on it yields two +// halves that both still compile — so the run would quietly scan hosts the +// operator excluded. See splitPatterns. +var patternFlags = map[string]bool{"exclude": true} + +// RegisterFlags defines the full option surface. Defaults live here and +// nowhere else, so `--help` is the reference for what a run does by default. +func RegisterFlags(fs *pflag.FlagSet) { + fs.StringP("domain", "d", "", "root domain to enumerate (required)") + fs.StringArray("exclude", nil, "exclusion pattern: host, *.suffix, or re: (repeatable)") + fs.String("exclude-file", "", "file of exclusion patterns, one per line, # for comments") + fs.Bool("exclude-strict-wildcard", false, "*.x.example.com excludes hosts under x.example.com but not x.example.com itself") + fs.Bool("report-excluded", false, "list excluded hosts and the pattern that matched in the report") + + fs.String("stages", string(stage.ScopeFull), fmt.Sprintf("pipeline scope: %s", scopeList())) + fs.Duration("timeout", 30*time.Minute, "global deadline for the whole run") + fs.Float64("output-margin", 0.10, "fraction of the deadline reserved to build and deliver the report") + + fs.String("provider-config", "", "path to the source credentials file (never baked into the image)") + fs.StringArray("sources", RequiredSources, "enumeration sources to query (repeatable); see `fastrecon sources`") + fs.StringArray("exclude-sources", nil, "sources to remove from the selection (repeatable)") + fs.Bool("all-sources", false, "query every source the engine knows, not just the selection") + fs.Duration("source-timeout", 30*time.Second, "time ceiling for a single source, retries and backoff included; whole seconds only") + + fs.StringArray("resolvers", nil, "DNS resolver IP to use (repeatable); empty uses the bundled set") + fs.String("resolvers-file", "", "file of resolver IPs, one per line, # for comments") + fs.String("resolvers-url", "", "https URL of a resolver list, fetched at startup (30s ceiling); for deployments with no volume to mount") + fs.Bool("validate-resolvers", true, "drop resolvers that are unreachable, answer a known name wrongly, or hijack NXDOMAIN") + fs.Duration("resolver-health-budget", 30*time.Second, "ceiling on the resolver health check; resolvers not reached in time are kept and counted") + fs.Int("resolver-concurrency", 100, "concurrent DNS queries") + fs.Int("resolver-retries", 2, "extra attempts per DNS query after the first") + fs.Duration("resolver-timeout", 5*time.Second, "timeout per DNS query") + fs.Int("wildcard-probes", 3, "random names resolved per parent domain to detect a wildcard record") + + fs.String("scan-mode", ScanModeConnect, fmt.Sprintf("port scan mode: %s (unprivileged) or %s (needs CAP_NET_RAW)", ScanModeConnect, ScanModeSYN)) + fs.String("ports", "top-100", "ports to scan: top-100, top-1000, web, or a list like 80,443,8000-8100") + fs.String("exclude-ports", "", "ports to subtract from the selection") + fs.Bool("skip-cdn", true, "on CDN/WAF addresses, scan only the standard web ports; detection runs and is reported either way") + fs.Int("scan-concurrency", 200, "concurrent port connections") + fs.Int("scan-rate", 1000, "port scan packets per second") + fs.Duration("scan-timeout", 3*time.Second, "timeout per port connection") + fs.Int("scan-retries", 2, "retries per port") + + fs.Int("probe-concurrency", 50, "concurrent HTTP probes") + fs.Int("probe-rate", 200, "HTTP probes per second") + fs.Duration("probe-timeout", 10*time.Second, "timeout per HTTP probe") + fs.Int("probe-retries", 1, "retries per HTTP probe") + fs.Bool("probe-follow-redirects", false, "follow redirects while probing; the Location target is recorded either way") + fs.Int("probe-max-redirects", 5, "maximum redirect hops") + fs.String("probe-user-agent", "", "User-Agent sent while probing; empty uses the built-in one") + fs.StringArray("probe-header", nil, "extra header sent while probing, as 'Name: value' (repeatable)") + + fs.StringP("output", "o", StdoutPath, "report destination: a file path, or - for stdout") + fs.String("format", "json", "report format: json, jsonl, text") + fs.String("webhook-url", "", "POST the report as raw JSON to this URL") + fs.String("webhook-method", "POST", "HTTP method for the webhook") + fs.StringArray("webhook-header", nil, "extra header for the webhook, as 'Name: value' (repeatable)") + fs.Duration("webhook-timeout", 30*time.Second, "timeout per webhook attempt") + fs.Int("webhook-retries", 3, "webhook retries on 5xx, 429 and transport errors") + + fs.String("log-level", "info", "log verbosity: debug, info, warn, error") + fs.String("log-format", "json", "log format on stderr: json, text") + fs.String("environment", "", "environment label recorded in the report; empty auto-detects") + fs.String("config", "", "config file path, or 'auto' to look in the user config directory") + + fs.String("listen", ":8080", "serve mode: address to bind") + fs.String("api-token", "", "serve mode: shared token required on every request") +} + +func scopeList() string { + scopes := stage.Scopes() + parts := make([]string, len(scopes)) + for i, s := range scopes { + parts[i] = string(s) + } + return strings.Join(parts, ", ") +} diff --git a/internal/config/resolve.go b/internal/config/resolve.go new file mode 100644 index 0000000..9e67efc --- /dev/null +++ b/internal/config/resolve.go @@ -0,0 +1,367 @@ +package config + +import ( + "bufio" + "errors" + "fmt" + "os" + "strconv" + "strings" + "time" + + "github.com/spf13/pflag" + + "github.com/JoshuaMart/FastRecon/internal/report" + "github.com/JoshuaMart/FastRecon/internal/stage" +) + +// Load resolves a configuration from parsed flags, the environment and an +// optional config file, then validates it. +func Load(fs *pflag.FlagSet) (*Config, error) { + return resolveConfig(fs, true) +} + +// LoadServe is Load for the HTTP handler, where the domain is not known at +// startup: it arrives with each request and is validated then, by the same +// rules. +func LoadServe(fs *pflag.FlagSet) (*Config, error) { + return resolveConfig(fs, false) +} + +func resolveConfig(fs *pflag.FlagSet, requireDomain bool) (*Config, error) { + l := &loader{fs: fs} + + // The config file path itself can only come from a flag or the + // environment — it cannot be defined by the file it selects. + cfgPath := l.str("config") + file, resolvedPath, err := loadFile(cfgPath) + if err != nil { + return nil, err + } + l.file = file + + cfg := &Config{ConfigFile: resolvedPath} + + cfg.Domain = l.str("domain") + cfg.Exclude = l.strs("exclude") + cfg.ExcludeFile = l.str("exclude-file") + cfg.ExcludeStrictWildcard = l.bool("exclude-strict-wildcard") + cfg.ReportExcluded = l.bool("report-excluded") + + cfg.Timeout = l.dur("timeout") + cfg.OutputMargin = l.f64("output-margin") + + cfg.ProviderConfig = l.str("provider-config") + cfg.Sources = l.strs("sources") + cfg.ExcludeSources = l.strs("exclude-sources") + cfg.AllSources = l.bool("all-sources") + cfg.SourceTimeout = l.dur("source-timeout") + + cfg.Resolvers = l.strs("resolvers") + cfg.ResolversFile = l.str("resolvers-file") + cfg.ResolversURL = l.str("resolvers-url") + cfg.ValidateResolvers = l.bool("validate-resolvers") + cfg.ResolverHealthBudget = l.dur("resolver-health-budget") + cfg.ResolverConcurrency = l.int("resolver-concurrency") + cfg.ResolverRetries = l.int("resolver-retries") + cfg.ResolverTimeout = l.dur("resolver-timeout") + cfg.WildcardProbes = l.int("wildcard-probes") + + cfg.ScanMode = l.str("scan-mode") + cfg.Ports = l.str("ports") + cfg.ExcludePorts = l.str("exclude-ports") + cfg.SkipCDN = l.bool("skip-cdn") + cfg.ScanConcurrency = l.int("scan-concurrency") + cfg.ScanRate = l.int("scan-rate") + cfg.ScanTimeout = l.dur("scan-timeout") + cfg.ScanRetries = l.int("scan-retries") + + cfg.ProbeConcurrency = l.int("probe-concurrency") + cfg.ProbeRate = l.int("probe-rate") + cfg.ProbeTimeout = l.dur("probe-timeout") + cfg.ProbeRetries = l.int("probe-retries") + cfg.ProbeFollowRedirects = l.bool("probe-follow-redirects") + cfg.ProbeMaxRedirects = l.int("probe-max-redirects") + cfg.ProbeUserAgent = l.str("probe-user-agent") + cfg.ProbeHeaders = l.strs("probe-header") + + cfg.Output = l.str("output") + cfg.WebhookURL = l.str("webhook-url") + cfg.WebhookMethod = l.str("webhook-method") + cfg.WebhookHeaders = l.strs("webhook-header") + cfg.WebhookTimeout = l.dur("webhook-timeout") + cfg.WebhookRetries = l.int("webhook-retries") + + cfg.LogLevel = l.str("log-level") + cfg.LogFormat = l.str("log-format") + cfg.Environment = l.str("environment") + cfg.Listen = l.str("listen") + cfg.APIToken = l.str("api-token") + + // Values needing a parse step of their own. + if scope, err := stage.ParseScope(l.str("stages")); err != nil { + l.errs = append(l.errs, err) + } else { + cfg.Scope = scope + } + if format, err := report.ParseFormat(l.str("format")); err != nil { + l.errs = append(l.errs, err) + } else { + cfg.Format = format + } + if err := errors.Join(l.errs...); err != nil { + return nil, err + } + + if err := cfg.mergeExcludeFile(); err != nil { + return nil, err + } + l.warnUnknownKeys(cfg) + + if err := cfg.validate(requireDomain); err != nil { + return nil, err + } + return cfg, nil +} + +// mergeExcludeFile appends the patterns of --exclude-file to the inline ones. +// Parsing the patterns themselves belongs to the exclusion stage; here they +// are only collected. +func (c *Config) mergeExcludeFile() error { + if c.ExcludeFile == "" { + return nil + } + f, err := os.Open(c.ExcludeFile) + if err != nil { + return fmt.Errorf("read exclude file: %w", err) + } + defer func() { _ = f.Close() }() + + sc := bufio.NewScanner(f) + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + c.Exclude = append(c.Exclude, line) + } + if err := sc.Err(); err != nil { + return fmt.Errorf("read exclude file %s: %w", c.ExcludeFile, err) + } + return nil +} + +// loader reads one option at a time, applying the documented precedence. +type loader struct { + fs *pflag.FlagSet + file map[string]any + errs []error + // read records every key the loader looked up, so leftover file keys can + // be reported as probable typos. + read map[string]bool +} + +func (l *loader) note(name string) { + if l.read == nil { + l.read = map[string]bool{} + } + l.read[name] = true +} + +func (l *loader) errf(format string, args ...any) { + l.errs = append(l.errs, fmt.Errorf(format, args...)) +} + +// raw returns the highest-precedence override for a flag, if any. A nil +// result means "no override": use the flag's own value, which is either the +// parsed flag or its default. +func (l *loader) raw(name string) *string { + l.note(name) + if l.fs.Changed(name) { + return nil + } + if v, ok := lookupEnv(name); ok { + return &v + } + if v, ok := l.file[name]; ok { + s := fmt.Sprint(v) + return &s + } + return nil +} + +func (l *loader) str(name string) string { + v, err := l.fs.GetString(name) + if err != nil { + l.errf("flag %s: %w", name, err) + return "" + } + if o := l.raw(name); o != nil { + return *o + } + return v +} + +func (l *loader) bool(name string) bool { + v, err := l.fs.GetBool(name) + if err != nil { + l.errf("flag %s: %w", name, err) + return false + } + o := l.raw(name) + if o == nil { + return v + } + b, err := strconv.ParseBool(strings.TrimSpace(*o)) + if err != nil { + l.errf("%s: %q is not a boolean", name, *o) + return v + } + return b +} + +func (l *loader) int(name string) int { + v, err := l.fs.GetInt(name) + if err != nil { + l.errf("flag %s: %w", name, err) + return 0 + } + o := l.raw(name) + if o == nil { + return v + } + n, err := strconv.Atoi(strings.TrimSpace(*o)) + if err != nil { + l.errf("%s: %q is not an integer", name, *o) + return v + } + return n +} + +func (l *loader) f64(name string) float64 { + v, err := l.fs.GetFloat64(name) + if err != nil { + l.errf("flag %s: %w", name, err) + return 0 + } + o := l.raw(name) + if o == nil { + return v + } + f, err := strconv.ParseFloat(strings.TrimSpace(*o), 64) + if err != nil { + l.errf("%s: %q is not a number", name, *o) + return v + } + return f +} + +func (l *loader) dur(name string) time.Duration { + v, err := l.fs.GetDuration(name) + if err != nil { + l.errf("flag %s: %w", name, err) + return 0 + } + o := l.raw(name) + if o == nil { + return v + } + d, err := time.ParseDuration(strings.TrimSpace(*o)) + if err != nil { + l.errf("%s: %q is not a duration (e.g. 30s, 10m)", name, *o) + return v + } + return d +} + +// strs resolves a repeatable flag. The environment form accepts newline +// separation always, and comma separation for options whose values cannot +// contain a comma. The file form accepts a list or a single scalar. +func (l *loader) strs(name string) []string { + v, err := l.fs.GetStringArray(name) + if err != nil { + l.errf("flag %s: %w", name, err) + return nil + } + l.note(name) + if l.fs.Changed(name) { + return v + } + if s, ok := lookupEnv(name); ok { + return splitValue(name, s) + } + if raw, ok := l.file[name]; ok { + switch t := raw.(type) { + case []any: + out := make([]string, 0, len(t)) + for _, item := range t { + out = append(out, strings.TrimSpace(fmt.Sprint(item))) + } + return out + case string: + return splitValue(name, t) + default: + l.errf("%s: expected a list in the config file", name) + return v + } + } + return v +} + +// splitValue applies the flag's separator rule to an environment or config +// value. +func splitValue(name, value string) []string { + if patternFlags[name] { + return splitPatterns(value) + } + return splitList(value, !newlineOnlyFlags[name]) +} + +// splitPatterns separates exclusion patterns. +// +// Lines are the outer separator. A line starting with "re:" is a single +// pattern, kept whole — a regexp may contain commas, and cutting one produces +// two halves that still compile, so the mistake would be silent. Any other +// line is a comma-separated list of hosts and wildcards, which is what makes +// the compact form usable in a job's environment variable. +func splitPatterns(value string) []string { + var out []string + for line := range strings.SplitSeq(value, "\n") { + line = strings.TrimSpace(strings.TrimSuffix(line, "\r")) + if line == "" { + continue + } + if strings.HasPrefix(line, "re:") { + out = append(out, line) + continue + } + out = append(out, splitList(line, true)...) + } + return out +} + +func splitList(s string, splitComma bool) []string { + fields := strings.FieldsFunc(s, func(r rune) bool { + if r == '\n' || r == '\r' { + return true + } + return splitComma && r == ',' + }) + out := make([]string, 0, len(fields)) + for _, f := range fields { + if f = strings.TrimSpace(f); f != "" { + out = append(out, f) + } + } + return out +} + +// warnUnknownKeys flags config-file keys that match no option. A silently +// ignored key looks identical to a key that took effect, so it must be loud. +func (l *loader) warnUnknownKeys(cfg *Config) { + for k := range l.file { + if !l.read[k] { + cfg.warnf("config file key %q matches no option and was ignored", k) + } + } +} diff --git a/internal/config/validate.go b/internal/config/validate.go new file mode 100644 index 0000000..4689345 --- /dev/null +++ b/internal/config/validate.go @@ -0,0 +1,240 @@ +package config + +import ( + "errors" + "fmt" + "net/http" + "net/url" + "os" + "regexp" + "strings" + "time" +) + +var domainRE = regexp.MustCompile(`^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?(\.[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)+$`) + +// Validate normalizes the configuration and reports every problem at once, +// rather than one per run. +func (c *Config) Validate() error { return c.validate(true) } + +// validate optionally tolerates an empty domain, which is the serve-mode case: +// the target arrives with each request and is validated then, by these same +// rules. +func (c *Config) validate(requireDomain bool) error { + var errs []error + fail := func(format string, args ...any) { errs = append(errs, fmt.Errorf(format, args...)) } + + if requireDomain || c.Domain != "" { + domain, err := NormalizeDomain(c.Domain) + if err != nil { + fail("%w", err) + } + c.Domain = domain + } + + if c.Timeout <= 0 { + fail("timeout must be positive, got %s", c.Timeout) + } + if c.OutputMargin < 0 || c.OutputMargin >= 0.5 { + fail("output-margin must be in [0, 0.5), got %v", c.OutputMargin) + } + + switch c.ScanMode { + case ScanModeConnect, ScanModeSYN: + default: + fail("unknown scan-mode %q (valid: %s, %s)", c.ScanMode, ScanModeConnect, ScanModeSYN) + } + if c.Ports == "" { + fail("ports must not be empty") + } + + // Source names are matched case-sensitively by the engine, so they are + // normalized here, once, rather than at each of the several places that + // look them up. + c.Sources = lowerAll(c.Sources) + c.ExcludeSources = lowerAll(c.ExcludeSources) + if !c.AllSources && len(c.Sources) == 0 { + fail("no enumeration source selected: set --sources or --all-sources") + } + + // The enumeration engine takes its per-source ceiling in whole seconds, + // and truncates: anything under a second silently becomes no ceiling at + // all, letting one hung source consume the entire stage budget. + if c.SourceTimeout > 0 && c.SourceTimeout < time.Second { + fail("source-timeout must be at least 1s, got %s: the engine takes whole seconds and would round it to no timeout", c.SourceTimeout) + } + + for _, p := range []struct { + name string + v int + }{ + {"resolver-concurrency", c.ResolverConcurrency}, + {"wildcard-probes", c.WildcardProbes}, + {"scan-concurrency", c.ScanConcurrency}, + {"probe-concurrency", c.ProbeConcurrency}, + {"probe-rate", c.ProbeRate}, + } { + if p.v < 1 { + fail("%s must be at least 1, got %d", p.name, p.v) + } + } + for _, p := range []struct { + name string + v int + }{ + {"resolver-retries", c.ResolverRetries}, + {"probe-max-redirects", c.ProbeMaxRedirects}, + {"probe-retries", c.ProbeRetries}, + {"webhook-retries", c.WebhookRetries}, + {"scan-rate", c.ScanRate}, + {"scan-retries", c.ScanRetries}, + } { + if p.v < 0 { + fail("%s must not be negative, got %d", p.name, p.v) + } + } + for _, p := range []struct { + name string + v time.Duration + }{ + {"source-timeout", c.SourceTimeout}, + {"resolver-timeout", c.ResolverTimeout}, + {"resolver-health-budget", c.ResolverHealthBudget}, + {"scan-timeout", c.ScanTimeout}, + {"probe-timeout", c.ProbeTimeout}, + {"webhook-timeout", c.WebhookTimeout}, + } { + if p.v <= 0 { + fail("%s must be positive, got %s", p.name, p.v) + } + } + + if err := validateHeaders("probe-header", c.ProbeHeaders); err != nil { + errs = append(errs, err) + } + + if c.WebhookURL != "" { + if err := validateWebhookURL(c.WebhookURL); err != nil { + errs = append(errs, err) + } + if err := validateHeaders("webhook-header", c.WebhookHeaders); err != nil { + errs = append(errs, err) + } + c.WebhookMethod = strings.ToUpper(strings.TrimSpace(c.WebhookMethod)) + switch c.WebhookMethod { + case http.MethodPost, http.MethodPut, http.MethodPatch: + default: + fail("webhook-method %q is not a method that carries a body (valid: POST, PUT, PATCH)", c.WebhookMethod) + } + } else if len(c.WebhookHeaders) > 0 { + fail("webhook-header set without webhook-url") + } + + if c.Output != StdoutPath && c.Output != "" { + if info, err := os.Stat(c.Output); err == nil && info.IsDir() { + fail("output %q is a directory, expected a file path", c.Output) + } + } + if c.Output == "" && c.WebhookURL == "" { + fail("no destination configured: set --output or --webhook-url") + } + + switch strings.ToLower(c.LogLevel) { + case "debug", "info", "warn", "warning", "error": + default: + fail("unknown log-level %q (valid: debug, info, warn, error)", c.LogLevel) + } + switch strings.ToLower(c.LogFormat) { + case "json", "text": + default: + fail("unknown log-format %q (valid: json, text)", c.LogFormat) + } + + // stdout carries the report; sending logs there too would corrupt it. + if c.Output == StdoutPath && c.Format == "" { + fail("format must be set when writing to stdout") + } + + if c.ResolversFile != "" { + if _, err := os.Stat(c.ResolversFile); err != nil { + fail("resolvers-file %q is not readable: %v", c.ResolversFile, err) + } + } + + if c.ProviderConfig != "" { + if _, err := os.Stat(c.ProviderConfig); err != nil { + fail("provider-config %q is not readable: %v", c.ProviderConfig, err) + } + } + + return errors.Join(errs...) +} + +// NormalizeDomain lowercases a root domain and rejects anything that is not +// one: a URL, a path, a wildcard, a single label. +func NormalizeDomain(d string) (string, error) { + d = strings.TrimSpace(strings.ToLower(d)) + if d == "" { + return "", errors.New("domain is required (-d/--domain or FASTRECON_DOMAIN)") + } + if strings.Contains(d, "://") || strings.ContainsAny(d, "/ \t") { + return "", fmt.Errorf("domain %q must be a bare domain, not a URL", d) + } + d = strings.TrimSuffix(d, ".") + d = strings.TrimPrefix(d, "*.") + if !domainRE.MatchString(d) { + if hasNonASCII(d) { + return "", fmt.Errorf("domain %q must be ASCII; pass the punycode form (xn--...)", d) + } + return "", fmt.Errorf("domain %q is not a valid domain name", d) + } + return d, nil +} + +// lowerAll normalizes a list of names, dropping the empties. +func lowerAll(in []string) []string { + if len(in) == 0 { + return in + } + out := make([]string, 0, len(in)) + for _, v := range in { + if v = strings.ToLower(strings.TrimSpace(v)); v != "" { + out = append(out, v) + } + } + return out +} + +func hasNonASCII(s string) bool { + for _, r := range s { + if r > 127 { + return true + } + } + return false +} + +func validateWebhookURL(raw string) error { + u, err := url.Parse(raw) + if err != nil { + return fmt.Errorf("webhook-url %q is not a URL: %w", raw, err) + } + if u.Scheme != "http" && u.Scheme != "https" { + return fmt.Errorf("webhook-url %q must use http or https", raw) + } + if u.Host == "" { + return fmt.Errorf("webhook-url %q has no host", raw) + } + return nil +} + +func validateHeaders(flag string, headers []string) error { + var errs []error + for _, h := range headers { + name, value, ok := strings.Cut(h, ":") + if !ok || strings.TrimSpace(name) == "" || strings.TrimSpace(value) == "" { + errs = append(errs, fmt.Errorf("%s %q must be in 'Name: value' form", flag, h)) + } + } + return errors.Join(errs...) +} diff --git a/internal/enumerate/helper_test.go b/internal/enumerate/helper_test.go new file mode 100644 index 0000000..1dd1605 --- /dev/null +++ b/internal/enumerate/helper_test.go @@ -0,0 +1,10 @@ +package enumerate + +import ( + "log/slog" + "os" +) + +func discardLogger() *slog.Logger { + return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.Level(99)})) +} diff --git a/internal/enumerate/subfaster.go b/internal/enumerate/subfaster.go new file mode 100644 index 0000000..7167086 --- /dev/null +++ b/internal/enumerate/subfaster.go @@ -0,0 +1,415 @@ +// Package enumerate collects subdomains from passive sources. +// +// The engine is subfaster used as a Go library — a subfinder fork, so its +// provider-config format is the upstream one. Only its passive agent is used: +// the CLI runner would read a provider config from the user's home directory, +// which a container run must not depend on. +package enumerate + +import ( + "context" + "errors" + "fmt" + "log/slog" + "sort" + "strings" + "sync" + "time" + + "github.com/melvinsh/subfaster/v2/pkg/passive" + "github.com/melvinsh/subfaster/v2/pkg/subscraping" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/gologger/levels" + "golang.org/x/net/idna" + + "github.com/JoshuaMart/FastRecon/internal/pipeline" + "github.com/JoshuaMart/FastRecon/internal/report" + "github.com/JoshuaMart/FastRecon/internal/secrets" +) + +// defaultMaxEnumeration bounds an enumeration whose context has no deadline. +const defaultMaxEnumeration = 10 * time.Minute + +// Options configures the enumerator. +type Options struct { + // Sources to query. Ignored when All is set. + Sources []string + // ExcludeSources are removed from the selection. + ExcludeSources []string + // All queries every source the engine knows, not just the selection. + All bool + // SourceTimeout bounds a single source's HTTP session. + SourceTimeout time.Duration + Proxy string + Credentials map[string]secrets.Credential + Redactor *secrets.Redactor + Logger *slog.Logger +} + +// Subfaster is the subfaster-backed Enumerator. +type Subfaster struct { + opts Options +} + +var loggerOnce sync.Once + +// NewSubfaster validates the source selection and prepares the engine. +func NewSubfaster(opts Options) (*Subfaster, error) { + if opts.Logger == nil { + return nil, errors.New("enumerate: logger is required") + } + if opts.Redactor == nil { + opts.Redactor = secrets.NewRedactor(nil) + } + if opts.SourceTimeout <= 0 { + return nil, errors.New("enumerate: source timeout must be positive") + } + + // The engine's source lookup is case-sensitive, and hands an empty set + // straight to os.Exit. Normalizing here means a caller that skipped the + // configuration layer cannot walk into that. + opts.Sources = lowerAll(opts.Sources) + opts.ExcludeSources = lowerAll(opts.ExcludeSources) + + if err := validateSources(append(append([]string{}, opts.Sources...), opts.ExcludeSources...)); err != nil { + return nil, err + } + // The engine calls os.Exit when it ends up with an empty source list, so + // the selection has to be checked here rather than discovered there. + if !opts.All && len(effective(opts.Sources, opts.ExcludeSources)) == 0 { + return nil, errors.New("enumerate: no sources selected; every source is excluded") + } + + // Credentials are not published here: the engine reads them from the + // process environment, which is global, so they are exported once by the + // caller before any enumerator exists. Doing it per construction would + // mean a process serving several runs rewrote them under the one in + // flight. + + loggerOnce.Do(func() { + gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose) + gologger.DefaultLogger.SetWriter(bridge{log: opts.Logger}) + }) + + return &Subfaster{opts: opts}, nil +} + +// Name identifies the stage implementation. +func (s *Subfaster) Name() string { return "subfaster" } + +// Enumerate queries every selected source concurrently and returns the +// deduplicated, in-scope result. +// +// A source that fails, is skipped or runs out of time degrades the run: it is +// recorded with its status and the enumeration continues. Losing one source +// is not a reason to lose the other four. +func (s *Subfaster) Enumerate(ctx context.Context, domain string) (pipeline.Enumeration, error) { + agent := passive.New(s.sourceNames(), s.opts.ExcludeSources, s.opts.All, false) + + budget := enumerationBudget(ctx) + s.opts.Logger.Debug("enumeration started", + "sources", s.sourceNames(), + "all_sources", s.opts.All, + "budget", budget.String(), + "source_timeout", s.opts.SourceTimeout.String(), + ) + + results := agent.EnumerateSubdomainsWithCtx(ctx, domain, s.opts.Proxy, int(s.opts.SourceTimeout.Seconds()), budget) + + var ( + hosts []string + seen = map[string]struct{}{} + errsBySource = map[string][]string{} + outOfScope int + ) + + for res := range results { + switch res.Type { + case subscraping.Error: + if res.Error != nil { + errsBySource[res.Source] = append(errsBySource[res.Source], s.opts.Redactor.RedactError(res.Error)) + } + case subscraping.Subdomain: + host, ok := normalize(res.Value, domain) + if !ok { + outOfScope++ + continue + } + if _, dup := seen[host]; dup { + continue + } + seen[host] = struct{}{} + hosts = append(hosts, host) + } + } + sort.Strings(hosts) + + timedOut := ctx.Err() != nil + out := pipeline.Enumeration{ + Hosts: hosts, + Sources: s.sourceStatuses(agent.GetStatistics(), errsBySource, timedOut), + } + out.Truncated = timedOut + if outOfScope > 0 { + s.opts.Logger.Debug("results dropped", "reason", "out of scope or malformed", "count", outOfScope) + } + for _, src := range out.Sources { + if src.Status == report.SourceError || src.Status == report.SourceRateLimited { + out.Warnings = append(out.Warnings, fmt.Sprintf("source %s: %s", src.Name, src.Error)) + } + } + return out, nil +} + +// sourceStatuses turns the engine's per-source counters into report entries. +// Every selected source appears, successful or not: a source that silently +// contributes nothing is exactly what this accounting exists to expose. +func (s *Subfaster) sourceStatuses(stats map[string]subscraping.Statistics, errs map[string][]string, timedOut bool) []report.Source { + names := s.reportedSources(stats) + out := make([]report.Source, 0, len(names)) + + for _, name := range names { + st := stats[name] + entry := report.Source{ + Name: name, + Found: st.Results, + Duration: st.TimeTaken.Milliseconds(), + } + if msgs := errs[name]; len(msgs) > 0 { + entry.Error = strings.Join(dedupe(msgs), "; ") + } + + switch { + case st.Skipped: + entry.Status = report.SourceSkipped + if s.needsKey(name) && !s.hasKey(name) { + entry.Status = report.SourceSkippedNoKey + entry.Error = "" + } + case rateLimited(entry.Error): + entry.Status = report.SourceRateLimited + entry.Partial = st.Results > 0 + case st.Errors > 0 && st.Results == 0: + entry.Status = report.SourceError + case st.Errors > 0: + entry.Status = report.SourceOK + entry.Partial = true + case st.Results == 0 && timedOut: + entry.Status = report.SourceTimeout + default: + entry.Status = report.SourceOK + } + out = append(out, entry) + } + return out +} + +// reportedSources lists every source that should appear in the report: those +// the engine ran, plus any selected source it never reached. +func (s *Subfaster) reportedSources(stats map[string]subscraping.Statistics) []string { + set := map[string]struct{}{} + for name := range stats { + set[name] = struct{}{} + } + if !s.opts.All { + for _, name := range effective(s.opts.Sources, s.opts.ExcludeSources) { + set[name] = struct{}{} + } + } + names := make([]string, 0, len(set)) + for name := range set { + names = append(names, name) + } + sort.Strings(names) + return names +} + +func (s *Subfaster) sourceNames() []string { + if s.opts.All { + return nil + } + return s.opts.Sources +} + +// needsKey reports whether a source can use a credential at all. Optional-key +// sources count: they mark themselves skipped precisely when no key is +// configured, so "skipped_no_key" is the accurate reason for them too. +func (s *Subfaster) needsKey(name string) bool { + src, ok := passive.NameSourceMap[name] + if !ok { + return false + } + req := src.KeyRequirement() + return req == subscraping.RequiredKey || req == subscraping.OptionalKey +} + +func (s *Subfaster) hasKey(name string) bool { + _, ok := s.opts.Credentials[name] + return ok +} + +// enumerationBudget derives the engine's overall time limit from the stage +// deadline the pipeline handed down. +func enumerationBudget(ctx context.Context) time.Duration { + dl, ok := ctx.Deadline() + if !ok { + return defaultMaxEnumeration + } + if d := time.Until(dl); d > 0 { + return d + } + // Already out of time: let the engine close immediately rather than + // starting requests that cannot finish. + return time.Millisecond +} + +// normalize lowercases a result, strips the trailing dot and a wildcard +// label, and rejects anything outside the target domain. +func normalize(value, domain string) (string, bool) { + h := strings.ToLower(strings.TrimSpace(value)) + h = strings.TrimSuffix(h, ".") + h = strings.TrimPrefix(h, "*.") + if h == "" || strings.ContainsAny(h, " \t/:@") || strings.Contains(h, "..") { + return "", false + } + // Only convert when there is something to convert: the IDNA lookup + // profile rejects underscores, which are legitimate in names like + // _dmarc.example.com. + if hasNonASCII(h) { + ascii, err := idna.ToASCII(h) + if err != nil { + return "", false + } + h = ascii + } + if h == domain { + return h, true + } + if !strings.HasSuffix(h, "."+domain) { + return "", false + } + return h, true +} + +func hasNonASCII(s string) bool { + for _, r := range s { + if r > 127 { + return true + } + } + return false +} + +// rateLimited recognises the throttling responses sources use, so a run can +// tell a source that refused to answer from one that had nothing to say. +func rateLimited(msg string) bool { + if msg == "" { + return false + } + lower := strings.ToLower(msg) + for _, marker := range []string{"429", "rate limit", "rate-limit", "ratelimit", "too many requests", "quota"} { + if strings.Contains(lower, marker) { + return true + } + } + return false +} + +func dedupe(in []string) []string { + seen := make(map[string]struct{}, len(in)) + out := make([]string, 0, len(in)) + for _, s := range in { + if _, ok := seen[s]; ok { + continue + } + seen[s] = struct{}{} + out = append(out, s) + } + return out +} + +// effective returns the selection minus the exclusions. +func effective(sources, excluded []string) []string { + drop := make(map[string]struct{}, len(excluded)) + for _, e := range excluded { + drop[strings.ToLower(e)] = struct{}{} + } + out := make([]string, 0, len(sources)) + for _, s := range sources { + s = strings.ToLower(s) + if _, ok := drop[s]; ok { + continue + } + out = append(out, s) + } + return out +} + +// lowerAll normalizes source names, dropping the empties. +func lowerAll(in []string) []string { + out := make([]string, 0, len(in)) + for _, v := range in { + if v = strings.ToLower(strings.TrimSpace(v)); v != "" { + out = append(out, v) + } + } + return out +} + +func validateSources(names []string) error { + var unknown []string + for _, n := range names { + if _, ok := passive.NameSourceMap[strings.ToLower(n)]; !ok { + unknown = append(unknown, n) + } + } + if len(unknown) == 0 { + return nil + } + return fmt.Errorf("unknown source(s): %s (run `fastrecon sources` for the list)", strings.Join(unknown, ", ")) +} + +// Available lists every source the engine knows, with its key requirement. +func Available() []SourceInfo { + out := make([]SourceInfo, 0, len(passive.NameSourceMap)) + for name, src := range passive.NameSourceMap { + info := SourceInfo{Name: name, Default: src.IsDefault()} + switch src.KeyRequirement() { + case subscraping.RequiredKey: + info.Key = "required" + case subscraping.OptionalKey: + info.Key = "optional" + default: + info.Key = "none" + } + out = append(out, info) + } + sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) + return out +} + +// SourceInfo describes an available source. +type SourceInfo struct { + Name string + Key string + Default bool +} + +// bridge routes the engine's own logging into the structured logger on +// stderr. stdout carries the report and must stay parseable, and unstructured +// lines interleaved with JSON logs are unreadable in a log-only environment. +type bridge struct{ log *slog.Logger } + +func (b bridge) Write(data []byte, level levels.Level) { + msg := strings.TrimSpace(string(data)) + if msg == "" { + return + } + // A source-level error is not a run failure, so it never rises above a + // warning here; the report's per-source status is the real signal. + if level == levels.LevelFatal || level == levels.LevelError { + b.log.Warn("enumeration engine", "level", level.String(), "msg", msg) + return + } + b.log.Debug("enumeration engine", "level", level.String(), "msg", msg) +} diff --git a/internal/enumerate/subfaster_test.go b/internal/enumerate/subfaster_test.go new file mode 100644 index 0000000..7625aef --- /dev/null +++ b/internal/enumerate/subfaster_test.go @@ -0,0 +1,228 @@ +package enumerate + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/melvinsh/subfaster/v2/pkg/subscraping" + + "github.com/JoshuaMart/FastRecon/internal/report" + "github.com/JoshuaMart/FastRecon/internal/secrets" +) + +func TestNormalize(t *testing.T) { + const domain = "example.com" + + ok := map[string]string{ + "API.Example.com": "api.example.com", + "api.example.com.": "api.example.com", + "*.dev.example.com": "dev.example.com", + " api.example.com ": "api.example.com", + "example.com": "example.com", + // Underscores are legitimate in DNS names and must survive. + "_dmarc.example.com": "_dmarc.example.com", + } + for in, want := range ok { + got, valid := normalize(in, domain) + if !valid { + t.Errorf("normalize(%q) rejected the value", in) + continue + } + if got != want { + t.Errorf("normalize(%q) = %q, want %q", in, got, want) + } + } + + // Out of scope, or not a hostname at all. + for _, in := range []string{ + "", "notexample.com", "example.com.evil.net", "evil.net", + "http://api.example.com", "api.example.com:443", "user@example.com", + "api..example.com", + } { + if got, valid := normalize(in, domain); valid { + t.Errorf("normalize(%q) = %q, want it rejected", in, got) + } + } +} + +func TestNormalizeConvertsInternationalizedNames(t *testing.T) { + got, ok := normalize("café.example.com", "example.com") + if !ok { + t.Fatal("an internationalized name was rejected") + } + if !strings.HasPrefix(got, "xn--") { + t.Errorf("normalize = %q, want the punycode form", got) + } +} + +func TestRateLimited(t *testing.T) { + for _, in := range []string{ + "unexpected status code 429", + "Rate limit exceeded", + "too many requests, retry later", + "monthly quota reached", + } { + if !rateLimited(in) { + t.Errorf("rateLimited(%q) = false, want true", in) + } + } + for _, in := range []string{"", "connection refused", "unexpected status code 500"} { + if rateLimited(in) { + t.Errorf("rateLimited(%q) = true, want false", in) + } + } +} + +func TestEffectiveRemovesExclusions(t *testing.T) { + got := effective([]string{"chaos", "CRT", "submd"}, []string{"crt"}) + if len(got) != 2 || got[0] != "chaos" || got[1] != "submd" { + t.Errorf("effective = %v, want chaos and submd", got) + } +} + +func TestValidateSourcesNamesTheUnknownOnes(t *testing.T) { + err := validateSources([]string{"chaos", "notasource", "crt"}) + if err == nil { + t.Fatal("validateSources accepted an unknown source") + } + if !strings.Contains(err.Error(), "notasource") { + t.Errorf("error %q does not name the offending source", err) + } + if strings.Contains(err.Error(), "chaos") { + t.Errorf("error %q blames a valid source", err) + } + if err := validateSources([]string{"chaos", "crt", "submd", "c99", "securitytrails"}); err != nil { + t.Errorf("the required sources must all exist: %v", err) + } +} + +func statuses(t *testing.T, s *Subfaster, stats map[string]subscraping.Statistics, errs map[string][]string, timedOut bool) map[string]report.Source { + t.Helper() + out := map[string]report.Source{} + for _, src := range s.sourceStatuses(stats, errs, timedOut) { + out[src.Name] = src + } + return out +} + +func TestSourceStatusMapping(t *testing.T) { + s := &Subfaster{opts: Options{ + Sources: []string{"chaos", "crt", "submd", "c99", "securitytrails"}, + Credentials: map[string]secrets.Credential{"c99": {Source: "c99", Value: "k"}}, + }} + + stats := map[string]subscraping.Statistics{ + "crt": {Results: 30, TimeTaken: 150 * time.Millisecond}, + "submd": {Results: 18, Errors: 2}, + "chaos": {Skipped: true}, + "c99": {Errors: 1}, + "securitytrails": {}, + } + errs := map[string][]string{ + "c99": {"unexpected status code 429"}, + "submd": {"one page failed"}, + } + + got := statuses(t, s, stats, errs, true) + + if got["crt"].Status != report.SourceOK || got["crt"].Found != 30 { + t.Errorf("crt = %+v, want ok with 30 results", got["crt"]) + } + // Errors alongside results is a degraded success, not a failure. + if got["submd"].Status != report.SourceOK || !got["submd"].Partial { + t.Errorf("submd = %+v, want ok and partial", got["submd"]) + } + // A keyed source with no credential is skipped for a known reason. + if got["chaos"].Status != report.SourceSkippedNoKey { + t.Errorf("chaos = %+v, want skipped_no_key", got["chaos"]) + } + // Throttling is not a generic error: it says the source refused, not that + // it had nothing. + if got["c99"].Status != report.SourceRateLimited { + t.Errorf("c99 = %+v, want rate_limited", got["c99"]) + } + // Nothing at all, while the run was out of time. + if got["securitytrails"].Status != report.SourceTimeout { + t.Errorf("securitytrails = %+v, want timeout", got["securitytrails"]) + } +} + +func TestSourceStatusErrorWithoutResultsIsAFailure(t *testing.T) { + s := &Subfaster{opts: Options{Sources: []string{"crt"}}} + got := statuses(t, s, map[string]subscraping.Statistics{"crt": {Errors: 3}}, map[string][]string{"crt": {"boom"}}, false) + if got["crt"].Status != report.SourceError { + t.Errorf("crt = %+v, want error", got["crt"]) + } + if got["crt"].Error != "boom" { + t.Errorf("error = %q, want the underlying message", got["crt"].Error) + } +} + +// A selected source the engine never reported on must still appear, otherwise +// a source that vanished looks the same as one that was never asked. +func TestSelectedSourcesAlwaysAppear(t *testing.T) { + s := &Subfaster{opts: Options{Sources: []string{"chaos", "crt"}}} + got := statuses(t, s, map[string]subscraping.Statistics{}, nil, false) + if len(got) != 2 { + t.Errorf("sources = %v, want both selected sources present", got) + } +} + +func TestEnumerationBudgetFollowsTheStageDeadline(t *testing.T) { + if got := enumerationBudget(context.Background()); got != defaultMaxEnumeration { + t.Errorf("budget without a deadline = %s, want the default", got) + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + if got := enumerationBudget(ctx); got <= 0 || got > time.Minute { + t.Errorf("budget = %s, want it bounded by the stage deadline", got) + } + + expired, cancel2 := context.WithTimeout(context.Background(), -time.Second) + defer cancel2() + if got := enumerationBudget(expired); got > time.Millisecond { + t.Errorf("budget past the deadline = %s, want it to close immediately", got) + } +} + +// The engine calls os.Exit when it is handed an empty source list. +func TestEmptySelectionIsRejectedBeforeReachingTheEngine(t *testing.T) { + _, err := NewSubfaster(Options{ + Sources: []string{"crt"}, + ExcludeSources: []string{"crt"}, + SourceTimeout: time.Second, + Logger: discardLogger(), + }) + if err == nil { + t.Fatal("an empty source selection was accepted") + } + if !strings.Contains(err.Error(), "no sources selected") { + t.Errorf("error = %q, want it to explain the empty selection", err) + } +} + +func TestAvailableListsTheRequiredSources(t *testing.T) { + byName := map[string]SourceInfo{} + for _, s := range Available() { + byName[s.Name] = s + } + for _, want := range []string{"chaos", "securitytrails", "c99", "submd", "crt"} { + if _, ok := byName[want]; !ok { + t.Errorf("source %q is missing from the engine", want) + } + } + if byName["chaos"].Key != "required" { + t.Errorf("chaos key requirement = %q, want required", byName["chaos"].Key) + } + // crt.name and sub.md work without a key but accept one for better + // results, which is why an enumeration with no credentials at all still + // returns data. + for _, name := range []string{"crt", "submd"} { + if byName[name].Key != "optional" { + t.Errorf("%s key requirement = %q, want optional", name, byName[name].Key) + } + } +} diff --git a/internal/exclude/exclude.go b/internal/exclude/exclude.go new file mode 100644 index 0000000..09028be --- /dev/null +++ b/internal/exclude/exclude.go @@ -0,0 +1,156 @@ +// Package exclude removes out-of-scope hosts before any network activity. +// +// Exclusions are applied to the enumeration output, so an excluded host is +// never resolved, scanned or probed — the point is to not touch it at all, +// not merely to hide it from the report. +package exclude + +import ( + "errors" + "fmt" + "regexp" + "strings" + + "github.com/JoshuaMart/FastRecon/internal/pipeline" + "github.com/JoshuaMart/FastRecon/internal/report" +) + +// RegexPrefix marks a pattern as a regular expression. +const RegexPrefix = "re:" + +type kind int + +const ( + kindExact kind = iota + kindWildcard + kindRegex +) + +type pattern struct { + raw string + kind kind + value string // exact host, or the suffix behind a wildcard + re *regexp.Regexp + hits int +} + +// Matcher applies a set of exclusion patterns. +type Matcher struct { + patterns []*pattern + // strictWildcard makes *.x.example.com exclude hosts under x.example.com + // without excluding x.example.com itself. + strictWildcard bool +} + +// New compiles the patterns. Every problem is reported at once, so a bad +// exclusion list is fixed in one pass rather than one error per run. +func New(patterns []string, strictWildcard bool) (*Matcher, error) { + m := &Matcher{strictWildcard: strictWildcard} + var errs []error + seen := make(map[string]bool, len(patterns)) + + for _, raw := range patterns { + raw = strings.TrimSpace(raw) + if raw == "" { + continue + } + if seen[raw] { + continue + } + seen[raw] = true + + p, err := compile(raw) + if err != nil { + errs = append(errs, err) + continue + } + m.patterns = append(m.patterns, p) + } + if err := errors.Join(errs...); err != nil { + return nil, err + } + return m, nil +} + +func compile(raw string) (*pattern, error) { + if expr, ok := strings.CutPrefix(raw, RegexPrefix); ok { + // (?i) up front: host matching is case-insensitive throughout, and a + // pattern that behaved differently from the other two forms would be + // a trap. + re, err := regexp.Compile("(?i)" + expr) + if err != nil { + return nil, fmt.Errorf("exclusion pattern %q: %w", raw, err) + } + return &pattern{raw: raw, kind: kindRegex, re: re}, nil + } + + lower := strings.ToLower(strings.TrimSuffix(raw, ".")) + if suffix, ok := strings.CutPrefix(lower, "*."); ok { + if suffix == "" { + return nil, fmt.Errorf("exclusion pattern %q has nothing behind the wildcard", raw) + } + return &pattern{raw: raw, kind: kindWildcard, value: suffix}, nil + } + if strings.Contains(lower, "*") { + return nil, fmt.Errorf("exclusion pattern %q: a wildcard is only supported as a leading '*.' label, use %s for anything else", raw, RegexPrefix) + } + if lower == "" { + return nil, fmt.Errorf("exclusion pattern %q is empty", raw) + } + return &pattern{raw: raw, kind: kindExact, value: lower}, nil +} + +// Name identifies the stage implementation. +func (m *Matcher) Name() string { return "patterns" } + +// Len returns the number of compiled patterns. +func (m *Matcher) Len() int { return len(m.patterns) } + +// Filter splits hosts into those kept and those excluded, recording which +// pattern removed each host and which patterns matched nothing. +func (m *Matcher) Filter(hosts []string) pipeline.Filtered { + out := pipeline.Filtered{Kept: make([]string, 0, len(hosts))} + + for _, host := range hosts { + if p := m.match(host); p != nil { + p.hits++ + out.Removed = append(out.Removed, report.Excluded{Host: host, Pattern: p.raw}) + continue + } + out.Kept = append(out.Kept, host) + } + + // A pattern that matched nothing is almost always a typo, and a typo in + // an exclusion means hosts were scanned that should not have been. + for _, p := range m.patterns { + if p.hits == 0 { + out.Unused = append(out.Unused, p.raw) + } + } + return out +} + +// match returns the first pattern excluding host, or nil. +func (m *Matcher) match(host string) *pattern { + h := strings.ToLower(strings.TrimSuffix(host, ".")) + for _, p := range m.patterns { + switch p.kind { + case kindExact: + if h == p.value { + return p + } + case kindWildcard: + if strings.HasSuffix(h, "."+p.value) { + return p + } + if !m.strictWildcard && h == p.value { + return p + } + case kindRegex: + if p.re.MatchString(h) { + return p + } + } + } + return nil +} diff --git a/internal/exclude/exclude_test.go b/internal/exclude/exclude_test.go new file mode 100644 index 0000000..08225af --- /dev/null +++ b/internal/exclude/exclude_test.go @@ -0,0 +1,132 @@ +package exclude + +import ( + "slices" + "strings" + "testing" +) + +func kept(t *testing.T, patterns []string, strict bool, hosts ...string) []string { + t.Helper() + m, err := New(patterns, strict) + if err != nil { + t.Fatalf("New(%v): %v", patterns, err) + } + return m.Filter(hosts).Kept +} + +func TestExactMatchIsCaseInsensitive(t *testing.T) { + got := kept(t, []string{"Admin.Example.com"}, false, "admin.example.com", "api.example.com") + if !slices.Equal(got, []string{"api.example.com"}) { + t.Errorf("kept = %v, want only api.example.com", got) + } +} + +// A trailing dot is the same host; an exclusion that missed it would leave a +// host in scope that the operator believed was excluded. +func TestTrailingDotIsNormalized(t *testing.T) { + got := kept(t, []string{"admin.example.com."}, false, "admin.example.com.", "api.example.com") + if !slices.Equal(got, []string{"api.example.com"}) { + t.Errorf("kept = %v, want only api.example.com", got) + } +} + +func TestWildcardCoversTheBaseByDefault(t *testing.T) { + hosts := []string{"dev.example.com", "a.dev.example.com", "deep.a.dev.example.com", "devil.example.com", "api.example.com"} + + got := kept(t, []string{"*.dev.example.com"}, false, hosts...) + if !slices.Equal(got, []string{"devil.example.com", "api.example.com"}) { + t.Errorf("kept = %v, want the base and its children excluded, and devil.example.com untouched", got) + } + + // Strict mode keeps the base itself in scope. + got = kept(t, []string{"*.dev.example.com"}, true, hosts...) + if !slices.Equal(got, []string{"dev.example.com", "devil.example.com", "api.example.com"}) { + t.Errorf("kept = %v, want the base kept under strict wildcards", got) + } +} + +func TestRegexPatternIsCaseInsensitive(t *testing.T) { + got := kept(t, []string{`re:^(staging|preprod)[0-9]*\.`}, false, + "staging.example.com", "STAGING2.example.com", "preprod.example.com", "prod.example.com") + if !slices.Equal(got, []string{"prod.example.com"}) { + t.Errorf("kept = %v, want only prod.example.com", got) + } +} + +func TestFilterRecordsThePatternThatMatched(t *testing.T) { + m, err := New([]string{"*.dev.example.com", "admin.example.com"}, false) + if err != nil { + t.Fatal(err) + } + res := m.Filter([]string{"a.dev.example.com", "admin.example.com", "api.example.com"}) + + if len(res.Removed) != 2 { + t.Fatalf("removed = %v, want 2", res.Removed) + } + byHost := map[string]string{} + for _, r := range res.Removed { + byHost[r.Host] = r.Pattern + } + if byHost["a.dev.example.com"] != "*.dev.example.com" { + t.Errorf("wrong pattern recorded: %v", byHost) + } + if byHost["admin.example.com"] != "admin.example.com" { + t.Errorf("wrong pattern recorded: %v", byHost) + } +} + +// A pattern matching nothing usually means a typo, and a typo in an exclusion +// means hosts got scanned that should not have been. +func TestUnusedPatternsAreReported(t *testing.T) { + m, err := New([]string{"*.dev.example.com", "typo.example.com"}, false) + if err != nil { + t.Fatal(err) + } + res := m.Filter([]string{"a.dev.example.com"}) + if !slices.Equal(res.Unused, []string{"typo.example.com"}) { + t.Errorf("unused = %v, want the typo pattern", res.Unused) + } +} + +func TestDuplicatePatternsAreCollapsed(t *testing.T) { + m, err := New([]string{"admin.example.com", "admin.example.com", " ", ""}, false) + if err != nil { + t.Fatal(err) + } + if m.Len() != 1 { + t.Errorf("Len() = %d, want 1", m.Len()) + } +} + +func TestInvalidPatternsAreAllReportedAtOnce(t *testing.T) { + _, err := New([]string{"re:[unclosed", "mid*dle.example.com", "*."}, false) + if err == nil { + t.Fatal("New succeeded on invalid patterns") + } + msg := err.Error() + for _, want := range []string{"[unclosed", "mid*dle.example.com", "*."} { + if !strings.Contains(msg, want) { + t.Errorf("error %q does not mention %q", msg, want) + } + } +} + +// A bare '*' inside a name is a common mistake; guessing at its meaning would +// silently exclude the wrong hosts. +func TestMidNameWildcardIsRejectedWithGuidance(t *testing.T) { + _, err := New([]string{"api-*.example.com"}, false) + if err == nil { + t.Fatal("mid-name wildcard accepted") + } + if !strings.Contains(err.Error(), RegexPrefix) { + t.Errorf("error %q should point at the regex form", err) + } +} + +func TestNoPatternsKeepsEverything(t *testing.T) { + got := kept(t, nil, false, "a.example.com", "b.example.com") + if len(got) != 2 { + t.Errorf("kept = %v, want both hosts", got) + } +} diff --git a/internal/logging/logging.go b/internal/logging/logging.go new file mode 100644 index 0000000..54fd4c4 --- /dev/null +++ b/internal/logging/logging.go @@ -0,0 +1,52 @@ +// Package logging configures the process logger. +// +// Everything goes to stderr, unconditionally: stdout carries the report and +// must stay parseable even when the run is piped into another tool. +package logging + +import ( + "fmt" + "log/slog" + "os" + "strings" +) + +// New builds a logger writing to stderr. format is "json" or "text". +func New(level, format string) (*slog.Logger, error) { + lvl, err := parseLevel(level) + if err != nil { + return nil, err + } + opts := &slog.HandlerOptions{Level: lvl} + + var h slog.Handler + switch strings.ToLower(format) { + case "json": + h = slog.NewJSONHandler(os.Stderr, opts) + case "text": + h = slog.NewTextHandler(os.Stderr, opts) + default: + return nil, fmt.Errorf("unknown log format %q (valid: json, text)", format) + } + return slog.New(h), nil +} + +func parseLevel(s string) (slog.Level, error) { + switch strings.ToLower(s) { + case "debug": + return slog.LevelDebug, nil + case "info": + return slog.LevelInfo, nil + case "warn", "warning": + return slog.LevelWarn, nil + case "error": + return slog.LevelError, nil + default: + return 0, fmt.Errorf("unknown log level %q (valid: debug, info, warn, error)", s) + } +} + +// Discard returns a logger that drops everything, for tests. +func Discard() *slog.Logger { + return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.Level(99)})) +} diff --git a/internal/pipeline/budget.go b/internal/pipeline/budget.go new file mode 100644 index 0000000..b5d94a8 --- /dev/null +++ b/internal/pipeline/budget.go @@ -0,0 +1,83 @@ +package pipeline + +import ( + "time" + + "github.com/JoshuaMart/FastRecon/internal/stage" +) + +// Stage weights, used to split the remaining time. They are relative, not +// absolute: what matters is their ratio. +// +// Exclusion has weight zero — it is pure CPU work on an in-memory list, so it +// gets no dedicated slice and is bounded only by the global deadline. +var weights = map[stage.Stage]int{ + stage.Enumerate: 25, + stage.Exclude: 0, + stage.Resolve: 25, + stage.PortScan: 30, + stage.HTTPProbe: 20, +} + +// Budget splits the run deadline across the stages still to come. +// +// Allocation is dynamic rather than fixed up front: each stage receives a +// share of the time *actually* left, so an enumeration that finishes early +// hands its unused seconds to the port scan instead of wasting them. +type Budget struct { + deadline time.Time + remaining []stage.Stage + now func() time.Time +} + +// NewBudget builds a budget over the stages of a scope. +func NewBudget(stages []stage.Stage, deadline time.Time) *Budget { + return &Budget{ + deadline: deadline, + remaining: append([]stage.Stage(nil), stages...), + now: time.Now, + } +} + +// Left is the time until the run deadline, never negative. +func (b *Budget) Left() time.Duration { + return max(b.deadline.Sub(b.now()), 0) +} + +// Expired reports whether the deadline has passed. +func (b *Budget) Expired() bool { return b.Left() <= 0 } + +// Deadline returns the run deadline. +func (b *Budget) Deadline() time.Time { return b.deadline } + +// Take allocates s its share of the remaining time and marks it, and every +// stage before it, as consumed. A zero-weight stage gets everything that is +// left, meaning "no dedicated budget". +func (b *Budget) Take(s stage.Stage) time.Duration { + left := b.Left() + total := 0 + found := false + for _, r := range b.remaining { + if r == s { + found = true + } + if found { + total += weights[r] + } + } + b.consume(s) + + if !found || total == 0 || weights[s] == 0 { + return left + } + return time.Duration(float64(left) * float64(weights[s]) / float64(total)) +} + +func (b *Budget) consume(s stage.Stage) { + for i, r := range b.remaining { + if r == s { + b.remaining = b.remaining[i+1:] + return + } + } +} diff --git a/internal/pipeline/budget_test.go b/internal/pipeline/budget_test.go new file mode 100644 index 0000000..752247b --- /dev/null +++ b/internal/pipeline/budget_test.go @@ -0,0 +1,63 @@ +package pipeline + +import ( + "testing" + "time" + + "github.com/JoshuaMart/FastRecon/internal/stage" +) + +func fixedBudget(stages []stage.Stage, total time.Duration) (*Budget, *time.Time) { + now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + clock := now + b := NewBudget(stages, now.Add(total)) + b.now = func() time.Time { return clock } + return b, &clock +} + +func TestTakeSplitsByWeight(t *testing.T) { + b, _ := fixedBudget(stage.ScopeFull.Stages(), 100*time.Minute) + + // enumerate weighs 25 of the 100 total across the full ladder. + if got := b.Take(stage.Enumerate); got != 25*time.Minute { + t.Errorf("enumerate budget = %s, want 25m", got) + } + // exclude is CPU-only: no dedicated slice, bounded by the deadline. + if got := b.Take(stage.Exclude); got != 100*time.Minute { + t.Errorf("exclude budget = %s, want the whole remaining window", got) + } +} + +func TestTakeReallocatesTimeUnusedByEarlierStages(t *testing.T) { + stages := stage.ScopeFull.Stages() + b, clock := fixedBudget(stages, 100*time.Minute) + + b.Take(stage.Enumerate) + b.Take(stage.Exclude) + // Enumeration finished instantly, so resolve should get a share of the + // full 100 minutes rather than of the 75 it was nominally left. + got := b.Take(stage.Resolve) + if got != time.Duration(float64(100*time.Minute)*25.0/75.0) { + t.Errorf("resolve budget = %s, want 25/75 of the remaining 100m", got) + } + + // And a stage that overran must shrink what follows. + *clock = clock.Add(90 * time.Minute) + if got := b.Take(stage.PortScan); got >= 10*time.Minute { + t.Errorf("portscan budget = %s, want less than the 10m actually left", got) + } +} + +func TestExpired(t *testing.T) { + b, clock := fixedBudget(stage.ScopeEnum.Stages(), time.Minute) + if b.Expired() { + t.Fatal("budget expired before any time passed") + } + *clock = clock.Add(2 * time.Minute) + if !b.Expired() { + t.Error("budget not expired past its deadline") + } + if got := b.Left(); got != 0 { + t.Errorf("Left() = %s past the deadline, want 0", got) + } +} diff --git a/internal/pipeline/pipeline.go b/internal/pipeline/pipeline.go new file mode 100644 index 0000000..1b8d0ec --- /dev/null +++ b/internal/pipeline/pipeline.go @@ -0,0 +1,365 @@ +// Package pipeline orchestrates the recon stages. +// +// The pipeline owns the ladder, the deadline budgeting and the report; the +// stages themselves are interfaces, so each one can be built, replaced or +// tested without touching the orchestration. +package pipeline + +import ( + "context" + "errors" + "log/slog" + "time" + + "github.com/JoshuaMart/FastRecon/internal/config" + "github.com/JoshuaMart/FastRecon/internal/report" + "github.com/JoshuaMart/FastRecon/internal/runid" + "github.com/JoshuaMart/FastRecon/internal/stage" + "github.com/JoshuaMart/FastRecon/internal/version" +) + +// Partial is embedded by every stage result. A stage can finish without +// having been exhaustive — a source timed out, half the hosts were resolved +// before the budget ran out — and that has to reach the report, because a +// truncated result that claims to be complete is worse than no result. +type Partial struct { + // Warnings are non-fatal problems worth putting in the report. + Warnings []string + // Truncated marks a stage cut short by its deadline. + Truncated bool +} + +// Enumeration is what an Enumerator produces: the hosts it found, plus the +// per-source accounting that makes a silently empty source visible. +type Enumeration struct { + Partial + Hosts []string + Sources []report.Source +} + +// Resolution is what a Resolver produces: every host it was given, each with +// its verdict. Dead hosts are kept — a dangling CNAME is a finding, not noise. +type Resolution struct { + Partial + Hosts []report.Host +} + +// Filtered is the outcome of applying the exclusion patterns. +type Filtered struct { + Kept []string + Removed []report.Excluded + // Unused lists patterns that matched nothing — almost always a typo. + Unused []string +} + +// Enumerator collects subdomains from passive sources. +type Enumerator interface { + Name() string + Enumerate(ctx context.Context, domain string) (Enumeration, error) +} + +// Excluder drops out-of-scope hosts before any network activity happens. +type Excluder interface { + Name() string + Filter(hosts []string) Filtered +} + +// Resolver splits hosts into live and dead. +type Resolver interface { + Name() string + Resolve(ctx context.Context, hosts []string) (Resolution, error) +} + +// PortScan is what a PortScanner produces: every host it was given, the live +// ones enriched with their open ports and the CDN determination. +type PortScan struct { + Partial + Hosts []report.Host +} + +// PortScanner enriches live hosts with their open ports. +type PortScanner interface { + Name() string + Scan(ctx context.Context, hosts []report.Host) (PortScan, error) +} + +// Probe is what a Prober produces: the hosts, with the open ports that +// answered HTTP carrying their service details. +type Probe struct { + Partial + Hosts []report.Host +} + +// Prober enriches open ports with the HTTP service behind them. +type Prober interface { + Name() string + Probe(ctx context.Context, hosts []report.Host) (Probe, error) +} + +// Stages holds the implementations wired into a run. A nil field means the +// stage is not available in this build: the run stops there and says so, +// rather than reporting an empty result as if it were a finding. +type Stages struct { + Enumerator Enumerator + Excluder Excluder + Resolver Resolver + PortScanner PortScanner + Prober Prober +} + +// Pipeline runs one recon job. +type Pipeline struct { + cfg *config.Config + stages Stages + log *slog.Logger + now func() time.Time +} + +// New builds a pipeline. +func New(cfg *config.Config, stages Stages, log *slog.Logger) *Pipeline { + return &Pipeline{cfg: cfg, stages: stages, log: log, now: time.Now} +} + +// ErrNoImplementation is returned by a stage that is not part of this build. +var ErrNoImplementation = errors.New("stage not implemented in this build") + +// Run executes the scope's stages and always returns a report. +// +// A stage failure or an expired deadline stops the ladder — every later stage +// consumes the previous one's output — but the report produced so far is +// still returned, marked incomplete. A run that ran out of time is data, not +// an error. +func (p *Pipeline) Run(ctx context.Context) (*report.Report, error) { + started := p.now() + cfg := p.cfg + + rep := report.New( + runid.New(started), + cfg.Domain, + cfg.Scope, + version.Version, + config.DetectEnvironment(cfg.Environment, false), + started, + ) + for _, w := range cfg.Warnings { + rep.Warnf("%s", w) + } + + // Reserve a slice of the deadline to build and deliver the report, so a + // tight budget yields a truncated report instead of a killed process. + usable := time.Duration(float64(cfg.Timeout) * (1 - cfg.OutputMargin)) + budget := NewBudget(cfg.Scope.Stages(), started.Add(usable)) + budget.now = p.now + + p.log.Info("run started", + "run_id", rep.Run.ID, + "domain", cfg.Domain, + "scope", cfg.Scope.String(), + "stages", cfg.Scope.StageNames(), + "deadline", budget.Deadline().UTC().Format(time.RFC3339), + ) + + state := &runState{report: rep} + for _, st := range cfg.Scope.Stages() { + if err := ctx.Err(); err != nil { + p.stop(rep, st, err) + break + } + if budget.Expired() { + p.stop(rep, st, context.DeadlineExceeded) + break + } + if err := p.runStage(ctx, st, budget, state); err != nil { + p.stop(rep, st, err) + break + } + } + + rep.Finish(p.now()) + p.logSummary(rep) + return rep, nil +} + +// runState carries values between stages. +type runState struct { + report *report.Report + hosts []string + found []report.Host +} + +func (p *Pipeline) runStage(ctx context.Context, st stage.Stage, budget *Budget, state *runState) error { + allotted := budget.Take(st) + if allotted <= 0 { + return context.DeadlineExceeded + } + stageCtx, cancel := context.WithTimeout(ctx, allotted) + defer cancel() + + start := p.now() + p.log.Debug("stage started", "stage", string(st), "budget", allotted.String()) + + err := p.dispatch(stageCtx, st, state) + if err != nil { + return err + } + + p.log.Info("stage finished", + "stage", string(st), + "duration_ms", p.now().Sub(start).Milliseconds(), + "hosts", len(state.hosts), + ) + return nil +} + +func (p *Pipeline) dispatch(ctx context.Context, st stage.Stage, state *runState) error { + rep := state.report + switch st { + case stage.Enumerate: + if p.stages.Enumerator == nil { + return ErrNoImplementation + } + res, err := p.stages.Enumerator.Enumerate(ctx, p.cfg.Domain) + if err != nil { + return err + } + state.hosts = res.Hosts + rep.Sources = res.Sources + rep.Stats.Enumerated = len(res.Hosts) + p.applyPartial(ctx, rep, st, res.Partial) + return nil + + case stage.Exclude: + if p.stages.Excluder == nil { + return ErrNoImplementation + } + res := p.stages.Excluder.Filter(state.hosts) + state.hosts = res.Kept + rep.Stats.Excluded = len(res.Removed) + rep.Stats.InScope = len(res.Kept) + // Publish the surviving hosts now, so an enumeration-only run + // reports the subdomains it found rather than just counting them. + // The resolve stage replaces these with their live/dead verdict. + rep.Hosts = make([]report.Host, 0, len(res.Kept)) + for _, h := range res.Kept { + rep.Hosts = append(rep.Hosts, report.Host{Host: h, Status: report.StatusDiscovered}) + } + if p.cfg.ReportExcluded { + rep.Excluded = res.Removed + } + for _, pattern := range res.Unused { + rep.Warnf("exclusion pattern %q matched nothing", pattern) + } + return nil + + case stage.Resolve: + if p.stages.Resolver == nil { + return ErrNoImplementation + } + res, err := p.stages.Resolver.Resolve(ctx, state.hosts) + if err != nil { + return err + } + state.found = res.Hosts + rep.Hosts = res.Hosts + p.applyPartial(ctx, rep, st, res.Partial) + return nil + + case stage.PortScan: + if p.stages.PortScanner == nil { + return ErrNoImplementation + } + res, err := p.stages.PortScanner.Scan(ctx, state.found) + if err != nil { + return err + } + state.found = res.Hosts + rep.Hosts = res.Hosts + p.applyPartial(ctx, rep, st, res.Partial) + return nil + + case stage.HTTPProbe: + if p.stages.Prober == nil { + return ErrNoImplementation + } + res, err := p.stages.Prober.Probe(ctx, state.found) + if err != nil { + return err + } + state.found = res.Hosts + rep.Hosts = res.Hosts + p.applyPartial(ctx, rep, st, res.Partial) + return nil + + default: + return errors.New("unknown stage " + string(st)) + } +} + +// applyPartial folds a stage's non-fatal outcome into the report. A truncated +// stage does not stop the ladder — the later stages still have their own +// budget and can work on what was found — but the run stops claiming to be +// complete. +// +// A deadline and an operator stopping the job both cut a stage short, and +// they are reported differently: a consumer may reasonably retry a run that +// ran out of time, and must not retry one somebody stopped on purpose. +func (p *Pipeline) applyPartial(ctx context.Context, rep *report.Report, st stage.Stage, part Partial) { + for _, w := range part.Warnings { + rep.Warnf("%s", w) + } + if !part.Truncated { + return + } + rep.Run.Completed = false + + if errors.Is(ctx.Err(), context.Canceled) { + rep.Warnf("stage %s: run canceled, results are partial", st) + p.log.Warn("stage canceled", "stage", string(st)) + return + } + rep.Run.TruncatedByTimeout = true + rep.Warnf("stage %s: cut short by its deadline, results are partial", st) + p.log.Warn("stage truncated", "stage", string(st)) +} + +// stop records why the ladder ended early. The report stays valid; only its +// completeness flags change. +func (p *Pipeline) stop(rep *report.Report, st stage.Stage, err error) { + rep.Run.Completed = false + switch { + case errors.Is(err, context.DeadlineExceeded): + rep.Run.TruncatedByTimeout = true + rep.Warnf("stage %s: run deadline reached, results are partial", st) + p.log.Warn("run truncated by timeout", "stage", string(st)) + case errors.Is(err, context.Canceled): + rep.Warnf("stage %s: run canceled, results are partial", st) + p.log.Warn("run canceled", "stage", string(st)) + case errors.Is(err, ErrNoImplementation): + rep.Warnf("stage %s: %s", st, ErrNoImplementation) + p.log.Warn("stage unavailable", "stage", string(st), "error", err) + default: + rep.Warnf("stage %s failed: %s", st, err) + p.log.Error("stage failed", "stage", string(st), "error", err) + } +} + +// logSummary emits the run counters to stderr. In a log-only environment the +// report itself may go to a webhook, so the outcome has to be legible here. +func (p *Pipeline) logSummary(rep *report.Report) { + p.log.Info("run finished", + "run_id", rep.Run.ID, + "domain", rep.Run.Domain, + "completed", rep.Run.Completed, + "truncated_by_timeout", rep.Run.TruncatedByTimeout, + "duration_ms", rep.Run.Duration, + "enumerated", rep.Stats.Enumerated, + "excluded", rep.Stats.Excluded, + "in_scope", rep.Stats.InScope, + "live", rep.Stats.Live, + "dead", rep.Stats.Dead, + "wildcard", rep.Stats.Wildcard, + "open_ports", rep.Stats.OpenPorts, + "http_services", rep.Stats.HTTPServices, + "warnings", len(rep.Warnings), + ) +} diff --git a/internal/pipeline/pipeline_test.go b/internal/pipeline/pipeline_test.go new file mode 100644 index 0000000..3ca08bf --- /dev/null +++ b/internal/pipeline/pipeline_test.go @@ -0,0 +1,278 @@ +package pipeline + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/JoshuaMart/FastRecon/internal/config" + "github.com/JoshuaMart/FastRecon/internal/logging" + "github.com/JoshuaMart/FastRecon/internal/report" + "github.com/JoshuaMart/FastRecon/internal/stage" +) + +type fakeEnumerator struct { + hosts []string + err error + slow time.Duration +} + +func (f fakeEnumerator) Name() string { return "fake" } +func (f fakeEnumerator) Enumerate(ctx context.Context, _ string) (Enumeration, error) { + if f.slow > 0 { + select { + case <-time.After(f.slow): + case <-ctx.Done(): + return Enumeration{}, ctx.Err() + } + } + if f.err != nil { + return Enumeration{}, f.err + } + return Enumeration{ + Hosts: f.hosts, + Sources: []report.Source{{Name: "fake", Status: report.SourceOK, Found: len(f.hosts)}}, + }, nil +} + +type fakeExcluder struct { + drop string + unused []string +} + +func (fakeExcluder) Name() string { return "fake" } +func (f fakeExcluder) Filter(hosts []string) Filtered { + out := Filtered{Unused: f.unused} + for _, h := range hosts { + if h == f.drop { + out.Removed = append(out.Removed, report.Excluded{Host: h, Pattern: f.drop}) + continue + } + out.Kept = append(out.Kept, h) + } + return out +} + +type fakeResolver struct{ live []string } + +func (fakeResolver) Name() string { return "fake" } +func (f fakeResolver) Resolve(_ context.Context, hosts []string) (Resolution, error) { + out := make([]report.Host, 0, len(hosts)) + for _, h := range hosts { + status, reason := report.StatusDead, report.ReasonNXDomain + for _, l := range f.live { + if l == h { + status, reason = report.StatusLive, "" + break + } + } + out = append(out, report.Host{Host: h, Status: status, Reason: reason}) + } + return Resolution{Hosts: out}, nil +} + +func testConfig(t *testing.T, scope stage.Scope) *config.Config { + t.Helper() + return &config.Config{ + Domain: "example.com", + Scope: scope, + Timeout: time.Minute, + OutputMargin: 0.1, + Output: config.StdoutPath, + Format: report.FormatJSON, + } +} + +func TestRunStopsAtTheFirstMissingStage(t *testing.T) { + cfg := testConfig(t, stage.ScopeFull) + rep, err := New(cfg, Stages{}, logging.Discard()).Run(context.Background()) + if err != nil { + t.Fatalf("Run returned an error instead of a report: %v", err) + } + if rep.Run.Completed { + t.Error("a run with no stage implementation must not report itself complete") + } + if len(rep.Warnings) != 1 || !strings.Contains(rep.Warnings[0], "enumerate") { + t.Errorf("warnings = %v, want one naming the enumerate stage", rep.Warnings) + } + if rep.Run.TruncatedByTimeout { + t.Error("a missing implementation is not a timeout") + } +} + +func TestRunEnumScopeStopsBeforeResolving(t *testing.T) { + cfg := testConfig(t, stage.ScopeEnum) + stages := Stages{ + Enumerator: fakeEnumerator{hosts: []string{"a.example.com", "b.example.com"}}, + Excluder: fakeExcluder{drop: "b.example.com"}, + // A resolver is wired but must never run at this scope. + Resolver: fakeResolver{live: []string{"a.example.com"}}, + } + rep, err := New(cfg, stages, logging.Discard()).Run(context.Background()) + if err != nil { + t.Fatal(err) + } + if !rep.Run.Completed { + t.Errorf("run not completed, warnings: %v", rep.Warnings) + } + if rep.Stats.Enumerated != 2 || rep.Stats.Excluded != 1 || rep.Stats.InScope != 1 { + t.Errorf("stats = %+v, want 2 enumerated / 1 excluded / 1 in scope", rep.Stats) + } + // An enumeration-only run must still carry the hosts it found. + if len(rep.Hosts) != 1 || rep.Hosts[0].Host != "a.example.com" { + t.Errorf("hosts = %v, want the surviving host listed", rep.Hosts) + } + if rep.Hosts[0].Status != report.StatusDiscovered { + t.Errorf("status = %q, want %q: nothing was resolved at this scope", rep.Hosts[0].Status, report.StatusDiscovered) + } + if rep.Stats.Live != 0 || rep.Stats.Dead != 0 { + t.Error("a discovered host must not count as live or dead") + } + if len(rep.Sources) != 1 { + t.Errorf("sources = %v, want the source accounting to reach the report", rep.Sources) + } +} + +func TestRunResolveScopeSplitsLiveAndDead(t *testing.T) { + cfg := testConfig(t, stage.ScopeResolve) + stages := Stages{ + Enumerator: fakeEnumerator{hosts: []string{"a.example.com", "old.example.com"}}, + Excluder: fakeExcluder{}, + Resolver: fakeResolver{live: []string{"a.example.com"}}, + } + rep, err := New(cfg, stages, logging.Discard()).Run(context.Background()) + if err != nil { + t.Fatal(err) + } + if rep.Stats.Live != 1 || rep.Stats.Dead != 1 { + t.Errorf("live/dead = %d/%d, want 1/1", rep.Stats.Live, rep.Stats.Dead) + } + // A dangling host is a finding, not noise: it stays in the report. + if len(rep.Hosts) != 2 { + t.Errorf("hosts = %d, want dead hosts kept in the report", len(rep.Hosts)) + } +} + +func TestUnusedExclusionPatternWarns(t *testing.T) { + cfg := testConfig(t, stage.ScopeEnum) + stages := Stages{ + Enumerator: fakeEnumerator{hosts: []string{"a.example.com"}}, + Excluder: fakeExcluder{unused: []string{"*.qa.example.com"}}, + } + rep, err := New(cfg, stages, logging.Discard()).Run(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(rep.Warnings) != 1 || !strings.Contains(rep.Warnings[0], "qa.example.com") { + t.Errorf("warnings = %v, want the unused pattern surfaced", rep.Warnings) + } + if !rep.Run.Completed { + t.Error("an unused pattern is a warning, not a failure") + } +} + +func TestRunReportsPartialResultsOnTimeout(t *testing.T) { + cfg := testConfig(t, stage.ScopeFull) + cfg.Timeout = 50 * time.Millisecond + stages := Stages{Enumerator: fakeEnumerator{slow: time.Second}} + + rep, err := New(cfg, stages, logging.Discard()).Run(context.Background()) + if err != nil { + t.Fatalf("a run that ran out of time must still return a report: %v", err) + } + if !rep.Run.TruncatedByTimeout { + t.Error("truncated_by_timeout not set") + } + if rep.Run.Completed { + t.Error("completed must be false on a truncated run") + } + if rep.Run.Finished.IsZero() { + t.Error("a truncated report must still be well formed") + } +} + +func TestRunSurfacesStageFailure(t *testing.T) { + cfg := testConfig(t, stage.ScopeResolve) + stages := Stages{Enumerator: fakeEnumerator{err: errors.New("all sources down")}} + + rep, err := New(cfg, stages, logging.Discard()).Run(context.Background()) + if err != nil { + t.Fatal(err) + } + if rep.Run.Completed || rep.Run.TruncatedByTimeout { + t.Error("a stage failure is neither a completion nor a timeout") + } + if len(rep.Warnings) != 1 || !strings.Contains(rep.Warnings[0], "all sources down") { + t.Errorf("warnings = %v, want the underlying error", rep.Warnings) + } +} + +func TestRunCarriesConfigWarningsIntoTheReport(t *testing.T) { + cfg := testConfig(t, stage.ScopeEnum) + cfg.Warnings = []string{`config file key "porst" matches no option and was ignored`} + stages := Stages{Enumerator: fakeEnumerator{}, Excluder: fakeExcluder{}} + + rep, err := New(cfg, stages, logging.Discard()).Run(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(rep.Warnings) != 1 || !strings.Contains(rep.Warnings[0], "porst") { + t.Errorf("warnings = %v, want the configuration warning carried through", rep.Warnings) + } +} + +// A stage that finished but was cut short must not leave the run claiming to +// be complete. +func TestTruncatedStageMarksTheRunIncomplete(t *testing.T) { + cfg := testConfig(t, stage.ScopeEnum) + stages := Stages{ + Enumerator: truncatingEnumerator{}, + Excluder: fakeExcluder{}, + } + rep, err := New(cfg, stages, logging.Discard()).Run(context.Background()) + if err != nil { + t.Fatal(err) + } + if rep.Run.Completed { + t.Error("a truncated stage must clear completed") + } + if !rep.Run.TruncatedByTimeout { + t.Error("truncated_by_timeout not set") + } + // The hosts it did find must survive. + if len(rep.Hosts) != 1 { + t.Errorf("hosts = %v, want the partial result kept", rep.Hosts) + } + if len(rep.Warnings) == 0 { + t.Error("a truncated stage must leave a warning in the report") + } +} + +type truncatingEnumerator struct{} + +func (truncatingEnumerator) Name() string { return "truncating" } +func (truncatingEnumerator) Enumerate(context.Context, string) (Enumeration, error) { + e := Enumeration{Hosts: []string{"a.example.com"}} + e.Truncated = true + e.Warnings = []string{"source crt: deadline reached"} + return e, nil +} + +func TestRunCanceledByContext(t *testing.T) { + cfg := testConfig(t, stage.ScopeFull) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + rep, err := New(cfg, Stages{Enumerator: fakeEnumerator{}}, logging.Discard()).Run(ctx) + if err != nil { + t.Fatal(err) + } + if rep.Run.Completed { + t.Error("a canceled run must be marked incomplete") + } + if rep.Run.TruncatedByTimeout { + t.Error("cancellation is not a timeout") + } +} diff --git a/internal/portscan/cdn.go b/internal/portscan/cdn.go new file mode 100644 index 0000000..afe525a --- /dev/null +++ b/internal/portscan/cdn.go @@ -0,0 +1,72 @@ +package portscan + +import ( + "net" + "sort" + + "github.com/projectdiscovery/cdncheck" + + "github.com/JoshuaMart/FastRecon/internal/report" +) + +// edge is what the CDN check found for one address. +type edge struct { + Provider string + Type string +} + +// classify determines, for every address, whether it belongs to a CDN, WAF or +// cloud provider range. +// +// This runs on every scan regardless of --skip-cdn. Only the restriction is +// optional: "80 and 443 are the only open ports" is indistinguishable from a +// genuinely minimal host unless the report says the scan was narrowed on +// purpose. +func classify(client *cdncheck.Client, addresses []string) map[string]edge { + out := make(map[string]edge, len(addresses)) + if client == nil { + return out + } + for _, addr := range addresses { + ip := net.ParseIP(addr) + if ip == nil { + continue + } + matched, provider, kind, err := client.Check(ip) + if err != nil || !matched { + continue + } + out[addr] = edge{Provider: provider, Type: kind} + } + return out +} + +// cdnEntries builds the report entries for one host: one per provider, each +// naming the addresses it matched, because a host can have both a CDN address +// and an origin address. +func cdnEntries(addresses []string, edges map[string]edge, limited bool) []report.CDN { + byProvider := map[edge][]string{} + for _, addr := range addresses { + e, ok := edges[addr] + if !ok { + continue + } + byProvider[e] = append(byProvider[e], addr) + } + if len(byProvider) == 0 { + return nil + } + + out := make([]report.CDN, 0, len(byProvider)) + for e, matched := range byProvider { + sort.Strings(matched) + out = append(out, report.CDN{ + Name: e.Provider, + Type: e.Type, + Addresses: matched, + ScanLimited: limited, + }) + } + sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) + return out +} diff --git a/internal/portscan/connect.go b/internal/portscan/connect.go new file mode 100644 index 0000000..90b4cd9 --- /dev/null +++ b/internal/portscan/connect.go @@ -0,0 +1,129 @@ +package portscan + +import ( + "context" + "errors" + "net" + "strconv" + "sync" + + "github.com/JoshuaMart/FastRecon/internal/ratelimit" +) + +// scanConnect performs a TCP connect scan. +// +// Connect scanning needs no privileges, which is the whole reason it is the +// default: it is the only mode that works in a serverless job. A completed +// handshake means the port is open; a refusal means it is closed; a timeout +// means it is filtered, and only that case is worth retrying. +func (s *Scanner) scanConnect(ctx context.Context, addresses []string, ports portSpec, limiter *ratelimit.Limiter) (map[string][]int, error) { + list, err := s.expand(ports) + if err != nil { + return nil, err + } + if len(list) == 0 || len(addresses) == 0 { + return map[string][]int{}, nil + } + + s.opts.Logger.Debug("connect scan started", + "addresses", len(addresses), + "ports", len(list), + "probes", len(addresses)*len(list), + "concurrency", s.opts.Concurrency, + "rate", s.opts.Rate, + ) + + var ( + found = map[string][]int{} + mu sync.Mutex + wg sync.WaitGroup + ) + + // A fixed pool consuming a stream of targets. Spawning one goroutine per + // probe would allocate a stack for every (address, port) pair up front — + // a full sweep of fifty addresses is millions of them, and the process is + // killed for memory long before the deadline it was budgeted. + queue := make(chan target) + for range s.opts.Concurrency { + wg.Add(1) + go func() { + defer wg.Done() + for t := range queue { + if ctx.Err() != nil { + return + } + if !limiter.Wait(ctx) { + return + } + if s.probe(ctx, t) { + mu.Lock() + found[t.address] = append(found[t.address], t.port) + mu.Unlock() + } + } + }() + } + + feed(ctx, queue, addresses, list) + wg.Wait() + + return found, nil +} + +type target struct { + address string + port int +} + +// feed streams the probes port-major: every address is tried on one port +// before moving to the next. Address-major order would hammer a single host +// with the whole port list back to back. +// +// Targets are generated rather than materialised: the full list for a wide +// sweep is itself large enough to be worth not holding. +func feed(ctx context.Context, queue chan<- target, addresses []string, ports []int) { + defer close(queue) + for _, p := range ports { + for _, a := range addresses { + select { + case queue <- target{address: a, port: p}: + case <-ctx.Done(): + return + } + } + } +} + +// probe reports whether a TCP handshake completes, retrying only the +// inconclusive cases. +func (s *Scanner) probe(ctx context.Context, t target) bool { + addr := net.JoinHostPort(t.address, strconv.Itoa(t.port)) + dialer := &net.Dialer{Timeout: s.opts.Timeout} + + for attempt := 0; attempt <= s.opts.Retries; attempt++ { + if ctx.Err() != nil { + return false + } + conn, err := dialer.DialContext(ctx, "tcp", addr) + if err == nil { + _ = conn.Close() + return true + } + // A refusal is a definitive answer: the port is closed. Retrying it + // would multiply the work for no new information. + if !isInconclusive(err) { + return false + } + } + return false +} + +// isInconclusive reports whether an error leaves the port's state unknown. +func isInconclusive(err error) bool { + var netErr net.Error + if errors.As(err, &netErr) && netErr.Timeout() { + return true + } + // Local resource exhaustion says nothing about the target either. + return errors.Is(err, syscallEMFILE) || errors.Is(err, syscallENFILE) +} diff --git a/internal/portscan/errno_other.go b/internal/portscan/errno_other.go new file mode 100644 index 0000000..403b4b3 --- /dev/null +++ b/internal/portscan/errno_other.go @@ -0,0 +1,11 @@ +//go:build !unix + +package portscan + +import "errors" + +// No portable equivalent outside unix; the sentinels simply never match. +var ( + syscallEMFILE = errors.New("emfile") + syscallENFILE = errors.New("enfile") +) diff --git a/internal/portscan/errno_unix.go b/internal/portscan/errno_unix.go new file mode 100644 index 0000000..2f4611c --- /dev/null +++ b/internal/portscan/errno_unix.go @@ -0,0 +1,12 @@ +//go:build unix + +package portscan + +import "syscall" + +// Running out of file descriptors is a local limit, not a verdict about the +// target, so a probe that hits one is retried rather than recorded as closed. +var ( + syscallEMFILE error = syscall.EMFILE + syscallENFILE error = syscall.ENFILE +) diff --git a/internal/portscan/ports.go b/internal/portscan/ports.go new file mode 100644 index 0000000..028a1c2 --- /dev/null +++ b/internal/portscan/ports.go @@ -0,0 +1,189 @@ +package portscan + +import ( + "fmt" + "sort" + "strconv" + "strings" +) + +// Named port selections. +const ( + PortsTop100 = "top-100" + PortsTop1000 = "top-1000" + PortsWeb = "web" + PortsFull = "full" +) + +// cdnPorts is what a CDN or WAF address is scanned for when the full scan is +// skipped. It matches the engine's own -exclude-cdn behaviour. +var cdnPorts = []int{80, 443} + +// webPorts is the curated HTTP-oriented selection: the ports a web service is +// actually found on, rather than the most common ports overall. +var webPorts = []int{ + 80, 81, 88, 443, 591, 2082, 2087, 2095, 2096, 3000, 4243, 4993, + 5000, 5104, 5108, 5800, 6543, 7000, 7396, 7474, 8000, 8001, 8008, + 8014, 8042, 8069, 8080, 8081, 8088, 8090, 8091, 8118, 8123, 8172, + 8222, 8243, 8280, 8281, 8333, 8443, 8500, 8834, 8880, 8888, 8983, + 9000, 9043, 9060, 9080, 9090, 9091, 9200, 9443, 9800, 9981, + 12443, 16080, 18091, 18092, 20720, 28017, +} + +// portSpec is a port selection in the form the engine expects: either a +// top-ports tier or an explicit list. +type portSpec struct { + // TopPorts is the engine's tier name ("100", "1000", "full"). + TopPorts string + // List is an explicit port expression, e.g. "80,443,8000-8100". + List string +} + +func (p portSpec) String() string { + if p.TopPorts != "" { + return "top-" + p.TopPorts + } + return p.List +} + +// parsePorts turns the configured selection into an engine port spec. +func parsePorts(ports string) (portSpec, error) { + switch strings.ToLower(strings.TrimSpace(ports)) { + case PortsTop100: + return portSpec{TopPorts: "100"}, nil + case PortsTop1000: + return portSpec{TopPorts: "1000"}, nil + case PortsFull: + return portSpec{TopPorts: "full"}, nil + case PortsWeb: + return portSpec{List: joinPorts(webPorts)}, nil + case "": + return portSpec{}, fmt.Errorf("ports must not be empty") + } + + if err := validatePortExpression(ports); err != nil { + return portSpec{}, err + } + return portSpec{List: ports}, nil +} + +// validatePortExpression checks an explicit list so a typo is caught here, +// with a message naming the offending part, rather than deep inside the +// engine. +func validatePortExpression(expr string) error { + for _, part := range strings.Split(expr, ",") { + part = strings.TrimSpace(part) + if part == "" { + return fmt.Errorf("ports %q contains an empty entry", expr) + } + lo, hi, isRange := strings.Cut(part, "-") + if err := validatePort(lo, expr); err != nil { + return err + } + if !isRange { + continue + } + if err := validatePort(hi, expr); err != nil { + return err + } + l, _ := strconv.Atoi(strings.TrimSpace(lo)) + h, _ := strconv.Atoi(strings.TrimSpace(hi)) + if l > h { + return fmt.Errorf("ports %q has a reversed range %q", expr, part) + } + } + return nil +} + +func validatePort(s, expr string) error { + n, err := strconv.Atoi(strings.TrimSpace(s)) + if err != nil { + return fmt.Errorf("ports %q contains %q, which is not a port number", expr, s) + } + if n < 1 || n > 65535 { + return fmt.Errorf("ports %q contains %d, outside 1-65535", expr, n) + } + return nil +} + +// expand turns a port spec into the concrete list to probe, minus whatever +// --exclude-ports removes. +func (s *Scanner) expand(spec portSpec) ([]int, error) { + var expr string + switch spec.TopPorts { + case "100": + expr = nmapTop100 + case "1000": + expr = nmapTop1000 + case "full": + expr = "1-65535" + case "": + expr = spec.List + default: + return nil, fmt.Errorf("unknown top-ports tier %q", spec.TopPorts) + } + + ports, err := expandExpression(expr) + if err != nil { + return nil, err + } + if s.opts.ExcludePorts == "" { + return ports, nil + } + + excluded, err := expandExpression(s.opts.ExcludePorts) + if err != nil { + return nil, fmt.Errorf("exclude-%w", err) + } + drop := make(map[int]struct{}, len(excluded)) + for _, p := range excluded { + drop[p] = struct{}{} + } + kept := ports[:0] + for _, p := range ports { + if _, skip := drop[p]; !skip { + kept = append(kept, p) + } + } + return kept, nil +} + +// expandExpression turns "80,443,8000-8100" into a sorted, deduplicated list. +func expandExpression(expr string) ([]int, error) { + if err := validatePortExpression(expr); err != nil { + return nil, err + } + + seen := map[int]struct{}{} + var out []int + add := func(p int) { + if _, dup := seen[p]; dup { + return + } + seen[p] = struct{}{} + out = append(out, p) + } + + for _, part := range strings.Split(expr, ",") { + lo, hi, isRange := strings.Cut(strings.TrimSpace(part), "-") + l, _ := strconv.Atoi(strings.TrimSpace(lo)) + if !isRange { + add(l) + continue + } + h, _ := strconv.Atoi(strings.TrimSpace(hi)) + for p := l; p <= h; p++ { + add(p) + } + } + sort.Ints(out) + return out, nil +} + +func joinPorts(ports []int) string { + parts := make([]string, len(ports)) + for i, p := range ports { + parts[i] = strconv.Itoa(p) + } + return strings.Join(parts, ",") +} diff --git a/internal/portscan/portscan_test.go b/internal/portscan/portscan_test.go new file mode 100644 index 0000000..c4d640c --- /dev/null +++ b/internal/portscan/portscan_test.go @@ -0,0 +1,474 @@ +package portscan + +import ( + "context" + "log/slog" + "net" + "os" + "runtime" + "slices" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/JoshuaMart/FastRecon/internal/ratelimit" + "github.com/JoshuaMart/FastRecon/internal/report" +) + +func discardLogger() *slog.Logger { + return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.Level(99)})) +} + +func TestParsePorts(t *testing.T) { + named := map[string]portSpec{ + PortsTop100: {TopPorts: "100"}, + "TOP-1000": {TopPorts: "1000"}, + PortsFull: {TopPorts: "full"}, + } + for in, want := range named { + got, err := parsePorts(in) + if err != nil { + t.Errorf("parsePorts(%q): %v", in, err) + continue + } + if got != want { + t.Errorf("parsePorts(%q) = %+v, want %+v", in, got, want) + } + } + + web, err := parsePorts(PortsWeb) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(web.List, "8443") || web.TopPorts != "" { + t.Errorf("web selection = %+v, want an explicit list containing 8443", web) + } + + explicit, err := parsePorts("80,443,8000-8100") + if err != nil { + t.Fatal(err) + } + if explicit.List != "80,443,8000-8100" { + t.Errorf("explicit = %+v, want the expression passed through", explicit) + } +} + +// A typo in a port list must be caught here, naming the offending part, +// rather than deep inside the engine. +func TestParsePortsRejectsBadExpressions(t *testing.T) { + for _, in := range []string{"", "80,,443", "80-", "http", "0", "70000", "8100-8000", "80,abc"} { + if got, err := parsePorts(in); err == nil { + t.Errorf("parsePorts(%q) = %+v, want an error", in, got) + } + } +} + +func TestIndexAddressesSkipsWhatCannotBeScanned(t *testing.T) { + hosts := []report.Host{ + {Host: "a.example.com", Status: report.StatusLive, Addresses: []string{"1.2.3.4"}}, + {Host: "b.example.com", Status: report.StatusLive, Addresses: []string{"1.2.3.4", "5.6.7.8"}}, + {Host: "dead.example.com", Status: report.StatusDead}, + // A wildcard artifact is not a host; scanning it proves nothing. + {Host: "junk.example.com", Status: report.StatusWildcard, Addresses: []string{"9.9.9.9"}}, + {Host: "bad.example.com", Status: report.StatusLive, Addresses: []string{"not-an-ip"}}, + } + + got := indexAddresses(hosts) + if len(got) != 2 { + t.Fatalf("addresses = %v, want only the two live ones", got) + } + if !slices.Equal(got["1.2.3.4"], []int{0, 1}) { + t.Errorf("1.2.3.4 = %v, want both hosts sharing it", got["1.2.3.4"]) + } + if _, ok := got["9.9.9.9"]; ok { + t.Error("a wildcard artifact was queued for scanning") + } +} + +func TestSplitByEdge(t *testing.T) { + addresses := []string{"1.1.1.1", "2.2.2.2", "3.3.3.3"} + edges := map[string]edge{"2.2.2.2": {Provider: "cloudflare", Type: "waf"}} + + plain, behind := splitByEdge(addresses, edges, true) + if !slices.Equal(plain, []string{"1.1.1.1", "3.3.3.3"}) || !slices.Equal(behind, []string{"2.2.2.2"}) { + t.Errorf("split = %v / %v, want the CDN address separated", plain, behind) + } + + // With the restriction lifted, everything gets the full sweep. + plain, behind = splitByEdge(addresses, edges, false) + if len(plain) != 3 || len(behind) != 0 { + t.Errorf("split = %v / %v, want no restriction", plain, behind) + } +} + +func TestCDNEntriesGroupPerProvider(t *testing.T) { + addresses := []string{"1.1.1.1", "2.2.2.2", "3.3.3.3"} + edges := map[string]edge{ + "1.1.1.1": {Provider: "cloudflare", Type: "waf"}, + "2.2.2.2": {Provider: "aws", Type: "cloud"}, + // Same provider as the first: they must land in one entry. + "3.3.3.3": {Provider: "cloudflare", Type: "waf"}, + } + + got := cdnEntries(addresses, edges, true) + if len(got) != 2 { + t.Fatalf("entries = %+v, want one per provider", got) + } + if got[0].Name != "aws" || got[1].Name != "cloudflare" { + t.Errorf("entries are not sorted by provider: %+v", got) + } + if !slices.Equal(got[1].Addresses, []string{"1.1.1.1", "3.3.3.3"}) { + t.Errorf("cloudflare addresses = %v, want both", got[1].Addresses) + } + if !got[1].ScanLimited { + t.Error("scan_limited must be set when the sweep was narrowed") + } + if cdnEntries(addresses, nil, false) != nil { + t.Error("no CDN means no entries") + } +} + +func newScanner(t *testing.T, skipCDN bool, scan func(context.Context, []string, portSpec, *ratelimit.Limiter) (map[string][]int, error)) *Scanner { + t.Helper() + return &Scanner{ + opts: Options{SkipCDN: skipCDN, Mode: ModeConnect, Logger: discardLogger()}, + ports: portSpec{TopPorts: "100"}, + scan: scan, + } +} + +// One address shared by several subdomains is scanned once and mapped back +// onto every one of them. +func TestScanMapsResultsBackOntoSharedAddresses(t *testing.T) { + var passes [][]string + n := newScanner(t, true, func(_ context.Context, addresses []string, _ portSpec, _ *ratelimit.Limiter) (map[string][]int, error) { + passes = append(passes, addresses) + return map[string][]int{"1.2.3.4": {80, 443}}, nil + }) + + hosts := []report.Host{ + {Host: "a.example.com", Status: report.StatusLive, Addresses: []string{"1.2.3.4"}}, + {Host: "b.example.com", Status: report.StatusLive, Addresses: []string{"1.2.3.4"}}, + {Host: "dead.example.com", Status: report.StatusDead}, + } + + res, err := n.Scan(context.Background(), hosts) + if err != nil { + t.Fatal(err) + } + if len(passes) != 1 || len(passes[0]) != 1 { + t.Fatalf("passes = %v, want the shared address scanned once", passes) + } + for _, i := range []int{0, 1} { + if len(res.Hosts[i].Ports) != 2 { + t.Errorf("%s ports = %+v, want both mapped back", res.Hosts[i].Host, res.Hosts[i].Ports) + } + } + if res.Hosts[2].Ports != nil { + t.Error("a dead host must not gain ports") + } +} + +func TestScanDeduplicatesPortsAcrossAHostAddresses(t *testing.T) { + n := newScanner(t, true, func(context.Context, []string, portSpec, *ratelimit.Limiter) (map[string][]int, error) { + return map[string][]int{"1.2.3.4": {443, 80}, "5.6.7.8": {80, 8080}}, nil + }) + + hosts := []report.Host{{Host: "a.example.com", Status: report.StatusLive, Addresses: []string{"1.2.3.4", "5.6.7.8"}}} + res, err := n.Scan(context.Background(), hosts) + if err != nil { + t.Fatal(err) + } + + var ports []int + for _, p := range res.Hosts[0].Ports { + ports = append(ports, p.Port) + if p.Protocol != "tcp" || p.State != "open" { + t.Errorf("port %+v is missing its protocol or state", p) + } + } + if !slices.Equal(ports, []int{80, 443, 8080}) { + t.Errorf("ports = %v, want them deduplicated and sorted", ports) + } +} + +func TestScanWithNoLiveHostDoesNothing(t *testing.T) { + called := false + n := newScanner(t, true, func(context.Context, []string, portSpec, *ratelimit.Limiter) (map[string][]int, error) { + called = true + return nil, nil + }) + + res, err := n.Scan(context.Background(), []report.Host{{Host: "dead.example.com", Status: report.StatusDead}}) + if err != nil { + t.Fatal(err) + } + if called { + t.Error("the engine was started with nothing to scan") + } + if len(res.Hosts) != 1 { + t.Error("the hosts must pass through untouched") + } +} + +func TestScanReportsTruncationWhenTheDeadlinePasses(t *testing.T) { + n := newScanner(t, true, func(context.Context, []string, portSpec, *ratelimit.Limiter) (map[string][]int, error) { + return map[string][]int{"1.2.3.4": {80}}, nil + }) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + res, err := n.Scan(ctx, []report.Host{{Host: "a.example.com", Status: report.StatusLive, Addresses: []string{"1.2.3.4"}}}) + if err != nil { + t.Fatal(err) + } + if !res.Truncated { + t.Error("a scan cut short must report itself truncated") + } + if len(res.Warnings) == 0 { + t.Error("a truncated scan must warn") + } +} + +// A SYN scan the process cannot perform finds nothing at all, which reads +// exactly like a host with nothing listening. +func TestSynModeIsRefusedNotSilentlyDowngraded(t *testing.T) { + _, err := New(Options{ + Mode: ModeSYN, + Ports: PortsTop100, + Concurrency: 10, + Rate: 100, + Timeout: time.Second, + Logger: discardLogger(), + }) + if err == nil { + t.Fatal("syn mode was accepted; it must refuse rather than scan differently than asked") + } + if !strings.Contains(err.Error(), "connect") { + t.Errorf("error = %q, want it to point at the usable mode", err) + } +} + +func TestExpandPorts(t *testing.T) { + s := &Scanner{opts: Options{Logger: discardLogger()}} + + got, err := s.expand(portSpec{List: "443,80,80,8000-8002"}) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(got, []int{80, 443, 8000, 8001, 8002}) { + t.Errorf("expand = %v, want it sorted and deduplicated", got) + } + + top, err := s.expand(portSpec{TopPorts: "100"}) + if err != nil { + t.Fatal(err) + } + if len(top) < 90 || !slices.Contains(top, 443) { + t.Errorf("top-100 expanded to %d ports, want the nmap selection", len(top)) + } + + full, err := s.expand(portSpec{TopPorts: "full"}) + if err != nil { + t.Fatal(err) + } + if len(full) != 65535 { + t.Errorf("full = %d ports, want 65535", len(full)) + } +} + +func TestExpandPortsAppliesExclusions(t *testing.T) { + s := &Scanner{opts: Options{ExcludePorts: "443,8001", Logger: discardLogger()}} + got, err := s.expand(portSpec{List: "80,443,8000-8002"}) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(got, []int{80, 8000, 8002}) { + t.Errorf("expand = %v, want the excluded ports removed", got) + } +} + +func TestFeedIsPortMajor(t *testing.T) { + // Address-major order would hammer one host with the whole port list + // back to back. + queue := make(chan target) + go feed(context.Background(), queue, []string{"1.1.1.1", "2.2.2.2"}, []int{80, 443}) + + var got []target + for t := range queue { + got = append(got, t) + } + want := []target{ + {"1.1.1.1", 80}, {"2.2.2.2", 80}, + {"1.1.1.1", 443}, {"2.2.2.2", 443}, + } + if !slices.Equal(got, want) { + t.Errorf("feed = %v, want %v", got, want) + } +} + +// The feeder must not block forever once the run is over. +func TestFeedStopsWithTheContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + queue := make(chan target) + done := make(chan struct{}) + go func() { feed(ctx, queue, []string{"1.1.1.1"}, []int{80, 443, 8080}); close(done) }() + + for range queue { + } + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("feed did not return after its context ended") + } +} + +// A refusal is a definitive answer; only inconclusive results are retried. +func TestConnectScanFindsAListeningPortAndNotAClosedOne(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Skip("cannot listen on loopback:", err) + } + defer func() { _ = ln.Close() }() + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + _ = c.Close() + } + }() + + openPort := ln.Addr().(*net.TCPAddr).Port + closed, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Skip("cannot reserve a closed port:", err) + } + closedPort := closed.Addr().(*net.TCPAddr).Port + _ = closed.Close() + + s := &Scanner{opts: Options{ + Concurrency: 4, + Rate: 1000, + Timeout: 2 * time.Second, + Logger: discardLogger(), + }} + + found, err := s.scanConnect(context.Background(), []string{"127.0.0.1"}, + portSpec{List: strconv.Itoa(openPort) + "," + strconv.Itoa(closedPort)}, ratelimit.New(0)) + if err != nil { + t.Fatal(err) + } + if !slices.Contains(found["127.0.0.1"], openPort) { + t.Errorf("found = %v, want the listening port %d", found, openPort) + } + if slices.Contains(found["127.0.0.1"], closedPort) { + t.Errorf("found = %v, must not contain the closed port %d", found, closedPort) + } +} + +func TestNewRejectsUnusableOptions(t *testing.T) { + base := Options{Mode: ModeConnect, Ports: PortsTop100, Concurrency: 10, Rate: 100, Timeout: time.Second, Logger: discardLogger()} + for name, mutate := range map[string]func(*Options){ + "no logger": func(o *Options) { o.Logger = nil }, + "no concurrency": func(o *Options) { o.Concurrency = 0 }, + "no rate": func(o *Options) { o.Rate = 0 }, + "no timeout": func(o *Options) { o.Timeout = 0 }, + "bad ports": func(o *Options) { o.Ports = "http" }, + "bad exclusion": func(o *Options) { o.ExcludePorts = "nope" }, + } { + opts := base + mutate(&opts) + if _, err := New(opts); err == nil { + t.Errorf("New accepted options with %s", name) + } + } + if _, err := New(base); err != nil { + t.Errorf("New rejected valid options: %v", err) + } +} + +// The worker count is what --scan-concurrency promises. A goroutine per probe +// would allocate a stack for every (address, port) pair up front: a full +// sweep is millions of them, and the process is killed for memory long before +// the deadline it was budgeted for. +func TestScanConnectGoroutinesStayBounded(t *testing.T) { + const ( + workers = 8 + ports = 20000 + ) + s := &Scanner{ + opts: Options{ + Concurrency: workers, + Rate: 1_000_000, + Timeout: 200 * time.Millisecond, + Logger: discardLogger(), + }, + } + + baseline := runtime.NumGoroutine() + var peak atomic.Int64 + stop := make(chan struct{}) + go func() { + for { + select { + case <-stop: + return + default: + if n := int64(runtime.NumGoroutine()); n > peak.Load() { + peak.Store(n) + } + runtime.Gosched() + } + } + }() + + // Loopback high ports refuse instantly, so this measures scheduling, not + // network waits. + if _, err := s.scanConnect(context.Background(), []string{"127.0.0.1"}, portSpec{List: "20000-" + strconv.Itoa(20000+ports-1)}, ratelimit.New(0)); err != nil { + t.Fatal(err) + } + close(stop) + + // Workers, the feeder, the sampler, and the test's own goroutines — far + // below one per probe. + if limit := int64(baseline + workers + 50); peak.Load() > limit { + t.Errorf("peak goroutines = %d, want at most %d for %d probes at concurrency %d", + peak.Load(), limit, ports, workers) + } +} + +// A warm instance serves many requests from one Scanner. Holding the limiter +// on the scanner and stopping it at the end of a run left every later run +// unlimited, which is invisible until a target notices. +func TestRateLimitStillAppliesOnASecondScan(t *testing.T) { + var seen []*ratelimit.Limiter + s := newScanner(t, false, func(_ context.Context, _ []string, _ portSpec, l *ratelimit.Limiter) (map[string][]int, error) { + seen = append(seen, l) + if !l.Wait(context.Background()) { + t.Error("the limiter refused a token on a live context") + } + return map[string][]int{"1.2.3.4": {80}}, nil + }) + s.opts.Rate = 100 + + hosts := []report.Host{{Host: "a.example.com", Status: report.StatusLive, Addresses: []string{"1.2.3.4"}}} + for run := range 2 { + if _, err := s.Scan(context.Background(), hosts); err != nil { + t.Fatalf("run %d: %v", run, err) + } + } + + if len(seen) != 2 { + t.Fatalf("scan ran %d times, want 2", len(seen)) + } + if seen[0] == seen[1] { + t.Error("both runs shared one limiter; the second would run unlimited once the first stopped it") + } +} diff --git a/internal/portscan/scanner.go b/internal/portscan/scanner.go new file mode 100644 index 0000000..55e4436 --- /dev/null +++ b/internal/portscan/scanner.go @@ -0,0 +1,264 @@ +// Package portscan finds the open ports of the live hosts. +// +// The scanner is a built-in TCP connect scanner. naabu was the intended +// engine, but it reaches libc through purego to batch raw sends, which forces +// a dynamically linked binary — one that cannot start in the distroless +// static image every deployment here is built on. The only thing it offered +// beyond this scanner was SYN mode, which needs privileges the target +// environment does not grant anyway. +package portscan + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net" + "sort" + "time" + + "github.com/projectdiscovery/cdncheck" + + "github.com/JoshuaMart/FastRecon/internal/pipeline" + "github.com/JoshuaMart/FastRecon/internal/ratelimit" + "github.com/JoshuaMart/FastRecon/internal/report" +) + +// Scan modes. +const ( + ModeConnect = "connect" + ModeSYN = "syn" +) + +// Options configures the port scanner. +type Options struct { + Mode string + Ports string + ExcludePorts string + // SkipCDN limits CDN and WAF addresses to the standard web ports. It does + // not affect detection, which always runs. + SkipCDN bool + Concurrency int + Rate int + Retries int + Timeout time.Duration + Logger *slog.Logger +} + +// Scanner is the built-in PortScanner. +type Scanner struct { + opts Options + ports portSpec + cdn *cdncheck.Client + // scan is the single point where sockets are opened. It is a field so the + // planning, batching and result-mapping logic can be tested without + // touching the network. + scan func(ctx context.Context, addresses []string, ports portSpec, limiter *ratelimit.Limiter) (map[string][]int, error) +} + +// New validates the options and prepares the scanner. +func New(opts Options) (*Scanner, error) { + if opts.Logger == nil { + return nil, errors.New("portscan: logger is required") + } + if opts.Concurrency < 1 || opts.Rate < 1 { + return nil, errors.New("portscan: concurrency and rate must be at least 1") + } + if opts.Timeout <= 0 { + return nil, errors.New("portscan: timeout must be positive") + } + + ports, err := parsePorts(opts.Ports) + if err != nil { + return nil, fmt.Errorf("portscan: %w", err) + } + if opts.ExcludePorts != "" { + if err := validatePortExpression(opts.ExcludePorts); err != nil { + return nil, fmt.Errorf("portscan: exclude-%w", err) + } + } + + // SYN mode is refused rather than silently downgraded: a SYN scan the + // process cannot perform finds no open ports at all, which reads exactly + // like a host with nothing listening. + if opts.Mode == ModeSYN { + return nil, errors.New("portscan: syn mode is not available in this build; it needs raw sockets, which the serverless and container deployments do not grant. Use --scan-mode=connect") + } + if opts.Mode != ModeConnect { + return nil, fmt.Errorf("portscan: unknown scan mode %q", opts.Mode) + } + + s := &Scanner{opts: opts, ports: ports, cdn: cdncheck.New()} + s.scan = s.scanConnect + return s, nil +} + +// Name identifies the stage implementation. +func (s *Scanner) Name() string { return "connect" } + +// Scan finds the open ports of every live host. +// +// Only live hosts are scanned. A dead host has no address to connect to, and +// a wildcard artifact is not a host at all — scanning either would spend the +// budget proving something already known. +func (s *Scanner) Scan(ctx context.Context, hosts []report.Host) (pipeline.PortScan, error) { + out := pipeline.PortScan{Hosts: hosts} + + // The limiter belongs to the run, not to the scanner. Holding it on the + // scanner and stopping it here left a reused instance with a stopped + // limiter, whose Wait returns immediately — the second run of a warm + // process would silently lose its rate limit entirely. + // + // One limiter for both passes: one per pass would hand the full + // configured rate to each, so --scan-rate would not describe the run. + limiter := ratelimit.New(s.opts.Rate) + defer limiter.Stop() + + // Several subdomains commonly resolve to one address; scanning it once + // and mapping the result back is the difference between one scan and + // fifty identical ones. + byAddress := indexAddresses(hosts) + if len(byAddress) == 0 { + s.opts.Logger.Info("port scan skipped", "reason", "no live host with an address") + return out, nil + } + + addresses := sortedKeys(byAddress) + edges := classify(s.cdn, addresses) + plain, behindEdge := splitByEdge(addresses, edges, s.opts.SkipCDN) + + s.opts.Logger.Debug("port scan planned", + "addresses", len(addresses), + "behind_cdn", len(edges), + "mode", s.opts.Mode, + "ports", s.ports.String(), + "limited_addresses", len(behindEdge), + ) + + open := map[string][]int{} + var truncated bool + + if len(plain) > 0 { + found, err := s.scan(ctx, plain, s.ports, limiter) + if err != nil { + return out, err + } + merge(open, found) + } + if len(behindEdge) > 0 { + // The restricted pass: a CDN edge answers for thousands of unrelated + // customers, so its full port list describes the provider, not this + // target. + found, err := s.scan(ctx, behindEdge, portSpec{List: joinPorts(cdnPorts)}, limiter) + if err != nil { + return out, err + } + merge(open, found) + } + if ctx.Err() != nil { + truncated = true + out.Warnings = append(out.Warnings, "port scan cut short by its deadline, open ports may be missing") + } + + out.Hosts = s.attach(hosts, byAddress, edges, open) + out.Truncated = truncated + if len(edges) > 0 && s.opts.SkipCDN { + out.Warnings = append(out.Warnings, fmt.Sprintf("%d address(es) behind a CDN or WAF were scanned for ports %s only", len(behindEdge), joinPorts(cdnPorts))) + } + return out, nil +} + +// attach maps the per-address results back onto every host that resolves to +// that address, and records the CDN determination. +func (s *Scanner) attach(hosts []report.Host, byAddress map[string][]int, edges map[string]edge, open map[string][]int) []report.Host { + out := make([]report.Host, len(hosts)) + copy(out, hosts) + + // Invert: which addresses belong to each host index. + hostAddresses := make(map[int][]string, len(hosts)) + for addr, indexes := range byAddress { + for _, i := range indexes { + hostAddresses[i] = append(hostAddresses[i], addr) + } + } + + for i := range out { + addrs, ok := hostAddresses[i] + if !ok { + continue + } + sort.Strings(addrs) + + limited := false + seen := map[int]struct{}{} + var ports []int + for _, addr := range addrs { + if _, behind := edges[addr]; behind && s.opts.SkipCDN { + limited = true + } + for _, p := range open[addr] { + if _, dup := seen[p]; dup { + continue + } + seen[p] = struct{}{} + ports = append(ports, p) + } + } + sort.Ints(ports) + + out[i].Ports = make([]report.Port, 0, len(ports)) + for _, p := range ports { + out[i].Ports = append(out[i].Ports, report.Port{Port: p, Protocol: "tcp", State: "open"}) + } + if len(out[i].Ports) == 0 { + out[i].Ports = nil + } + out[i].CDN = cdnEntries(addrs, edges, limited) + } + return out +} + +// indexAddresses maps every scannable address to the hosts that resolve to it. +func indexAddresses(hosts []report.Host) map[string][]int { + out := map[string][]int{} + for i, h := range hosts { + if h.Status != report.StatusLive { + continue + } + for _, addr := range h.Addresses { + if net.ParseIP(addr) == nil { + continue + } + out[addr] = append(out[addr], i) + } + } + return out +} + +// splitByEdge separates the addresses that get the full port sweep from those +// restricted to the web ports. +func splitByEdge(addresses []string, edges map[string]edge, skipCDN bool) (plain, behindEdge []string) { + for _, addr := range addresses { + if _, behind := edges[addr]; behind && skipCDN { + behindEdge = append(behindEdge, addr) + continue + } + plain = append(plain, addr) + } + return plain, behindEdge +} + +func merge(dst, src map[string][]int) { + for k, v := range src { + dst[k] = append(dst[k], v...) + } +} + +func sortedKeys(m map[string][]int) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/internal/portscan/topports.go b/internal/portscan/topports.go new file mode 100644 index 0000000..d89f93b --- /dev/null +++ b/internal/portscan/topports.go @@ -0,0 +1,57 @@ +package portscan + +// The nmap top-ports selections, as published in nmap-services. They are +// embedded rather than fetched so a run has no data dependency at startup. +const ( + nmapTop100 = "" + + "7,9,13,21-23,25-26,37,53,79-81,88,106,110-111,113,119,135,139,143-144,179,199,389,427,443-445," + + "465,513-515,543-544,548,554,587,631,646,873,990,993,995,1025-1029,1110,1433,1720,1723,1755," + + "1900,2000-2001,2049,2121,2717,3000,3128,3306,3389,3986,4899,5000,5009,5051,5060,5101,5190," + + "5357,5432,5631,5666,5800,5900,6000-6001,6646,7070,8000,8008-8009,8080-8081,8443,8888,9100," + + "9999-10000,32768,49152-49157" + + nmapTop1000 = "" + + "1,3-4,6-7,9,13,17,19-26,30,32-33,37,42-43,49,53,70,79-85,88-90,99-100,106,109-111,113,119,125," + + "135,139,143-144,146,161,163,179,199,211-212,222,254-256,259,264,280,301,306,311,340,366,389," + + "406-407,416-417,425,427,443-445,458,464-465,481,497,500,512-515,524,541,543-545,548,554-555," + + "563,587,593,616-617,625,631,636,646,648,666-668,683,687,691,700,705,711,714,720,722,726,749," + + "765,777,783,787,800-801,808,843,873,880,888,898,900-903,911-912,981,987,990,992-993,995," + + "999-1002,1007,1009-1011,1021-1100,1102,1104-1108,1110-1114,1117,1119,1121-1124,1126," + + "1130-1132,1137-1138,1141,1145,1147-1149,1151-1152,1154,1163-1166,1169,1174-1175,1183," + + "1185-1187,1192,1198-1199,1201,1213,1216-1218,1233-1234,1236,1244,1247-1248,1259,1271-1272," + + "1277,1287,1296,1300-1301,1309-1311,1322,1328,1334,1352,1417,1433-1434,1443,1455,1461,1494," + + "1500-1501,1503,1521,1524,1533,1556,1580,1583,1594,1600,1641,1658,1666,1687-1688,1700," + + "1717-1721,1723,1755,1761,1782-1783,1801,1805,1812,1839-1840,1862-1864,1875,1900,1914,1935," + + "1947,1971-1972,1974,1984,1998-2010,2013,2020-2022,2030,2033-2035,2038,2040-2043,2045-2049," + + "2065,2068,2099-2100,2103,2105-2107,2111,2119,2121,2126,2135,2144,2160-2161,2170,2179," + + "2190-2191,2196,2200,2222,2251,2260,2288,2301,2323,2366,2381-2383,2393-2394,2399,2401,2492," + + "2500,2522,2525,2557,2601-2602,2604-2605,2607-2608,2638,2701-2702,2710,2717-2718,2725,2800," + + "2809,2811,2869,2875,2909-2910,2920,2967-2968,2998,3000-3001,3003,3005-3007,3011,3013,3017," + + "3030-3031,3052,3071,3077,3128,3168,3211,3221,3260-3261,3268-3269,3283,3300-3301,3306," + + "3322-3325,3333,3351,3367,3369-3372,3389-3390,3404,3476,3493,3517,3527,3546,3551,3580,3659," + + "3689-3690,3703,3737,3766,3784,3800-3801,3809,3814,3826-3828,3851,3869,3871,3878,3880,3889," + + "3905,3914,3918,3920,3945,3971,3986,3995,3998,4000-4006,4045,4111,4125-4126,4129,4224,4242," + + "4279,4321,4343,4443-4446,4449,4550,4567,4662,4848,4899-4900,4998,5000-5004,5009,5030,5033," + + "5050-5051,5054,5060-5061,5080,5087,5100-5102,5120,5190,5200,5214,5221-5222,5225-5226,5269," + + "5280,5298,5357,5405,5414,5431-5432,5440,5500,5510,5544,5550,5555,5560,5566,5631,5633,5666," + + "5678-5679,5718,5730,5800-5802,5810-5811,5815,5822,5825,5850,5859,5862,5877,5900-5904," + + "5906-5907,5910-5911,5915,5922,5925,5950,5952,5959-5963,5987-5989,5998-6007,6009,6025,6059," + + "6100-6101,6106,6112,6123,6129,6156,6346,6389,6502,6510,6543,6547,6565-6567,6580,6646," + + "6666-6669,6689,6692,6699,6779,6788-6789,6792,6839,6881,6901,6969,7000-7002,7004,7007,7019," + + "7025,7070,7100,7103,7106,7200-7201,7402,7435,7443,7496,7512,7625,7627,7676,7741,7777-7778," + + "7800,7911,7920-7921,7937-7938,7999-8002,8007-8011,8021-8022,8031,8042,8045,8080-8090,8093," + + "8099-8100,8180-8181,8192-8194,8200,8222,8254,8290-8292,8300,8333,8383,8400,8402,8443,8500," + + "8600,8649,8651-8652,8654,8701,8800,8873,8888,8899,8994,9000-9003,9009-9011,9040,9050,9071," + + "9080-9081,9090-9091,9099-9103,9110-9111,9200,9207,9220,9290,9415,9418,9485,9500,9502-9503," + + "9535,9575,9593-9595,9618,9666,9876-9878,9898,9900,9917,9929,9943-9944,9968,9998-10004," + + "10009-10010,10012,10024-10025,10082,10180,10215,10243,10566,10616-10617,10621,10626," + + "10628-10629,10778,11110-11111,11967,12000,12174,12265,12345,13456,13722,13782-13783,14000," + + "14238,14441-14442,15000,15002-15004,15660,15742,16000-16001,16012,16016,16018,16080,16113," + + "16992-16993,17877,17988,18040,18101,18988,19101,19283,19315,19350,19780,19801,19842,20000," + + "20005,20031,20221-20222,20828,21571,22939,23502,24444,24800,25734-25735,26214,27000," + + "27352-27353,27355-27356,27715,28201,30000,30718,30951,31038,31337,32768-32785,33354,33899," + + "34571-34573,35500,38292,40193,40911,41511,42510,44176,44442-44443,44501,45100,48080," + + "49152-49161,49163,49165,49167,49175-49176,49400,49999-50003,50006,50300,50389,50500,50636," + + "50800,51103,51493,52673,52822,52848,52869,54045,54328,55055-55056,55555,55600,56737-56738," + + "57294,57797,58080,60020,60443,61532,61900,62078,63331,64623,64680,65000,65129,65389" +) diff --git a/internal/probe/probe.go b/internal/probe/probe.go new file mode 100644 index 0000000..95a70ab --- /dev/null +++ b/internal/probe/probe.go @@ -0,0 +1,406 @@ +// Package probe identifies the HTTP services behind the open ports. +// +// The client is httpx's, used at the request level rather than through its +// CLI runner: the runner calls gologger.Fatal — and therefore os.Exit — on +// several ordinary paths, and its enumeration entry point takes no context, +// so a run could neither be cancelled nor survive a bad input. +package probe + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net" + "net/http" + "sort" + "strconv" + "strings" + "sync" + "time" + "unicode/utf8" + + "github.com/projectdiscovery/httpx/common/httpx" + wappalyzer "github.com/projectdiscovery/wappalyzergo" + + "github.com/JoshuaMart/FastRecon/internal/pipeline" + "github.com/JoshuaMart/FastRecon/internal/ratelimit" + "github.com/JoshuaMart/FastRecon/internal/report" + "github.com/JoshuaMart/FastRecon/internal/version" +) + +// maxTitleLength keeps a pathological from bloating the report. +const maxTitleLength = 300 + +// Options configures the prober. +type Options struct { + Concurrency int + // Rate caps probes per second. An HTTP request costs a target far more + // than a TCP handshake, so the probe sweep is rate-limited like the scan. + Rate int + Timeout time.Duration + Retries int + FollowRedirects bool + MaxRedirects int + UserAgent string + Headers []string + Logger *slog.Logger +} + +// HTTPX is the httpx-backed Prober. +type HTTPX struct { + opts Options + client *httpx.HTTPX + // direct never follows redirects. It is the fallback for a service whose + // redirect target is unreachable: following the chain would fail the + // whole request and lose a response that is itself a finding. + direct *httpx.HTTPX + tech *wappalyzer.Wappalyze + // probe is the single point where requests happen, so the scheme + // selection and result mapping can be tested without a network. + probe func(ctx context.Context, host string, port int) *report.HTTP +} + +// New builds the prober. +func New(opts Options) (*HTTPX, error) { + switch { + case opts.Logger == nil: + return nil, errors.New("probe: logger is required") + case opts.Concurrency < 1: + return nil, errors.New("probe: concurrency must be at least 1") + case opts.Timeout <= 0: + return nil, errors.New("probe: timeout must be positive") + case opts.Retries < 0: + return nil, errors.New("probe: retries must not be negative") + } + + headers, err := parseHeaders(opts.Headers) + if err != nil { + return nil, fmt.Errorf("probe: %w", err) + } + + userAgent := opts.UserAgent + if userAgent == "" { + userAgent = version.UserAgent() + } + + newClient := func(follow bool) (*httpx.HTTPX, error) { + return httpx.New(&httpx.Options{ + Timeout: opts.Timeout, + RetryMax: opts.Retries, + FollowRedirects: follow, + MaxRedirects: opts.MaxRedirects, + DefaultUserAgent: userAgent, + CustomHeaders: headers, + // The certificate is a finding of its own: its SANs routinely + // name hosts the enumeration never saw. + TLSGrab: true, + }) + } + + client, err := newClient(opts.FollowRedirects) + if err != nil { + return nil, fmt.Errorf("probe: %w", err) + } + direct := client + if opts.FollowRedirects { + if direct, err = newClient(false); err != nil { + return nil, fmt.Errorf("probe: %w", err) + } + } + + tech, err := wappalyzer.New() + if err != nil { + return nil, fmt.Errorf("probe: technology fingerprints: %w", err) + } + + h := &HTTPX{opts: opts, client: client, direct: direct, tech: tech} + h.probe = h.probeOne + return h, nil +} + +// Name identifies the stage implementation. +func (h *HTTPX) Name() string { return "httpx" } + +// Probe checks every open port for an HTTP service. +// +// Only the ports the scan actually found are probed. Assuming 80 and 443 +// would report services that were never observed and miss the ones on +// unusual ports, which is the entire reason the scan runs first. +func (h *HTTPX) Probe(ctx context.Context, hosts []report.Host) (pipeline.Probe, error) { + out := pipeline.Probe{Hosts: hosts} + + targets := plan(hosts) + if len(targets) == 0 { + h.opts.Logger.Info("http probe skipped", "reason", "no open port to probe") + return out, nil + } + + h.opts.Logger.Debug("http probe started", "targets", len(targets), "concurrency", h.opts.Concurrency, "rate", h.opts.Rate) + + // The limiter belongs to the run, not to the prober: a stopped limiter's + // Wait returns immediately, so a reused instance would lose its rate + // limit from the second run onwards. + limiter := ratelimit.New(h.opts.Rate) + defer limiter.Stop() + + results := make([]*report.HTTP, len(targets)) + var ( + wg sync.WaitGroup + mu sync.Mutex + unchecked int + ) + skip := func() { + mu.Lock() + unchecked++ + mu.Unlock() + } + + // A fixed pool over a stream of indexes, rather than a goroutine per + // target: the worker count is what --probe-concurrency promises, and + // nothing is allocated for work that may never start. + queue := make(chan int) + for range h.opts.Concurrency { + wg.Add(1) + go func() { + defer wg.Done() + for i := range queue { + if ctx.Err() != nil || !limiter.Wait(ctx) { + skip() + continue + } + results[i] = h.probe(ctx, targets[i].host, targets[i].port) + } + }() + } + + for i := range targets { + select { + case queue <- i: + case <-ctx.Done(): + skip() + } + } + close(queue) + wg.Wait() + + out.Hosts = attach(hosts, targets, results) + if unchecked > 0 { + out.Truncated = true + out.Warnings = append(out.Warnings, fmt.Sprintf("%d of %d open ports were not probed before the stage deadline", unchecked, len(targets))) + } + return out, nil +} + +// target is one open port to probe. +type target struct { + hostIndex int + portIndex int + host string + port int +} + +// plan lists every open port found by the scan. +func plan(hosts []report.Host) []target { + var out []target + for hi, h := range hosts { + for pi, p := range h.Ports { + if p.State != "open" { + continue + } + out = append(out, target{hostIndex: hi, portIndex: pi, host: h.Host, port: p.Port}) + } + } + return out +} + +// attach writes the probe results back onto their ports. +func attach(hosts []report.Host, targets []target, results []*report.HTTP) []report.Host { + out := make([]report.Host, len(hosts)) + copy(out, hosts) + for i := range out { + out[i].Ports = append([]report.Port(nil), hosts[i].Ports...) + } + for i, t := range targets { + if results[i] == nil { + continue + } + out[t.hostIndex].Ports[t.portIndex].HTTP = results[i] + } + return out +} + +// probeOne tries a port over HTTPS, then over plain HTTP. +// +// HTTPS is always tried first, on every port, because the TLS handshake is +// the only reliable discriminator. Trying HTTP first would misclassify TLS +// ports: a plain request to an HTTPS port commonly returns a real HTTP 400 +// ("The plain HTTP request was sent to HTTPS port"), which looks exactly like +// a working HTTP service. A handshake, by contrast, either succeeds or fails. +func (h *HTTPX) probeOne(ctx context.Context, host string, port int) *report.HTTP { + for _, scheme := range []string{"https", "http"} { + if ctx.Err() != nil { + return nil + } + if svc := h.request(ctx, scheme, host, port); svc != nil { + return svc + } + } + return nil +} + +func (h *HTTPX) request(ctx context.Context, scheme, host string, port int) *report.HTTP { + target := scheme + "://" + net.JoinHostPort(host, strconv.Itoa(port)) + + resp, err := h.do(ctx, h.client, target) + unfollowed := false + if err != nil && h.opts.FollowRedirects { + // The chain may have broken on a later hop — a redirect to a host + // whose TLS handshake fails, say. The first hop still answered, and + // a 301 from an open port is a finding worth keeping. + if direct, directErr := h.do(ctx, h.direct, target); directErr == nil { + resp, err, unfollowed = direct, nil, true + } + } + if err != nil { + h.opts.Logger.Debug("probe failed", "target", target, "error", err) + return nil + } + + svc := &report.HTTP{ + URL: target, + Scheme: scheme, + RedirectUnfollowed: unfollowed, + StatusCode: resp.StatusCode, + ContentLength: int64(resp.ContentLength), + ResponseTime: resp.Duration.Milliseconds(), + Title: truncate(httpx.ExtractTitle(resp), maxTitleLength), + Server: firstHeader(resp.Headers, "Server"), + Tech: h.fingerprint(resp), + Redirects: chain(resp), + } + // The certificate is only recorded for a connection that was itself TLS. + // A plain-HTTP probe that followed a redirect to an HTTPS host would + // otherwise attach that other endpoint's certificate to this port. + if scheme == "https" { + svc.TLS = certificate(resp) + } + if final := finalURL(resp); final != "" && final != target { + svc.FinalURL = final + } + return svc +} + +// do issues one request with the given client. +func (h *HTTPX) do(ctx context.Context, client *httpx.HTTPX, target string) (*httpx.Response, error) { + req, err := client.NewRequestWithContext(ctx, http.MethodGet, target) + if err != nil { + return nil, err + } + return client.Do(req, httpx.UnsafeOptions{}) +} + +// fingerprint identifies the technologies behind a response. +func (h *HTTPX) fingerprint(resp *httpx.Response) []string { + if h.tech == nil { + return nil + } + found := h.tech.Fingerprint(resp.Headers, resp.Data) + if len(found) == 0 { + return nil + } + out := make([]string, 0, len(found)) + for name := range found { + out = append(out, name) + } + sort.Strings(out) + return out +} + +// certificate extracts the parts of the TLS response worth reporting. SANs +// may name hosts the enumeration missed; they are recorded but, by design, +// not fed back into the pipeline. +func certificate(resp *httpx.Response) *report.TLS { + if resp.TLSData == nil || resp.TLSData.CertificateResponse == nil { + return nil + } + cert := resp.TLSData.CertificateResponse + out := &report.TLS{ + SubjectCN: cert.SubjectCN, + Issuer: cert.IssuerCN, + NotAfter: cert.NotAfter, + SANs: cert.SubjectAN, + } + if out.SubjectCN == "" && out.Issuer == "" && len(out.SANs) == 0 { + return nil + } + return out +} + +// chain lists the URLs a redirect walked through, so a service that ends +// somewhere unexpected can be traced back. +func chain(resp *httpx.Response) []string { + if len(resp.Chain) == 0 { + return nil + } + out := make([]string, 0, len(resp.Chain)) + for _, item := range resp.Chain { + if item.RequestURL != "" { + out = append(out, item.RequestURL) + } + } + if len(out) == 0 { + return nil + } + return out +} + +// finalURL is where the redirects landed, which is the URL worth recording. +func finalURL(resp *httpx.Response) string { + for i := len(resp.Chain) - 1; i >= 0; i-- { + if url := resp.Chain[i].Location; url != "" { + return url + } + } + return "" +} + +func firstHeader(headers map[string][]string, name string) string { + for k, v := range headers { + if strings.EqualFold(k, name) && len(v) > 0 { + return v[0] + } + } + return "" +} + +func truncate(s string, limit int) string { + s = strings.TrimSpace(s) + if len(s) <= limit { + return s + } + // Cutting at a byte offset can land inside a multi-byte rune, and the + // JSON encoder then rewrites the broken tail to U+FFFD. Non-ASCII titles + // are routine on real targets. + for limit > 0 && !utf8.ValidString(s[:limit]) { + limit-- + } + return s[:limit] + "…" +} + +// parseHeaders turns "Name: value" entries into the client's header map. +func parseHeaders(headers []string) (map[string][]string, error) { + if len(headers) == 0 { + return nil, nil + } + out := make(map[string][]string, len(headers)) + for _, h := range headers { + name, value, ok := strings.Cut(h, ":") + name, value = strings.TrimSpace(name), strings.TrimSpace(value) + if !ok || name == "" || value == "" { + return nil, fmt.Errorf("header %q must be in 'Name: value' form", h) + } + out[name] = append(out[name], value) + } + return out, nil +} diff --git a/internal/probe/probe_test.go b/internal/probe/probe_test.go new file mode 100644 index 0000000..cbae04a --- /dev/null +++ b/internal/probe/probe_test.go @@ -0,0 +1,311 @@ +package probe + +import ( + "context" + "crypto/tls" + "log/slog" + "net/http" + "net/http/httptest" + "net/url" + "os" + "strconv" + "strings" + "testing" + "time" + "unicode/utf8" + + "github.com/JoshuaMart/FastRecon/internal/report" +) + +func discardLogger() *slog.Logger { + return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.Level(99)})) +} + +func TestPlanCoversOnlyOpenPorts(t *testing.T) { + hosts := []report.Host{ + {Host: "a.example.com", Status: report.StatusLive, Ports: []report.Port{ + {Port: 80, State: "open"}, + {Port: 22, State: "filtered"}, + }}, + {Host: "b.example.com", Status: report.StatusLive, Ports: []report.Port{{Port: 8443, State: "open"}}}, + {Host: "dead.example.com", Status: report.StatusDead}, + } + + got := plan(hosts) + if len(got) != 2 { + t.Fatalf("targets = %+v, want the two open ports", got) + } + if got[0].port != 80 || got[1].port != 8443 { + t.Errorf("targets = %+v, want ports 80 and 8443", got) + } + // Assuming 80 and 443 is exactly what the scan exists to avoid. + for _, tg := range got { + if tg.port == 22 { + t.Error("a non-open port was queued for probing") + } + } +} + +func TestAttachWritesResultsOntoTheRightPort(t *testing.T) { + hosts := []report.Host{ + {Host: "a.example.com", Ports: []report.Port{{Port: 80, State: "open"}, {Port: 8080, State: "open"}}}, + {Host: "b.example.com", Ports: []report.Port{{Port: 443, State: "open"}}}, + } + targets := plan(hosts) + results := []*report.HTTP{ + {URL: "http://a.example.com:80", Scheme: "http", StatusCode: http.StatusOK}, + nil, // 8080 answered nothing + {URL: "https://b.example.com:443", Scheme: "https", StatusCode: http.StatusNoContent}, + } + + got := attach(hosts, targets, results) + if got[0].Ports[0].HTTP == nil || got[0].Ports[0].HTTP.StatusCode != http.StatusOK { + t.Errorf("port 80 = %+v, want the first result", got[0].Ports[0].HTTP) + } + if got[0].Ports[1].HTTP != nil { + t.Error("port 8080 answered nothing and must stay bare") + } + if got[1].Ports[0].HTTP == nil || got[1].Ports[0].HTTP.Scheme != "https" { + t.Errorf("port 443 = %+v, want the third result", got[1].Ports[0].HTTP) + } + // The input must not be mutated: the caller still holds it. + if hosts[0].Ports[0].HTTP != nil { + t.Error("attach mutated the hosts it was given") + } +} + +func newProber(t *testing.T, probe func(context.Context, string, int) *report.HTTP) *HTTPX { + t.Helper() + return &HTTPX{opts: Options{Concurrency: 4, Logger: discardLogger()}, probe: probe} +} + +func TestProbeSkipsWhenNothingIsOpen(t *testing.T) { + called := false + h := newProber(t, func(context.Context, string, int) *report.HTTP { + called = true + return nil + }) + + res, err := h.Probe(context.Background(), []report.Host{{Host: "a.example.com", Status: report.StatusLive}}) + if err != nil { + t.Fatal(err) + } + if called { + t.Error("a probe was issued with no open port") + } + if len(res.Hosts) != 1 { + t.Error("hosts must pass through untouched") + } +} + +func TestProbeReportsTruncationWhenOutOfTime(t *testing.T) { + h := newProber(t, func(context.Context, string, int) *report.HTTP { + return &report.HTTP{StatusCode: http.StatusOK} + }) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + res, err := h.Probe(ctx, []report.Host{ + {Host: "a.example.com", Status: report.StatusLive, Ports: []report.Port{{Port: 80, State: "open"}}}, + }) + if err != nil { + t.Fatal(err) + } + if !res.Truncated { + t.Error("a probe stage cut short must report itself truncated") + } + if len(res.Warnings) == 0 { + t.Error("a truncated probe must warn") + } +} + +func TestParseHeaders(t *testing.T) { + got, err := parseHeaders([]string{"X-Trace: 1", "Accept: text/html,application/json"}) + if err != nil { + t.Fatal(err) + } + if got["Accept"][0] != "text/html,application/json" { + t.Errorf("headers = %v, a comma in the value was mangled", got) + } + for _, bad := range []string{"NoColon", ": empty-name", "Name:"} { + if _, err := parseHeaders([]string{bad}); err == nil { + t.Errorf("parseHeaders(%q) accepted a malformed header", bad) + } + } +} + +func TestTruncateKeepsPathologicalTitlesOutOfTheReport(t *testing.T) { + if got := truncate(" spaced ", 100); got != "spaced" { + t.Errorf("truncate = %q, want it trimmed", got) + } + long := strings.Repeat("x", maxTitleLength+50) + got := truncate(long, maxTitleLength) + if len([]rune(got)) != maxTitleLength+1 { + t.Errorf("truncate produced %d runes, want the limit plus an ellipsis", len([]rune(got))) + } +} + +// A title cut at a byte offset can land inside a multi-byte rune, and the +// JSON encoder then silently rewrites the broken tail. +func TestTruncateNeverSplitsARune(t *testing.T) { + for _, r := range []string{"あ", "é", "→", "🙂"} { + title := strings.Repeat(r, maxTitleLength) + got := truncate(title, maxTitleLength) + if !utf8.ValidString(got) { + t.Errorf("truncate(%q…) produced invalid UTF-8", r) + } + if len(got) > maxTitleLength+len("…") { + t.Errorf("truncate(%q…) = %d bytes, want it within the limit", r, len(got)) + } + } +} + +func hostPort(t *testing.T, raw string) (string, int) { + t.Helper() + u, err := url.Parse(raw) + if err != nil { + t.Fatal(err) + } + port, err := strconv.Atoi(u.Port()) + if err != nil { + t.Fatal(err) + } + return u.Hostname(), port +} + +func newRealProber(t *testing.T, follow bool) *HTTPX { + t.Helper() + h, err := New(Options{ + Concurrency: 2, + Timeout: 5 * time.Second, + Retries: 0, + FollowRedirects: follow, + MaxRedirects: 3, + Logger: discardLogger(), + }) + if err != nil { + t.Skip("cannot build a prober here:", err) + } + return h +} + +func TestProbeOneDetectsPlainHTTP(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Server", "test-server") + _, _ = w.Write([]byte("<html><head><title>Plain")) + })) + defer srv.Close() + + host, port := hostPort(t, srv.URL) + got := newRealProber(t, true).probeOne(context.Background(), host, port) + if got == nil { + t.Fatal("no service found on a listening HTTP server") + } + if got.Scheme != "http" || got.StatusCode != http.StatusOK { + t.Errorf("service = %+v, want http/200", got) + } + if got.Title != "Plain" { + t.Errorf("title = %q, want Plain", got.Title) + } + if got.Server != "test-server" { + t.Errorf("server = %q, want test-server", got.Server) + } + if got.TLS != nil { + t.Error("a plain HTTP service must carry no certificate") + } +} + +// HTTPS is tried first on every port, so a TLS service on an unusual port is +// reported with the scheme that actually worked. +func TestProbeOneDetectsHTTPSOnAnUnusualPort(t *testing.T) { + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("Secure")) + })) + srv.TLS = &tls.Config{MinVersion: tls.VersionTLS12} + srv.StartTLS() + defer srv.Close() + + host, port := hostPort(t, srv.URL) + got := newRealProber(t, true).probeOne(context.Background(), host, port) + if got == nil { + t.Fatal("no service found on a listening HTTPS server") + } + if got.Scheme != "https" { + t.Errorf("scheme = %q, want https", got.Scheme) + } + if got.URL != "https://"+host+":"+strconv.Itoa(port) { + t.Errorf("url = %q, want it to match the probed scheme and port", got.URL) + } + if got.TLS == nil { + t.Error("an HTTPS service must carry its certificate") + } +} + +// A plain request to an HTTPS port often returns a real HTTP 400, which looks +// exactly like a working HTTP service. Trying TLS first is what prevents that +// misclassification. +func TestHTTPSIsPreferredOverAPlainTextErrorOnTheSamePort(t *testing.T) { + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("Secure")) + })) + srv.TLS = &tls.Config{MinVersion: tls.VersionTLS12} + srv.StartTLS() + defer srv.Close() + + host, port := hostPort(t, srv.URL) + got := newRealProber(t, true).probeOne(context.Background(), host, port) + if got == nil || got.Scheme != "https" { + t.Fatalf("service = %+v, want the TLS scheme to win", got) + } +} + +// A service whose redirect target is unreachable still answered, and that +// answer is a finding. +func TestBrokenRedirectStillReportsTheFirstHop(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Port 1 is reserved and refuses instantly, so the chain cannot be + // followed. + http.Redirect(w, r, "http://127.0.0.1:1/gone", http.StatusMovedPermanently) + })) + defer srv.Close() + + host, port := hostPort(t, srv.URL) + got := newRealProber(t, true).probeOne(context.Background(), host, port) + if got == nil { + t.Fatal("a service that redirects to a dead target was reported as absent") + } + if got.StatusCode != http.StatusMovedPermanently { + t.Errorf("status = %d, want the 301 preserved", got.StatusCode) + } + if !got.RedirectUnfollowed { + t.Error("a broken chain must be marked, not hidden") + } +} + +func TestProbeOneReturnsNothingOnAClosedPort(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + host, port := hostPort(t, srv.URL) + srv.Close() + + if got := newRealProber(t, false).probeOne(context.Background(), host, port); got != nil { + t.Errorf("service = %+v, want nothing on a closed port", got) + } +} + +func TestNewRejectsUnusableOptions(t *testing.T) { + base := Options{Concurrency: 1, Timeout: time.Second, Logger: discardLogger()} + for name, mutate := range map[string]func(*Options){ + "no logger": func(o *Options) { o.Logger = nil }, + "no concurrency": func(o *Options) { o.Concurrency = 0 }, + "no timeout": func(o *Options) { o.Timeout = 0 }, + "negative retries": func(o *Options) { o.Retries = -1 }, + "bad header": func(o *Options) { o.Headers = []string{"nope"} }, + } { + opts := base + mutate(&opts) + if _, err := New(opts); err == nil { + t.Errorf("New accepted options with %s", name) + } + } +} diff --git a/internal/ratelimit/ratelimit.go b/internal/ratelimit/ratelimit.go new file mode 100644 index 0000000..b5b736f --- /dev/null +++ b/internal/ratelimit/ratelimit.go @@ -0,0 +1,78 @@ +// Package ratelimit caps how fast a stage issues probes. +// +// A scan or a probe sweep that ignores rate is indistinguishable, from the +// target's side, from an attack. +package ratelimit + +import ( + "context" + "sync" + "time" +) + +// Limiter is a token bucket. A zero or negative rate disables it, in which +// case Wait never blocks. +type Limiter struct { + tokens chan struct{} + done chan struct{} + once sync.Once +} + +// New starts a limiter delivering perSecond tokens per second. +func New(perSecond int) *Limiter { + l := &Limiter{ + tokens: make(chan struct{}, max(perSecond/10, 1)), + done: make(chan struct{}), + } + if perSecond <= 0 { + // Through the once, so a later Stop is not a second close. + l.Stop() + return l + } + + interval := time.Second / time.Duration(perSecond) + // Below the timer's practical resolution, refill in batches instead of + // ticking once per token. + batch := 1 + if interval < time.Millisecond { + batch = int(time.Millisecond / interval) + interval = time.Millisecond + } + + go func() { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-l.done: + return + case <-ticker.C: + for range batch { + select { + case l.tokens <- struct{}{}: + default: + } + } + } + } + }() + return l +} + +// Wait blocks for a token, reporting false if ctx ended first. +func (l *Limiter) Wait(ctx context.Context) bool { + select { + case <-l.done: + return true + default: + } + select { + case <-l.tokens: + return true + case <-ctx.Done(): + return false + } +} + +// Stop releases the limiter's ticker. +func (l *Limiter) Stop() { l.once.Do(func() { close(l.done) }) } diff --git a/internal/ratelimit/ratelimit_test.go b/internal/ratelimit/ratelimit_test.go new file mode 100644 index 0000000..ee4c18b --- /dev/null +++ b/internal/ratelimit/ratelimit_test.go @@ -0,0 +1,66 @@ +package ratelimit + +import ( + "context" + "testing" + "time" +) + +func TestLimiterCapsThroughput(t *testing.T) { + l := New(200) + defer l.Stop() + + start := time.Now() + for range 40 { + if !l.Wait(context.Background()) { + t.Fatal("Wait returned false on a live context") + } + } + // 40 tokens at 200/s cannot arrive in much under 100ms once the initial + // bucket is drained. + if elapsed := time.Since(start); elapsed < 50*time.Millisecond { + t.Errorf("40 tokens took %s at 200/s, want the rate to bite", elapsed) + } +} + +// A disabled limiter must not block, or a stage configured without a rate +// would stall forever. +func TestZeroRateNeverBlocks(t *testing.T) { + l := New(0) + defer l.Stop() + + done := make(chan struct{}) + go func() { + for range 1000 { + l.Wait(context.Background()) + } + close(done) + }() + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("a zero rate blocked") + } +} + +func TestWaitReleasesOnContextEnd(t *testing.T) { + l := New(1) + defer l.Stop() + + // Drain whatever the bucket starts with. + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + for l.Wait(ctx) { + } + + if l.Wait(ctx) { + t.Error("Wait returned true after its context ended") + } +} + +func TestStopIsIdempotent(t *testing.T) { + l := New(10) + l.Stop() + l.Stop() +} diff --git a/internal/report/format.go b/internal/report/format.go new file mode 100644 index 0000000..1ad01ea --- /dev/null +++ b/internal/report/format.go @@ -0,0 +1,200 @@ +package report + +import ( + "bytes" + "encoding/json" + "fmt" + "sort" + "strings" +) + +// Format selects how a report is rendered. Every sink receives the same bytes, +// so the format is a property of the run, not of the destination. +type Format string + +const ( + // FormatJSON is one indented document: the default, and the only format + // that carries the whole report. + FormatJSON Format = "json" + // FormatJSONL is one host per line, for scopes too large to hold in + // memory downstream. Run metadata is not in the stream — it goes to the + // log, which carries the same counters. + FormatJSONL Format = "jsonl" + // FormatText is a human summary. + FormatText Format = "text" +) + +// ParseFormat resolves a format name. +func ParseFormat(s string) (Format, error) { + switch Format(strings.ToLower(s)) { + case FormatJSON: + return FormatJSON, nil + case FormatJSONL: + return FormatJSONL, nil + case FormatText: + return FormatText, nil + default: + return "", fmt.Errorf("unknown format %q (valid: json, jsonl, text)", s) + } +} + +func (f Format) String() string { return string(f) } + +// Ext returns the file extension conventionally used for the format. +func (f Format) Ext() string { + switch f { + case FormatJSONL: + return ".jsonl" + case FormatText: + return ".txt" + default: + return ".json" + } +} + +// Render encodes the report in the given format. +func (r *Report) Render(f Format) ([]byte, error) { + switch f { + case FormatJSON: + return renderJSON(r) + case FormatJSONL: + return renderJSONL(r) + case FormatText: + return renderText(r), nil + default: + return nil, fmt.Errorf("unknown format %q", f) + } +} + +func renderJSON(r *Report) ([]byte, error) { + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetIndent("", " ") + enc.SetEscapeHTML(false) + if err := enc.Encode(r); err != nil { + return nil, fmt.Errorf("encode report: %w", err) + } + return bytes.TrimRight(buf.Bytes(), "\n"), nil +} + +func renderJSONL(r *Report) ([]byte, error) { + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetEscapeHTML(false) + for i := range r.Hosts { + if err := enc.Encode(r.Hosts[i]); err != nil { + return nil, fmt.Errorf("encode host %s: %w", r.Hosts[i].Host, err) + } + } + return bytes.TrimRight(buf.Bytes(), "\n"), nil +} + +func renderText(r *Report) []byte { + var b strings.Builder + + fmt.Fprintf(&b, "run %s\n", r.Run.ID) + fmt.Fprintf(&b, "domain %s\n", r.Run.Domain) + fmt.Fprintf(&b, "scope %s (%s)\n", r.Run.Scope, strings.Join(r.Run.Stages, " > ")) + fmt.Fprintf(&b, "duration %s\n", formatMillis(r.Run.Duration)) + status := "complete" + if r.Run.TruncatedByTimeout { + status = "TRUNCATED BY TIMEOUT" + } else if !r.Run.Completed { + status = "INCOMPLETE" + } + fmt.Fprintf(&b, "status %s\n", status) + + if len(r.Sources) > 0 { + b.WriteString("\nsources\n") + sources := append([]Source(nil), r.Sources...) + sort.Slice(sources, func(i, j int) bool { return sources[i].Name < sources[j].Name }) + for _, s := range sources { + line := fmt.Sprintf(" %-18s %-15s %6d", s.Name, s.Status, s.Found) + if s.Partial { + line += " (partial)" + } + if s.Error != "" { + line += " " + s.Error + } + b.WriteString(line + "\n") + } + } + + fmt.Fprintf(&b, "\nstats\n") + for _, kv := range []struct { + k string + v int + }{ + {"enumerated", r.Stats.Enumerated}, + {"excluded", r.Stats.Excluded}, + {"in scope", r.Stats.InScope}, + {"live", r.Stats.Live}, + {"dead", r.Stats.Dead}, + {"wildcard", r.Stats.Wildcard}, + {"open ports", r.Stats.OpenPorts}, + {"http services", r.Stats.HTTPServices}, + } { + fmt.Fprintf(&b, " %-14s %d\n", kv.k, kv.v) + } + + if len(r.Hosts) > 0 { + b.WriteString("\nhosts\n") + for _, h := range r.Hosts { + fmt.Fprintf(&b, " %-45s %-9s %s\n", h.Host, h.Status, hostDetail(h)) + } + } + + if len(r.Warnings) > 0 { + b.WriteString("\nwarnings\n") + for _, w := range r.Warnings { + fmt.Fprintf(&b, " - %s\n", w) + } + } + return []byte(strings.TrimRight(b.String(), "\n")) +} + +func hostDetail(h Host) string { + switch { + case h.Status == StatusDead: + return h.Reason + case len(h.Ports) > 0: + parts := make([]string, 0, len(h.Ports)+1) + for _, p := range h.Ports { + if p.HTTP != nil { + parts = append(parts, fmt.Sprintf("%d/%s(%d)", p.Port, p.HTTP.Scheme, p.HTTP.StatusCode)) + continue + } + parts = append(parts, fmt.Sprintf("%d", p.Port)) + } + if cdn := cdnDetail(h.CDN); cdn != "" { + parts = append(parts, cdn) + } + return strings.Join(parts, " ") + default: + return strings.TrimSpace(strings.Join(h.Addresses, " ") + " " + cdnDetail(h.CDN)) + } +} + +// cdnDetail renders the CDN providers behind a host, marking a port list that +// was deliberately narrowed so it is not read as an exhaustive scan. +func cdnDetail(cdns []CDN) string { + if len(cdns) == 0 { + return "" + } + parts := make([]string, 0, len(cdns)) + for _, c := range cdns { + name := c.Name + if c.ScanLimited { + name += ",ports-limited" + } + parts = append(parts, name) + } + return "[cdn:" + strings.Join(parts, " ") + "]" +} + +func formatMillis(ms int64) string { + if ms < 1000 { + return fmt.Sprintf("%dms", ms) + } + return fmt.Sprintf("%.1fs", float64(ms)/1000) +} diff --git a/internal/report/report.go b/internal/report/report.go new file mode 100644 index 0000000..88f40ac --- /dev/null +++ b/internal/report/report.go @@ -0,0 +1,228 @@ +// Package report defines the run report: the single document every sink emits. +// +// There is one shape, versioned by SchemaVersion, for every consumer. A change +// that removes or repurposes a field is a schema version bump. +package report + +import ( + "time" + + "github.com/JoshuaMart/FastRecon/internal/stage" +) + +// SchemaVersion identifies the report contract. +const SchemaVersion = "1.0" + +// Host status values. +const ( + // StatusDiscovered is a host found by enumeration and kept by the + // exclusion filter, but not yet resolved. It is what every host in an + // enumeration-only run looks like. + StatusDiscovered = "discovered" + StatusLive = "live" + StatusDead = "dead" + StatusWildcard = "wildcard" +) + +// Source status values. +const ( + SourceOK = "ok" + SourceSkippedNoKey = "skipped_no_key" + SourceSkipped = "skipped" + SourceError = "error" + SourceTimeout = "timeout" + SourceRateLimited = "rate_limited" +) + +// Reasons a host ended up in the dead bucket. +const ( + ReasonNXDomain = "nxdomain" + ReasonNoAnswer = "no_answer" + ReasonTimeout = "timeout" + ReasonWildcard = "wildcard" +) + +// Report is the complete output of one run. +type Report struct { + SchemaVersion string `json:"schema_version"` + Run Run `json:"run"` + Sources []Source `json:"sources"` + Stats Stats `json:"stats"` + Hosts []Host `json:"hosts"` + Excluded []Excluded `json:"excluded,omitempty"` + Warnings []string `json:"warnings,omitempty"` +} + +// Run holds the metadata of the execution itself. +type Run struct { + ID string `json:"id"` + Domain string `json:"domain"` + Scope string `json:"scope"` + Stages []string `json:"stages"` + Started time.Time `json:"started_at"` + Finished time.Time `json:"finished_at"` + Duration int64 `json:"duration_ms"` + // Completed is false when a stage was cut short or failed outright. The + // report is still emitted and still valid — it is just not exhaustive. + Completed bool `json:"completed"` + TruncatedByTimeout bool `json:"truncated_by_timeout"` + Version string `json:"version"` + Environment string `json:"environment"` +} + +// Source records what one enumeration source contributed and how it went. A +// source that silently returns nothing is a bug worth seeing, so every source +// appears here whether it succeeded or not. +type Source struct { + Name string `json:"name"` + Status string `json:"status"` + Found int `json:"found"` + Duration int64 `json:"duration_ms,omitempty"` + // Partial marks results kept from a source that was abandoned mid-way, + // typically after exhausting its rate-limit budget. + Partial bool `json:"partial,omitempty"` + Error string `json:"error,omitempty"` +} + +// Stats are the run counters, also emitted as the final log line so a run is +// legible in a log-only environment. +type Stats struct { + Enumerated int `json:"enumerated"` + Excluded int `json:"excluded"` + InScope int `json:"in_scope"` + Live int `json:"live"` + Dead int `json:"dead"` + Wildcard int `json:"wildcard"` + OpenPorts int `json:"open_ports"` + HTTPServices int `json:"http_services"` +} + +// Host is one discovered subdomain and everything learned about it. +type Host struct { + Host string `json:"host"` + Status string `json:"status"` + Addresses []string `json:"addresses,omitempty"` + CNAME []string `json:"cname,omitempty"` + Reason string `json:"reason,omitempty"` + CDN []CDN `json:"cdn,omitempty"` + Ports []Port `json:"ports,omitempty"` +} + +// CDN records that some of a host's addresses sit behind a CDN, WAF or cloud +// edge. It is populated whether or not the scan was restricted: a port list +// narrowed to 80 and 443 is indistinguishable from a genuinely minimal host +// unless the report says the narrowing was deliberate. +type CDN struct { + Name string `json:"name"` + // Type is the kind of provider matched: cdn, waf or cloud. + Type string `json:"type,omitempty"` + // Addresses are the host addresses this provider matched. A host can have + // a CDN address and an origin address at once. + Addresses []string `json:"addresses,omitempty"` + // ScanLimited marks a port list restricted to the standard web ports + // because of this provider. + ScanLimited bool `json:"scan_limited"` +} + +// Port is an open port on a host, with the HTTP service behind it if any. +type Port struct { + Port int `json:"port"` + Protocol string `json:"protocol"` + State string `json:"state"` + HTTP *HTTP `json:"http,omitempty"` +} + +// HTTP describes the service answering on a port, with the scheme that +// actually worked rather than one assumed from the port number. +type HTTP struct { + // URL is the URL that was probed. It always matches Scheme and the port, + // so it identifies this service rather than wherever it redirects to. + URL string `json:"url"` + // FinalURL is where the redirects landed, when they went anywhere else. + FinalURL string `json:"final_url,omitempty"` + Scheme string `json:"scheme"` + StatusCode int `json:"status_code"` + Title string `json:"title,omitempty"` + ContentLength int64 `json:"content_length,omitempty"` + ResponseTime int64 `json:"response_time_ms,omitempty"` + Server string `json:"server,omitempty"` + Redirects []string `json:"redirects,omitempty"` + // RedirectUnfollowed marks a service whose redirect target could not be + // reached. The response recorded here is the first hop, which is a real + // finding even though the chain is broken. + RedirectUnfollowed bool `json:"redirect_unfollowed,omitempty"` + Tech []string `json:"tech,omitempty"` + TLS *TLS `json:"tls,omitempty"` +} + +// TLS is the certificate seen on an HTTPS service. SANs may name hosts the +// enumeration missed; they are recorded, not fed back into the pipeline. +type TLS struct { + SubjectCN string `json:"subject_cn,omitempty"` + Issuer string `json:"issuer,omitempty"` + NotAfter time.Time `json:"not_after,omitzero"` + SANs []string `json:"sans,omitempty"` +} + +// Excluded records a host dropped by an exclusion pattern, with the pattern +// responsible — needed to debug an over-broad exclusion. +type Excluded struct { + Host string `json:"host"` + Pattern string `json:"pattern"` +} + +// New starts a report for a run. +func New(id, domain string, scope stage.Scope, version, environment string, started time.Time) *Report { + return &Report{ + SchemaVersion: SchemaVersion, + Run: Run{ + ID: id, + Domain: domain, + Scope: scope.String(), + Stages: scope.StageNames(), + Started: started.UTC(), + Completed: true, + Version: version, + Environment: environment, + }, + Sources: []Source{}, + Hosts: []Host{}, + } +} + +// Warnf appends a warning. Warnings are part of the report, not just the log: +// a consumer must be able to tell a clean run from a degraded one. +func (r *Report) Warnf(format string, args ...any) { + r.Warnings = append(r.Warnings, sprintf(format, args...)) +} + +// Finish stamps the end of the run and recomputes the counters. +func (r *Report) Finish(finished time.Time) { + r.Run.Finished = finished.UTC() + r.Run.Duration = finished.Sub(r.Run.Started).Milliseconds() + r.recount() +} + +// recount derives the per-host counters from Hosts. Enumerated, Excluded and +// InScope describe stages that happen before the host list exists, so they are +// set by the pipeline and left alone here. +func (r *Report) recount() { + r.Stats.Live, r.Stats.Dead, r.Stats.Wildcard = 0, 0, 0 + r.Stats.OpenPorts, r.Stats.HTTPServices = 0, 0 + for _, h := range r.Hosts { + switch h.Status { + case StatusLive: + r.Stats.Live++ + case StatusWildcard: + r.Stats.Wildcard++ + case StatusDead: + r.Stats.Dead++ + } + for _, p := range h.Ports { + r.Stats.OpenPorts++ + if p.HTTP != nil { + r.Stats.HTTPServices++ + } + } + } +} diff --git a/internal/report/report_test.go b/internal/report/report_test.go new file mode 100644 index 0000000..005df0f --- /dev/null +++ b/internal/report/report_test.go @@ -0,0 +1,141 @@ +package report + +import ( + "bytes" + "encoding/json" + "strings" + "testing" + "time" + + "github.com/JoshuaMart/FastRecon/internal/stage" +) + +func sample(t *testing.T) *Report { + t.Helper() + started := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + r := New("01ABC", "example.com", stage.ScopeFull, "1.2.3", "serverless-job", started) + r.Stats.Enumerated = 3 + r.Stats.Excluded = 1 + r.Stats.InScope = 2 + r.Sources = []Source{{Name: "chaos", Status: SourceOK, Found: 3}} + r.Hosts = []Host{ + { + Host: "api.example.com", + Status: StatusLive, + Addresses: []string{"93.184.216.34"}, + CDN: []CDN{{Name: "cloudflare", Type: "waf", Addresses: []string{"93.184.216.34"}, ScanLimited: true}}, + Ports: []Port{ + {Port: 443, Protocol: "tcp", State: "open", HTTP: &HTTP{URL: "https://api.example.com", Scheme: "https", StatusCode: 200}}, + {Port: 22, Protocol: "tcp", State: "open"}, + }, + }, + {Host: "old.example.com", Status: StatusDead, Reason: ReasonNXDomain}, + } + r.Finish(started.Add(7 * time.Second)) + return r +} + +func TestFinishDerivesCounters(t *testing.T) { + r := sample(t) + if r.Stats.Live != 1 || r.Stats.Dead != 1 { + t.Errorf("live/dead = %d/%d, want 1/1", r.Stats.Live, r.Stats.Dead) + } + if r.Stats.OpenPorts != 2 { + t.Errorf("open ports = %d, want 2", r.Stats.OpenPorts) + } + if r.Stats.HTTPServices != 1 { + t.Errorf("http services = %d, want 1: only one port answered HTTP", r.Stats.HTTPServices) + } + // Pre-host counters are the pipeline's to set and must survive Finish. + if r.Stats.Enumerated != 3 || r.Stats.Excluded != 1 || r.Stats.InScope != 2 { + t.Errorf("pre-host counters were overwritten: %+v", r.Stats) + } + if r.Run.Duration != 7000 { + t.Errorf("duration = %dms, want 7000", r.Run.Duration) + } +} + +func TestRenderJSONRoundTrips(t *testing.T) { + data, err := sample(t).Render(FormatJSON) + if err != nil { + t.Fatalf("Render: %v", err) + } + var back Report + if err := json.Unmarshal(data, &back); err != nil { + t.Fatalf("report is not valid JSON: %v", err) + } + if back.SchemaVersion != SchemaVersion { + t.Errorf("schema_version = %q, want %q", back.SchemaVersion, SchemaVersion) + } + if len(back.Hosts) != 2 { + t.Errorf("hosts = %d, want 2", len(back.Hosts)) + } + if back.Hosts[0].Ports[0].HTTP.Scheme != "https" { + t.Error("the working scheme must survive a round trip") + } + if len(back.Hosts[0].CDN) != 1 || !back.Hosts[0].CDN[0].ScanLimited { + t.Error("the CDN determination and its scan_limited marker must survive a round trip") + } +} + +// A port list narrowed to the web ports must never read as an exhaustive scan. +func TestRenderTextMarksNarrowedCDNScans(t *testing.T) { + out, err := sample(t).Render(FormatText) + if err != nil { + t.Fatalf("Render: %v", err) + } + s := string(out) + if !strings.Contains(s, "cdn:cloudflare") { + t.Error("the CDN provider is missing from the text output") + } + if !strings.Contains(s, "ports-limited") { + t.Error("a deliberately narrowed port list must be marked in the text output") + } +} + +func TestRenderJSONLIsOneHostPerLine(t *testing.T) { + data, err := sample(t).Render(FormatJSONL) + if err != nil { + t.Fatalf("Render: %v", err) + } + lines := bytes.Split(bytes.TrimSpace(data), []byte("\n")) + if len(lines) != 2 { + t.Fatalf("got %d lines, want one per host", len(lines)) + } + for i, line := range lines { + var h Host + if err := json.Unmarshal(line, &h); err != nil { + t.Errorf("line %d is not a host object: %v", i, err) + } + } +} + +func TestRenderTextMarksTruncatedRuns(t *testing.T) { + r := sample(t) + r.Run.Completed = false + r.Run.TruncatedByTimeout = true + r.Warnf("stage %s: run deadline reached", stage.PortScan) + + out, err := r.Render(FormatText) + if err != nil { + t.Fatalf("Render: %v", err) + } + s := string(out) + if !strings.Contains(s, "TRUNCATED BY TIMEOUT") { + t.Error("a truncated run must be obvious in the text output") + } + if !strings.Contains(s, "deadline reached") { + t.Error("warnings are missing from the text output") + } +} + +func TestParseFormat(t *testing.T) { + for _, in := range []string{"json", "JSON", "jsonl", "text"} { + if _, err := ParseFormat(in); err != nil { + t.Errorf("ParseFormat(%q): %v", in, err) + } + } + if _, err := ParseFormat("yaml"); err == nil { + t.Error("ParseFormat(yaml) succeeded, want an error") + } +} diff --git a/internal/report/sprintf.go b/internal/report/sprintf.go new file mode 100644 index 0000000..6499f6c --- /dev/null +++ b/internal/report/sprintf.go @@ -0,0 +1,10 @@ +package report + +import "fmt" + +func sprintf(format string, args ...any) string { + if len(args) == 0 { + return format + } + return fmt.Sprintf(format, args...) +} diff --git a/internal/resolve/dnsx.go b/internal/resolve/dnsx.go new file mode 100644 index 0000000..1797da3 --- /dev/null +++ b/internal/resolve/dnsx.go @@ -0,0 +1,215 @@ +// Package resolve separates the hosts that answer DNS from those that do not. +// +// The engine is dnsx used as a Go library: pure Go, unprivileged, and +// therefore usable in a serverless job where a raw-socket resolver is not. +package resolve + +import ( + "context" + "errors" + "fmt" + "log/slog" + "strings" + "sync" + "time" + + "github.com/miekg/dns" + "github.com/projectdiscovery/dnsx/libs/dnsx" + "github.com/projectdiscovery/retryabledns" + + "github.com/JoshuaMart/FastRecon/internal/pipeline" + "github.com/JoshuaMart/FastRecon/internal/report" +) + +// Options configures the resolver. +type Options struct { + // Domain is the run's root domain, always probed for a wildcard record. + Domain string + Resolvers []string + Concurrency int + Retries int + Timeout time.Duration + // WildcardProbes is how many random names are resolved per parent domain + // to decide whether it carries a wildcard record. + WildcardProbes int + Logger *slog.Logger +} + +// DNSX is the dnsx-backed Resolver. +type DNSX struct { + opts Options + // query is the single point where DNS actually happens, shared by host + // resolution and wildcard probing. Tests replace it. + query func(host string) (*retryabledns.DNSData, error) +} + +// New builds the resolver. +func New(opts Options) (*DNSX, error) { + switch { + case opts.Logger == nil: + return nil, errors.New("resolve: logger is required") + case opts.Domain == "": + return nil, errors.New("resolve: domain is required") + case opts.Concurrency < 1: + return nil, errors.New("resolve: concurrency must be at least 1") + case opts.Timeout <= 0: + return nil, errors.New("resolve: timeout must be positive") + case opts.WildcardProbes < 1: + return nil, errors.New("resolve: wildcard probes must be at least 1") + case opts.Retries < 0: + return nil, errors.New("resolve: retries must not be negative") + } + + resolvers := opts.Resolvers + if len(resolvers) == 0 { + resolvers = DefaultResolvers + } + + client, err := dnsx.New(dnsx.Options{ + BaseResolvers: resolvers, + // The engine counts total attempts, not extra ones, so zero retries + // still has to mean one attempt. + MaxRetries: opts.Retries + 1, + Timeout: opts.Timeout, + QuestionTypes: []uint16{dns.TypeA, dns.TypeAAAA, dns.TypeCNAME}, + }) + if err != nil { + return nil, fmt.Errorf("resolve: %w", err) + } + + opts.Resolvers = resolvers + return &DNSX{opts: opts, query: client.QueryMultiple}, nil +} + +// Name identifies the stage implementation. +func (r *DNSX) Name() string { return "dnsx" } + +// Resolve classifies every host as live, dead or a wildcard artifact. +// +// Nothing is dropped: a host that does not resolve stays in the report with +// the reason it failed, because a dangling CNAME or a vanished host is a +// finding in its own right. +func (r *DNSX) Resolve(ctx context.Context, hosts []string) (pipeline.Resolution, error) { + var out pipeline.Resolution + if len(hosts) == 0 { + return out, nil + } + + r.opts.Logger.Debug("resolution started", + "hosts", len(hosts), + "resolvers", r.opts.Resolvers, + "concurrency", r.opts.Concurrency, + ) + + // Wildcards are established first: without them a single wildcard record + // turns thousands of junk names into apparently live hosts. + wc := r.detectWildcards(ctx, hosts) + if len(wc.byParent) > 0 { + parents := make([]string, 0, len(wc.byParent)) + for p := range wc.byParent { + parents = append(parents, p) + } + out.Warnings = append(out.Warnings, fmt.Sprintf("wildcard dns on %s: matching hosts are reported as wildcard, not live", strings.Join(parents, ", "))) + } + + results := make([]report.Host, len(hosts)) + var ( + wg sync.WaitGroup + sem = make(chan struct{}, r.opts.Concurrency) + mu sync.Mutex + unchecked int + ) + + for i, host := range hosts { + wg.Add(1) + go func(i int, host string) { + defer wg.Done() + select { + case sem <- struct{}{}: + defer func() { <-sem }() + case <-ctx.Done(): + // Out of time before this host was even started: report it as + // discovered but unresolved rather than inventing a verdict. + results[i] = report.Host{Host: host, Status: report.StatusDiscovered} + mu.Lock() + unchecked++ + mu.Unlock() + return + } + if ctx.Err() != nil { + results[i] = report.Host{Host: host, Status: report.StatusDiscovered} + mu.Lock() + unchecked++ + mu.Unlock() + return + } + results[i] = r.resolveOne(host, wc) + }(i, host) + } + wg.Wait() + + out.Hosts = results + if unchecked > 0 { + out.Truncated = true + out.Warnings = append(out.Warnings, fmt.Sprintf("%d of %d hosts were not resolved before the stage deadline", unchecked, len(hosts))) + } + return out, nil +} + +// resolveOne queries a single host and turns the answer into a verdict. +func (r *DNSX) resolveOne(host string, wc *wildcards) report.Host { + out := report.Host{Host: host} + + data, err := r.query(host) + switch { + case err != nil: + out.Status = report.StatusDead + out.Reason = report.ReasonTimeout + return out + case data == nil: + out.Status = report.StatusDead + out.Reason = report.ReasonNoAnswer + return out + } + + addresses, cnames := answersOf(data) + out.Addresses = addresses + out.CNAME = cnames + + if parent, covered := wc.covers(host, addresses, cnames); covered { + out.Status = report.StatusWildcard + out.Reason = report.ReasonWildcard + r.opts.Logger.Debug("wildcard artifact", "host", host, "parent", parent) + return out + } + + switch { + case len(addresses) > 0: + out.Status = report.StatusLive + case strings.EqualFold(data.StatusCode, "NXDOMAIN"): + out.Status = report.StatusDead + out.Reason = report.ReasonNXDomain + default: + // No address, but a CNAME: a dangling alias, which is exactly the kind + // of thing this stage exists to surface. + out.Status = report.StatusDead + out.Reason = report.ReasonNoAnswer + } + return out +} + +// answersOf extracts the addresses and aliases from a DNS answer. +func answersOf(data *retryabledns.DNSData) (addresses, cnames []string) { + addresses = make([]string, 0, len(data.A)+len(data.AAAA)) + addresses = append(addresses, data.A...) + addresses = append(addresses, data.AAAA...) + for _, c := range data.CNAME { + if n := normalizeName(c); n != "" { + cnames = append(cnames, n) + } + } + if len(addresses) == 0 { + addresses = nil + } + return addresses, cnames +} diff --git a/internal/resolve/health.go b/internal/resolve/health.go new file mode 100644 index 0000000..d578531 --- /dev/null +++ b/internal/resolve/health.go @@ -0,0 +1,181 @@ +package resolve + +import ( + "context" + "log/slog" + "slices" + "sync" + "time" + + "github.com/miekg/dns" + "github.com/projectdiscovery/retryabledns" +) + +// Health-check anchors. +// +// The positive anchor is a name with a well-known, stable answer, so a +// resolver can be caught lying rather than merely being reachable. The +// negative anchor is a random name that must not exist: a resolver answering +// it with an address hijacks NXDOMAIN, which would turn every dead host in +// the report into a live one. +const ( + healthAnchor = "one.one.one.one" + healthNegativeTLD = "com" + defaultHealthWorkers = 200 + healthQueryMaxTimeout = 3 * time.Second +) + +var healthAnchorAddresses = []string{"1.1.1.1", "1.0.0.1"} + +// Dropped records a resolver removed from the pool and why. +type Dropped struct { + Resolver string + Reason string +} + +// Drop reasons. +const ( + dropUnreachable = "no usable answer" + dropLying = "wrong answer for a known name" + dropHijacker = "hijacks NXDOMAIN" +) + +// HealthOptions configures the validation pass. +type HealthOptions struct { + // Budget bounds the whole pass. Resolvers not reached within it are kept + // unchecked and counted, never silently discarded. + Budget time.Duration + Timeout time.Duration + Concurrency int + Logger *slog.Logger +} + +// HealthResult is the outcome of validating a resolver pool. +type HealthResult struct { + Good []string + Dropped []Dropped + Unchecked int + Duration time.Duration +} + +// CheckResolvers removes the resolvers that cannot be trusted to answer +// correctly. +// +// This matters most for the large published lists: they are validated for +// reachability by whoever publishes them, from wherever their validator runs, +// which says nothing about reachability from inside this job's network, nor +// about filtering or NXDOMAIN redirection. +func CheckResolvers(ctx context.Context, resolvers []string, opts HealthOptions) HealthResult { + start := time.Now() + res := HealthResult{} + if len(resolvers) == 0 { + return res + } + + if opts.Concurrency < 1 { + opts.Concurrency = defaultHealthWorkers + } + timeout := min(opts.Timeout, healthQueryMaxTimeout) + if timeout <= 0 { + timeout = healthQueryMaxTimeout + } + + budgetCtx := ctx + if opts.Budget > 0 { + var cancel context.CancelFunc + budgetCtx, cancel = context.WithTimeout(ctx, opts.Budget) + defer cancel() + } + + type outcome struct { + resolver string + reason string + checked bool + } + outcomes := make([]outcome, len(resolvers)) + + var ( + wg sync.WaitGroup + sem = make(chan struct{}, opts.Concurrency) + ) + for i, resolver := range resolvers { + wg.Add(1) + go func(i int, resolver string) { + defer wg.Done() + select { + case sem <- struct{}{}: + defer func() { <-sem }() + case <-budgetCtx.Done(): + outcomes[i] = outcome{resolver: resolver} + return + } + if budgetCtx.Err() != nil { + outcomes[i] = outcome{resolver: resolver} + return + } + outcomes[i] = outcome{resolver: resolver, reason: checkOne(resolver, timeout), checked: true} + }(i, resolver) + } + wg.Wait() + + for _, o := range outcomes { + switch { + case !o.checked: + res.Unchecked++ + res.Good = append(res.Good, o.resolver) + case o.reason == "": + res.Good = append(res.Good, o.resolver) + default: + res.Dropped = append(res.Dropped, Dropped{Resolver: o.resolver, Reason: o.reason}) + } + } + res.Duration = time.Since(start) + + opts.Logger.Info("resolver health check", + "checked", len(resolvers)-res.Unchecked, + "kept", len(res.Good), + "dropped", len(res.Dropped), + "unchecked", res.Unchecked, + "duration_ms", res.Duration.Milliseconds(), + ) + for _, d := range res.Dropped { + opts.Logger.Debug("resolver dropped", "resolver", d.Resolver, "reason", d.Reason) + } + return res +} + +// checkOne is the per-resolver probe. It is a variable so the surrounding +// budget and accounting logic can be tested without touching the network. +var checkOne = checkResolver + +// checkResolver returns the reason a resolver is unusable, or "" if it is fine. +func checkResolver(resolver string, timeout time.Duration) string { + client, err := retryabledns.NewWithOptions(retryabledns.Options{ + BaseResolvers: []string{resolver}, + MaxRetries: 1, + Timeout: timeout, + }) + if err != nil { + return dropUnreachable + } + + positive, err := client.Query(healthAnchor, dns.TypeA) + if err != nil || positive == nil || len(positive.A) == 0 { + return dropUnreachable + } + if !slices.ContainsFunc(positive.A, func(a string) bool { return slices.Contains(healthAnchorAddresses, a) }) { + return dropLying + } + + negative, err := client.Query(randomLabel()+"."+healthNegativeTLD, dns.TypeA) + if err != nil { + // A resolver that answered the first query but not the second is + // flaky rather than malicious; it stays, and the run's own retries + // absorb it. + return "" + } + if negative != nil && len(negative.A) > 0 { + return dropHijacker + } + return "" +} diff --git a/internal/resolve/health_test.go b/internal/resolve/health_test.go new file mode 100644 index 0000000..edc3e40 --- /dev/null +++ b/internal/resolve/health_test.go @@ -0,0 +1,88 @@ +package resolve + +import ( + "context" + "slices" + "testing" + "time" +) + +func withCheck(t *testing.T, fn func(resolver string, timeout time.Duration) string) { + t.Helper() + original := checkOne + checkOne = fn + t.Cleanup(func() { checkOne = original }) +} + +func TestCheckResolversDropsTheBadOnes(t *testing.T) { + withCheck(t, func(resolver string, _ time.Duration) string { + switch resolver { + case "2.2.2.2:53": + return dropHijacker + case "3.3.3.3:53": + return dropUnreachable + default: + return "" + } + }) + + res := CheckResolvers(context.Background(), []string{"1.1.1.1:53", "2.2.2.2:53", "3.3.3.3:53", "4.4.4.4:53"}, HealthOptions{ + Concurrency: 4, + Logger: discardLogger(), + }) + + slices.Sort(res.Good) + if !slices.Equal(res.Good, []string{"1.1.1.1:53", "4.4.4.4:53"}) { + t.Errorf("good = %v, want the two healthy resolvers", res.Good) + } + if len(res.Dropped) != 2 { + t.Fatalf("dropped = %v, want 2", res.Dropped) + } + byResolver := map[string]string{} + for _, d := range res.Dropped { + byResolver[d.Resolver] = d.Reason + } + if byResolver["2.2.2.2:53"] != dropHijacker { + t.Errorf("reasons = %v, want the hijacker named as such", byResolver) + } + if res.Unchecked != 0 { + t.Errorf("unchecked = %d, want 0", res.Unchecked) + } +} + +// Resolvers the budget did not reach are kept and counted. Dropping them +// would silently shrink the pool; claiming they passed would be a lie. +func TestCheckResolversKeepsAndCountsWhatItCouldNotCheck(t *testing.T) { + withCheck(t, func(string, time.Duration) string { + time.Sleep(50 * time.Millisecond) + return dropUnreachable + }) + + resolvers := make([]string, 200) + for i := range resolvers { + resolvers[i] = "1.1.1." + string(rune('0'+i%10)) + ":53" + } + + res := CheckResolvers(context.Background(), resolvers, HealthOptions{ + Budget: 80 * time.Millisecond, + Concurrency: 2, + Logger: discardLogger(), + }) + + if res.Unchecked == 0 { + t.Fatal("nothing was reported unchecked despite an exhausted budget") + } + if len(res.Good)+len(res.Dropped) != len(resolvers) { + t.Errorf("accounting lost resolvers: %d good + %d dropped != %d", len(res.Good), len(res.Dropped), len(resolvers)) + } + if len(res.Good) < res.Unchecked { + t.Error("unchecked resolvers must be kept in the pool") + } +} + +func TestCheckResolversHandlesAnEmptyPool(t *testing.T) { + res := CheckResolvers(context.Background(), nil, HealthOptions{Logger: discardLogger()}) + if len(res.Good) != 0 || len(res.Dropped) != 0 { + t.Errorf("res = %+v, want everything empty", res) + } +} diff --git a/internal/resolve/resolve_test.go b/internal/resolve/resolve_test.go new file mode 100644 index 0000000..1d31e7e --- /dev/null +++ b/internal/resolve/resolve_test.go @@ -0,0 +1,306 @@ +package resolve + +import ( + "context" + "log/slog" + "os" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/projectdiscovery/retryabledns" + + "github.com/JoshuaMart/FastRecon/internal/report" +) + +func discardLogger() *slog.Logger { + return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.Level(99)})) +} + +// fakeDNS answers from a fixed table. Any name not in the table is treated as +// a wildcard hit when its parent has a wildcard entry, which is how a real +// wildcard record behaves. +type fakeDNS struct { + mu sync.Mutex + answers map[string]*retryabledns.DNSData + wildcards map[string]*retryabledns.DNSData + nxdomain map[string]bool + failures map[string]bool + queries int +} + +func (f *fakeDNS) query(host string) (*retryabledns.DNSData, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.queries++ + + if f.failures[host] { + return nil, context.DeadlineExceeded + } + if d, ok := f.answers[host]; ok { + return d, nil + } + for parent, d := range f.wildcards { + if strings.HasSuffix(host, "."+parent) { + return d, nil + } + } + if f.nxdomain[host] { + return &retryabledns.DNSData{Host: host, StatusCode: "NXDOMAIN"}, nil + } + return &retryabledns.DNSData{Host: host, StatusCode: "NOERROR"}, nil +} + +func newResolver(t *testing.T, domain string, f *fakeDNS) *DNSX { + t.Helper() + return &DNSX{ + opts: Options{ + Domain: domain, + Concurrency: 4, + WildcardProbes: 3, + Logger: discardLogger(), + }, + query: f.query, + } +} + +func byHost(hosts []report.Host) map[string]report.Host { + out := make(map[string]report.Host, len(hosts)) + for _, h := range hosts { + out[h.Host] = h + } + return out +} + +func TestResolveClassifiesHosts(t *testing.T) { + f := &fakeDNS{ + answers: map[string]*retryabledns.DNSData{ + "api.example.com": {Host: "api.example.com", A: []string{"93.184.216.34"}, StatusCode: "NOERROR"}, + "v6.example.com": {Host: "v6.example.com", AAAA: []string{"2606:2800::1"}, StatusCode: "NOERROR"}, + // Exists, has an alias, but no address: a dangling CNAME. + "old.example.com": {Host: "old.example.com", CNAME: []string{"bucket.s3.amazonaws.com."}, StatusCode: "NOERROR"}, + }, + nxdomain: map[string]bool{"gone.example.com": true}, + failures: map[string]bool{"slow.example.com": true}, + } + r := newResolver(t, "example.com", f) + + res, err := r.Resolve(context.Background(), []string{ + "api.example.com", "v6.example.com", "old.example.com", "gone.example.com", "slow.example.com", "nodata.example.com", + }) + if err != nil { + t.Fatal(err) + } + + got := byHost(res.Hosts) + if len(res.Hosts) != 6 { + t.Fatalf("hosts = %d, want every input host reported", len(res.Hosts)) + } + if got["api.example.com"].Status != report.StatusLive { + t.Errorf("api = %+v, want live", got["api.example.com"]) + } + // An AAAA-only host is live: v6-only services exist. + if got["v6.example.com"].Status != report.StatusLive { + t.Errorf("v6 = %+v, want live", got["v6.example.com"]) + } + if got["gone.example.com"].Reason != report.ReasonNXDomain { + t.Errorf("gone = %+v, want nxdomain", got["gone.example.com"]) + } + if got["slow.example.com"].Reason != report.ReasonTimeout { + t.Errorf("slow = %+v, want timeout", got["slow.example.com"]) + } + // NOERROR with no records is not NXDOMAIN: the name exists. + if got["nodata.example.com"].Reason != report.ReasonNoAnswer { + t.Errorf("nodata = %+v, want no_answer", got["nodata.example.com"]) + } + + // The dangling alias must survive with its target intact — that is the + // finding, and dropping it would erase it. + dangling := got["old.example.com"] + if dangling.Status != report.StatusDead || dangling.Reason != report.ReasonNoAnswer { + t.Errorf("old = %+v, want dead/no_answer", dangling) + } + if !slices.Equal(dangling.CNAME, []string{"bucket.s3.amazonaws.com"}) { + t.Errorf("cname = %v, want the normalized target kept", dangling.CNAME) + } +} + +func TestWildcardHostsAreNotLive(t *testing.T) { + wildcardIP := "203.0.113.10" + f := &fakeDNS{ + answers: map[string]*retryabledns.DNSData{ + "real.example.com": {Host: "real.example.com", A: []string{"93.184.216.34"}, StatusCode: "NOERROR"}, + }, + wildcards: map[string]*retryabledns.DNSData{ + "example.com": {A: []string{wildcardIP}, StatusCode: "NOERROR"}, + }, + } + r := newResolver(t, "example.com", f) + + res, err := r.Resolve(context.Background(), []string{"real.example.com", "junk.example.com", "more.example.com"}) + if err != nil { + t.Fatal(err) + } + + got := byHost(res.Hosts) + if got["real.example.com"].Status != report.StatusLive { + t.Errorf("real = %+v, want live: it has its own address", got["real.example.com"]) + } + for _, h := range []string{"junk.example.com", "more.example.com"} { + if got[h].Status != report.StatusWildcard { + t.Errorf("%s = %+v, want wildcard", h, got[h]) + } + // The host stays in the report, in its own bucket. + if got[h].Addresses == nil { + t.Errorf("%s lost its answer; a wildcard host is still reported", h) + } + } + if len(res.Warnings) == 0 { + t.Error("a detected wildcard must warn: it changes how the whole result reads") + } +} + +// A wildcard can sit on any label, not just the apex. +func TestWildcardIsDetectedOnIntermediateParents(t *testing.T) { + f := &fakeDNS{ + wildcards: map[string]*retryabledns.DNSData{ + "dev.example.com": {A: []string{"203.0.113.20"}, StatusCode: "NOERROR"}, + }, + answers: map[string]*retryabledns.DNSData{ + "api.example.com": {Host: "api.example.com", A: []string{"93.184.216.34"}, StatusCode: "NOERROR"}, + }, + } + r := newResolver(t, "example.com", f) + + res, err := r.Resolve(context.Background(), []string{"a.dev.example.com", "api.example.com"}) + if err != nil { + t.Fatal(err) + } + got := byHost(res.Hosts) + if got["a.dev.example.com"].Status != report.StatusWildcard { + t.Errorf("a.dev = %+v, want wildcard", got["a.dev.example.com"]) + } + if got["api.example.com"].Status != report.StatusLive { + t.Errorf("api = %+v, want live: the apex carries no wildcard", got["api.example.com"]) + } +} + +// A host that answers with the wildcard address *and* one of its own is a +// real host that happens to share infrastructure. +func TestHostWithAnExtraAddressIsNotAWildcardArtifact(t *testing.T) { + f := &fakeDNS{ + wildcards: map[string]*retryabledns.DNSData{ + "example.com": {A: []string{"203.0.113.10"}, StatusCode: "NOERROR"}, + }, + answers: map[string]*retryabledns.DNSData{ + "api.example.com": {Host: "api.example.com", A: []string{"203.0.113.10", "93.184.216.34"}, StatusCode: "NOERROR"}, + }, + } + r := newResolver(t, "example.com", f) + + res, err := r.Resolve(context.Background(), []string{"api.example.com"}) + if err != nil { + t.Fatal(err) + } + if got := byHost(res.Hosts)["api.example.com"]; got.Status != report.StatusLive { + t.Errorf("api = %+v, want live", got) + } +} + +// One flaky lookup must not condemn a whole branch as a wildcard. +func TestSingleFlakyProbeDoesNotCreateAWildcard(t *testing.T) { + var once sync.Once + f := &fakeDNS{answers: map[string]*retryabledns.DNSData{}} + base := f.query + r := newResolver(t, "example.com", f) + r.query = func(host string) (*retryabledns.DNSData, error) { + if strings.HasPrefix(host, "fr") { + answered := false + once.Do(func() { answered = true }) + if answered { + return &retryabledns.DNSData{A: []string{"203.0.113.99"}, StatusCode: "NOERROR"}, nil + } + return &retryabledns.DNSData{StatusCode: "NXDOMAIN"}, nil + } + return base(host) + } + + res, err := r.Resolve(context.Background(), []string{"a.example.com"}) + if err != nil { + t.Fatal(err) + } + if got := byHost(res.Hosts)["a.example.com"]; got.Status == report.StatusWildcard { + t.Error("a single answering probe out of three must not establish a wildcard") + } + if len(res.Warnings) != 0 { + t.Errorf("warnings = %v, want none", res.Warnings) + } +} + +func TestResolveReportsUnresolvedHostsWhenOutOfTime(t *testing.T) { + f := &fakeDNS{} + r := newResolver(t, "example.com", f) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + res, err := r.Resolve(ctx, []string{"a.example.com", "b.example.com"}) + if err != nil { + t.Fatal(err) + } + if !res.Truncated { + t.Error("a resolution that never ran must report itself truncated") + } + for _, h := range res.Hosts { + if h.Status != report.StatusDiscovered { + t.Errorf("%s = %q, want discovered rather than an invented verdict", h.Host, h.Status) + } + } +} + +func TestParentsOf(t *testing.T) { + cases := map[string][]string{ + "a.b.example.com": {"b.example.com", "example.com"}, + "www.example.com": {"example.com"}, + "example.com": nil, + "com": nil, + } + for host, want := range cases { + got := parentsOf(host) + if !slices.Equal(got, want) { + t.Errorf("parentsOf(%q) = %v, want %v", host, got, want) + } + } +} + +func TestCandidateParentsStaysInScope(t *testing.T) { + got := candidateParents([]string{"a.dev.example.com", "www.example.com", "evil.other.net"}, "example.com") + slices.Sort(got) + want := []string{"dev.example.com", "example.com"} + slices.Sort(want) + if !slices.Equal(got, want) { + t.Errorf("candidateParents = %v, want %v: out-of-scope parents must not be probed", got, want) + } +} + +func TestNewRejectsUnusableOptions(t *testing.T) { + base := Options{Domain: "example.com", Concurrency: 1, Timeout: time.Second, WildcardProbes: 1, Logger: discardLogger()} + for name, mutate := range map[string]func(*Options){ + "no logger": func(o *Options) { o.Logger = nil }, + "no domain": func(o *Options) { o.Domain = "" }, + "no concurrency": func(o *Options) { o.Concurrency = 0 }, + "no timeout": func(o *Options) { o.Timeout = 0 }, + "no probes": func(o *Options) { o.WildcardProbes = 0 }, + } { + opts := base + mutate(&opts) + if _, err := New(opts); err == nil { + t.Errorf("New accepted options with %s", name) + } + } + if _, err := New(base); err != nil { + t.Errorf("New rejected valid options: %v", err) + } +} diff --git a/internal/resolve/resolvers.go b/internal/resolve/resolvers.go new file mode 100644 index 0000000..25a5099 --- /dev/null +++ b/internal/resolve/resolvers.go @@ -0,0 +1,216 @@ +package resolve + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "net/url" + "os" + "strconv" + "strings" + "time" + + "github.com/JoshuaMart/FastRecon/internal/version" +) + +// DefaultResolvers is the bundled set: Cloudflare, Google and Quad9's +// unfiltered endpoints. +// +// All three are non-filtering and answer NXDOMAIN for names that do not +// exist. That matters more here than raw resolver count: a filtering resolver +// returns a block-page address for a name it dislikes, and a resolver that +// redirects NXDOMAIN turns every dead host into a live one — corrupting +// exactly the live/dead split this stage produces. +var DefaultResolvers = []string{ + "1.1.1.1:53", "1.0.0.1:53", // Cloudflare + "8.8.8.8:53", "8.8.4.4:53", // Google + "9.9.9.10:53", "149.112.112.10:53", // Quad9, unfiltered endpoints +} + +// maxResolverListBytes caps a fetched resolver list. The published lists are +// well under a megabyte; anything larger is a wrong URL, not a resolver list. +const maxResolverListBytes = 8 << 20 + +// LoadOptions describes where the resolver list comes from. The sources are +// merged, so an inline resolver can be added to a fetched list. +type LoadOptions struct { + Inline []string + File string + URL string + Timeout time.Duration + Logger *slog.Logger +} + +// LoadResolvers assembles the resolver list from every configured source, +// falling back to the bundled set when none is given. +func LoadResolvers(ctx context.Context, opts LoadOptions) ([]string, error) { + if len(opts.Inline) == 0 && opts.File == "" && opts.URL == "" { + return DefaultResolvers, nil + } + + var ( + raw []string + errs []error + ) + raw = append(raw, opts.Inline...) + + if opts.File != "" { + fromFile, err := readResolverFile(opts.File) + if err != nil { + errs = append(errs, err) + } + raw = append(raw, fromFile...) + opts.Logger.Debug("resolvers loaded", "source", opts.File, "count", len(fromFile)) + } + + if opts.URL != "" { + fromURL, err := fetchResolverList(ctx, opts.URL, opts.Timeout) + if err != nil { + errs = append(errs, err) + } + raw = append(raw, fromURL...) + opts.Logger.Debug("resolvers loaded", "source", opts.URL, "count", len(fromURL)) + } + if err := errors.Join(errs...); err != nil { + return nil, err + } + + resolvers, malformed := parseResolvers(raw) + if len(malformed) > 0 { + // Naming them is the point: a list where half the lines are junk is a + // wrong file, and silently using the good half hides that. + opts.Logger.Warn("resolver entries ignored", + "count", len(malformed), + "sample", malformed[:min(len(malformed), 5)], + ) + } + if len(resolvers) == 0 { + return nil, errors.New("resolve: no usable resolver in the configured sources") + } + return resolvers, nil +} + +func readResolverFile(path string) ([]string, error) { + f, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("read resolvers file: %w", err) + } + defer func() { _ = f.Close() }() + + out, err := scanResolvers(f) + if err != nil { + return nil, fmt.Errorf("read resolvers file %s: %w", path, err) + } + return out, nil +} + +// fetchResolverList downloads a resolver list. This exists for the serverless +// deployments, which have no volume to mount a file from. +func fetchResolverList(ctx context.Context, raw string, timeout time.Duration) ([]string, error) { + u, err := url.Parse(raw) + if err != nil { + return nil, fmt.Errorf("resolvers url %q: %w", raw, err) + } + if u.Scheme != "https" { + // The list decides where every DNS query goes; fetching it over a + // channel anyone can rewrite would hand that decision away. + return nil, fmt.Errorf("resolvers url %q must use https", raw) + } + + if timeout <= 0 { + timeout = 30 * time.Second + } + reqCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, raw, nil) + if err != nil { + return nil, fmt.Errorf("resolvers url: %w", err) + } + req.Header.Set("User-Agent", version.UserAgent()) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("fetch resolvers from %s: %w", raw, err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("fetch resolvers from %s: unexpected status %s", raw, resp.Status) + } + + out, err := scanResolvers(io.LimitReader(resp.Body, maxResolverListBytes)) + if err != nil { + return nil, fmt.Errorf("fetch resolvers from %s: %w", raw, err) + } + return out, nil +} + +func scanResolvers(r io.Reader) ([]string, error) { + var out []string + sc := bufio.NewScanner(r) + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + out = append(out, line) + } + return out, sc.Err() +} + +// parseResolvers normalizes and deduplicates entries, returning the ones it +// could not make sense of. +func parseResolvers(raw []string) (resolvers, malformed []string) { + seen := make(map[string]struct{}, len(raw)) + for _, entry := range raw { + addr, err := parseResolver(entry) + if err != nil { + malformed = append(malformed, entry) + continue + } + if _, dup := seen[addr]; dup { + continue + } + seen[addr] = struct{}{} + resolvers = append(resolvers, addr) + } + return resolvers, malformed +} + +// parseResolver accepts an IP with or without a port and returns host:port. +// +// Only literal addresses are accepted: a resolver given as a hostname would +// have to be resolved by some other resolver first, which is a dependency +// this stage should not have. +func parseResolver(raw string) (string, error) { + s := strings.TrimSpace(raw) + if s == "" { + return "", errors.New("empty resolver") + } + // Strip an inline comment, as published lists sometimes carry one. + if i := strings.IndexAny(s, "#;"); i >= 0 { + s = strings.TrimSpace(s[:i]) + } + + if ip := net.ParseIP(s); ip != nil { + return net.JoinHostPort(s, "53"), nil + } + + host, port, err := net.SplitHostPort(s) + if err != nil { + return "", fmt.Errorf("resolver %q is not an IP address", raw) + } + if net.ParseIP(host) == nil { + return "", fmt.Errorf("resolver %q is not an IP address", raw) + } + if n, err := strconv.Atoi(port); err != nil || n < 1 || n > 65535 { + return "", fmt.Errorf("resolver %q has an invalid port", raw) + } + return net.JoinHostPort(host, port), nil +} diff --git a/internal/resolve/resolvers_test.go b/internal/resolve/resolvers_test.go new file mode 100644 index 0000000..16fc9a2 --- /dev/null +++ b/internal/resolve/resolvers_test.go @@ -0,0 +1,126 @@ +package resolve + +import ( + "context" + "os" + "path/filepath" + "slices" + "strings" + "testing" +) + +func TestParseResolver(t *testing.T) { + ok := map[string]string{ + "1.1.1.1": "1.1.1.1:53", + " 8.8.8.8 ": "8.8.8.8:53", + "9.9.9.10:5353": "9.9.9.10:5353", + "2606:4700:4700::1111": "[2606:4700:4700::1111]:53", + "[2606:4700:4700::1111]:53": "[2606:4700:4700::1111]:53", + "1.1.1.1 # cloudflare": "1.1.1.1:53", + "8.8.4.4 ; google secondary": "8.8.4.4:53", + } + for in, want := range ok { + got, err := parseResolver(in) + if err != nil { + t.Errorf("parseResolver(%q): %v", in, err) + continue + } + if got != want { + t.Errorf("parseResolver(%q) = %q, want %q", in, got, want) + } + } + + // A hostname would have to be resolved by some other resolver first, + // which is a dependency this stage must not have. + for _, in := range []string{"", "dns.google", "1.1.1.1:0", "1.1.1.1:99999", "not-an-ip", "1.1.1.1:abc"} { + if got, err := parseResolver(in); err == nil { + t.Errorf("parseResolver(%q) = %q, want an error", in, got) + } + } +} + +func TestParseResolversDeduplicatesAndReportsJunk(t *testing.T) { + resolvers, malformed := parseResolvers([]string{"1.1.1.1", "1.1.1.1:53", "8.8.8.8", "garbage", "dns.google"}) + if !slices.Equal(resolvers, []string{"1.1.1.1:53", "8.8.8.8:53"}) { + t.Errorf("resolvers = %v, want the deduplicated pair", resolvers) + } + if !slices.Equal(malformed, []string{"garbage", "dns.google"}) { + t.Errorf("malformed = %v, want both junk entries named", malformed) + } +} + +func TestLoadResolversFallsBackToTheBundledSet(t *testing.T) { + got, err := LoadResolvers(context.Background(), LoadOptions{Logger: discardLogger()}) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(got, DefaultResolvers) { + t.Errorf("resolvers = %v, want the bundled set", got) + } +} + +func TestLoadResolversMergesInlineAndFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "resolvers.txt") + body := "# public resolvers\n\n8.8.8.8\n9.9.9.10:53\n8.8.8.8\n" + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + + got, err := LoadResolvers(context.Background(), LoadOptions{ + Inline: []string{"1.1.1.1"}, + File: path, + Logger: discardLogger(), + }) + if err != nil { + t.Fatal(err) + } + want := []string{"1.1.1.1:53", "8.8.8.8:53", "9.9.9.10:53"} + if !slices.Equal(got, want) { + t.Errorf("resolvers = %v, want %v", got, want) + } +} + +func TestLoadResolversRejectsAFileOfJunk(t *testing.T) { + path := filepath.Join(t.TempDir(), "resolvers.txt") + if err := os.WriteFile(path, []byte("not-an-ip\nalso-not\n"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := LoadResolvers(context.Background(), LoadOptions{File: path, Logger: discardLogger()}); err == nil { + t.Error("a resolver file with nothing usable in it must fail, not fall back silently") + } +} + +// The list decides where every DNS query goes; fetching it over a channel +// anyone can rewrite would hand that decision away. +func TestLoadResolversRefusesPlainHTTP(t *testing.T) { + _, err := LoadResolvers(context.Background(), LoadOptions{ + URL: "http://example.com/resolvers.txt", + Logger: discardLogger(), + }) + if err == nil || !strings.Contains(err.Error(), "https") { + t.Errorf("err = %v, want a refusal naming https", err) + } +} + +func TestLoadResolversReportsAnUnreadableFile(t *testing.T) { + _, err := LoadResolvers(context.Background(), LoadOptions{ + File: filepath.Join(t.TempDir(), "missing.txt"), + Logger: discardLogger(), + }) + if err == nil { + t.Error("an unreadable resolver file must fail loudly") + } +} + +func TestDefaultResolversAreWellFormed(t *testing.T) { + for _, r := range DefaultResolvers { + got, err := parseResolver(r) + if err != nil { + t.Errorf("bundled resolver %q is malformed: %v", r, err) + continue + } + if got != r { + t.Errorf("bundled resolver %q should already be normalized as %q", r, got) + } + } +} diff --git a/internal/resolve/wildcard.go b/internal/resolve/wildcard.go new file mode 100644 index 0000000..84e480e --- /dev/null +++ b/internal/resolve/wildcard.go @@ -0,0 +1,199 @@ +package resolve + +import ( + "context" + "crypto/rand" + "encoding/hex" + "strings" + "sync" +) + +// wildcardSet is the set of answers a wildcard record hands out for one parent +// domain. +type wildcardSet struct { + addresses map[string]struct{} + cnames map[string]struct{} +} + +func (w *wildcardSet) covers(addresses, cnames []string) bool { + // A host with no answers is not a wildcard artifact; it is simply dead. + if len(addresses) == 0 && len(cnames) == 0 { + return false + } + for _, c := range cnames { + if _, ok := w.cnames[normalizeName(c)]; ok { + return true + } + } + if len(addresses) == 0 { + return false + } + // Every address must belong to the wildcard set. A host that resolves to + // the wildcard address *and* one of its own is a real host. + for _, a := range addresses { + if _, ok := w.addresses[a]; !ok { + return false + } + } + return true +} + +// wildcards maps a parent domain to the answers its wildcard record returns. +type wildcards struct { + byParent map[string]*wildcardSet +} + +// covers reports whether a host's answers are indistinguishable from the +// wildcard of one of its parents. +func (w *wildcards) covers(host string, addresses, cnames []string) (string, bool) { + if len(w.byParent) == 0 { + return "", false + } + for _, parent := range parentsOf(host) { + set, ok := w.byParent[parent] + if !ok { + continue + } + if set.covers(addresses, cnames) { + return parent, true + } + } + return "", false +} + +// detectWildcards probes random names under every parent domain that appears +// in the host list. +// +// Detection is per parent, not only at the root: a wildcard on +// *.dev.example.com is just as capable of flooding the live set as one on the +// apex, and only the parent it sits on can reveal it. +func (r *DNSX) detectWildcards(ctx context.Context, hosts []string) *wildcards { + parents := candidateParents(hosts, r.opts.Domain) + out := &wildcards{byParent: make(map[string]*wildcardSet)} + if len(parents) == 0 { + return out + } + + var ( + mu sync.Mutex + wg sync.WaitGroup + ) + sem := make(chan struct{}, r.opts.Concurrency) + + for _, parent := range parents { + wg.Add(1) + go func(parent string) { + defer wg.Done() + select { + case sem <- struct{}{}: + defer func() { <-sem }() + case <-ctx.Done(): + return + } + if set := r.probeWildcard(ctx, parent); set != nil { + mu.Lock() + out.byParent[parent] = set + mu.Unlock() + } + }(parent) + } + wg.Wait() + + if len(out.byParent) > 0 { + names := make([]string, 0, len(out.byParent)) + for p := range out.byParent { + names = append(names, p) + } + r.opts.Logger.Info("wildcard dns detected", "parents", names) + } + r.opts.Logger.Debug("wildcard probing finished", "parents_probed", len(parents), "wildcards_found", len(out.byParent)) + return out +} + +// probeWildcard resolves random names under parent. It returns the answers +// only when a majority of the probes agree, so one flaky lookup cannot mark a +// whole branch as a wildcard. +func (r *DNSX) probeWildcard(ctx context.Context, parent string) *wildcardSet { + set := &wildcardSet{addresses: map[string]struct{}{}, cnames: map[string]struct{}{}} + answered := 0 + + for range r.opts.WildcardProbes { + if ctx.Err() != nil { + return nil + } + data, err := r.query(randomLabel() + "." + parent) + if err != nil || data == nil { + continue + } + addresses, cnames := answersOf(data) + if len(addresses) == 0 && len(cnames) == 0 { + continue + } + answered++ + for _, a := range addresses { + set.addresses[a] = struct{}{} + } + for _, c := range cnames { + set.cnames[normalizeName(c)] = struct{}{} + } + } + + if answered*2 <= r.opts.WildcardProbes { + return nil + } + return set +} + +// candidateParents lists every domain that could carry a wildcard record for +// the given hosts: each host's ancestors, down to and including the root. +func candidateParents(hosts []string, root string) []string { + seen := map[string]struct{}{} + var out []string + add := func(d string) { + if d == "" { + return + } + if _, ok := seen[d]; ok { + return + } + seen[d] = struct{}{} + out = append(out, d) + } + + add(root) + for _, h := range hosts { + for _, p := range parentsOf(h) { + if p == root || strings.HasSuffix(p, "."+root) { + add(p) + } + } + } + return out +} + +// parentsOf returns a host's ancestors, closest first: for a.b.example.com, +// b.example.com then example.com. It stops before a bare TLD, which cannot +// carry a wildcard record anyone here cares about. +func parentsOf(host string) []string { + var out []string + rest := host + for { + _, after, found := strings.Cut(rest, ".") + if !found || !strings.Contains(after, ".") { + return out + } + out = append(out, after) + rest = after + } +} + +func randomLabel() string { + var b [8]byte + // crypto/rand.Read never fails; it panics on a broken source. + _, _ = rand.Read(b[:]) + return "fr" + hex.EncodeToString(b[:]) +} + +func normalizeName(s string) string { + return strings.ToLower(strings.TrimSuffix(strings.TrimSpace(s), ".")) +} diff --git a/internal/runid/runid.go b/internal/runid/runid.go new file mode 100644 index 0000000..3a7f1c7 --- /dev/null +++ b/internal/runid/runid.go @@ -0,0 +1,44 @@ +// Package runid generates ULID-style identifiers for a run: a millisecond +// timestamp followed by randomness, so ids sort chronologically as strings. +package runid + +import ( + "crypto/rand" + "time" +) + +const crockford = "0123456789ABCDEFGHJKMNPQRSTVWXYZ" + +// New builds a 26-character identifier from t plus 80 bits of randomness. +func New(t time.Time) string { + var buf [16]byte + ms := uint64(t.UTC().UnixMilli()) + for i := 0; i < 6; i++ { + buf[5-i] = byte(ms >> (8 * i)) + } + // crypto/rand.Read never returns an error; it panics on a broken source. + _, _ = rand.Read(buf[6:]) + return encode(buf[:]) +} + +// encode renders 128 bits as 26 base32 characters, left-padded to 130 bits. +func encode(b []byte) string { + out := make([]byte, 26) + for i := range out { + var v uint + for j := 0; j < 5; j++ { + v = v<<1 | bitAt(b, i*5+j) + } + out[i] = crockford[v] + } + return string(out) +} + +// bitAt reads bit p of the padded stream: two zero bits, then b. +func bitAt(b []byte, p int) uint { + if p < 2 { + return 0 + } + q := p - 2 + return uint(b[q/8]>>(7-uint(q%8))) & 1 +} diff --git a/internal/runid/runid_test.go b/internal/runid/runid_test.go new file mode 100644 index 0000000..c1fb667 --- /dev/null +++ b/internal/runid/runid_test.go @@ -0,0 +1,40 @@ +package runid + +import ( + "strings" + "testing" + "time" +) + +func TestNewIsChronologicallySortable(t *testing.T) { + base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + earlier := New(base) + later := New(base.Add(time.Second)) + if earlier >= later { + t.Errorf("ids must sort by time: %q >= %q", earlier, later) + } +} + +func TestNewShapeAndAlphabet(t *testing.T) { + id := New(time.Now()) + if len(id) != 26 { + t.Fatalf("len(%q) = %d, want 26", id, len(id)) + } + for _, r := range id { + if !strings.ContainsRune(crockford, r) { + t.Errorf("id %q contains %q, outside the Crockford alphabet", id, r) + } + } +} + +func TestNewIsUnique(t *testing.T) { + now := time.Now() + seen := make(map[string]bool, 1000) + for range 1000 { + id := New(now) + if seen[id] { + t.Fatalf("duplicate id %q generated within the same millisecond", id) + } + seen[id] = true + } +} diff --git a/internal/secrets/redact.go b/internal/secrets/redact.go new file mode 100644 index 0000000..d8666b1 --- /dev/null +++ b/internal/secrets/redact.go @@ -0,0 +1,67 @@ +package secrets + +import ( + "regexp" + "strings" +) + +// Placeholder replaces a credential wherever one would otherwise be printed. +const Placeholder = "[REDACTED]" + +// queryKeyRE matches a credential carried in a URL query string. Some sources +// authenticate that way — c99 puts the key in the URL — so an error message +// quoting the request URL would otherwise leak it verbatim into the report. +var queryKeyRE = regexp.MustCompile(`(?i)([?&](?:key|api_?key|token|access_?key|apitoken)=)[^&\s"']+`) + +// Redactor scrubs known credential values, and anything shaped like one, out +// of text on its way to a log or a report. +type Redactor struct { + values []string +} + +// NewRedactor builds a redactor for the resolved credentials. +func NewRedactor(creds map[string]Credential) *Redactor { + r := &Redactor{} + for _, c := range creds { + // Very short values would match everywhere and mangle the text. + if len(c.Value) >= 8 { + r.values = append(r.values, c.Value) + } + } + return r +} + +// Redact returns s with every known credential replaced. +// +// The query-string pass runs whether or not any credential value is known: a +// run whose provider config failed to load has no values to match, and is +// exactly the run whose source errors are most likely to quote a URL. +func (r *Redactor) Redact(s string) string { + if s == "" { + return s + } + for _, v := range r.values { + s = strings.ReplaceAll(s, v, Placeholder) + } + return queryKeyRE.ReplaceAllString(s, "${1}"+Placeholder) +} + +// RedactBytes scrubs a rendered document. It is the last line of defence +// before the report leaves the process: everything that reaches the report is +// meant to be redacted at the point it is created, and this catches whatever +// was not. +func (r *Redactor) RedactBytes(data []byte) []byte { + if len(data) == 0 { + return data + } + out := r.Redact(string(data)) + return []byte(out) +} + +// RedactError renders an error through the redactor, tolerating a nil error. +func (r *Redactor) RedactError(err error) string { + if err == nil { + return "" + } + return r.Redact(err.Error()) +} diff --git a/internal/secrets/secrets.go b/internal/secrets/secrets.go new file mode 100644 index 0000000..dc3d6c0 --- /dev/null +++ b/internal/secrets/secrets.go @@ -0,0 +1,131 @@ +// Package secrets resolves source API keys and keeps them out of the output. +// +// The container image never contains a credential — it is built by CI and +// published — so every key arrives at runtime, through one of the channels +// below. +package secrets + +import ( + "fmt" + "os" + "strings" + + "gopkg.in/yaml.v3" +) + +// EnvPrefix is the namespaced spelling of a source key: +// FASTRECON_KEY_CHAOS. The upstream spelling (CHAOS_API_KEY) is accepted too. +const EnvPrefix = "FASTRECON_KEY_" + +// Credential is a resolved key and where it came from. The value is never +// logged or serialized; Origin is what gets reported. +type Credential struct { + Source string + Value string + Origin string +} + +// Resolver resolves credentials for a set of sources. +type Resolver struct { + providerConfig map[string][]string + configPath string +} + +// NewResolver loads the optional provider config file. Its format is the +// subfinder/subfaster one — a source name mapped to a list of keys — so an +// existing file works unchanged. +func NewResolver(providerConfigPath string) (*Resolver, error) { + r := &Resolver{configPath: providerConfigPath} + if providerConfigPath == "" { + return r, nil + } + data, err := os.ReadFile(providerConfigPath) + if err != nil { + return nil, fmt.Errorf("read provider config: %w", err) + } + if err := yaml.Unmarshal(data, &r.providerConfig); err != nil { + return nil, fmt.Errorf("parse provider config %s: %w", providerConfigPath, err) + } + return r, nil +} + +// Resolve returns the credential for each source that has one. +// +// Precedence, highest first: +// +// FASTRECON_KEY_ namespaced environment variable +// _API_KEY upstream environment variable +// provider config file the mounted YAML +// FASTRECON_KEY__FILE a file holding the key, for mounted secrets +func (r *Resolver) Resolve(sources []string) map[string]Credential { + out := make(map[string]Credential, len(sources)) + for _, source := range sources { + if c, ok := r.resolveOne(source); ok { + out[source] = c + } + } + return out +} + +func (r *Resolver) resolveOne(source string) (Credential, bool) { + upper := strings.ToUpper(source) + + if v, ok := nonEmptyEnv(EnvPrefix + upper); ok { + return Credential{source, v, "env:" + EnvPrefix + upper}, true + } + if v, ok := nonEmptyEnv(upper + "_API_KEY"); ok { + return Credential{source, v, "env:" + upper + "_API_KEY"}, true + } + if keys := r.providerConfig[source]; len(keys) > 0 && strings.TrimSpace(keys[0]) != "" { + return Credential{source, strings.TrimSpace(keys[0]), "provider-config:" + r.configPath}, true + } + if path, ok := nonEmptyEnv(EnvPrefix + upper + "_FILE"); ok { + data, err := os.ReadFile(path) + if err == nil { + if v := strings.TrimSpace(string(data)); v != "" { + return Credential{source, v, "file:" + path}, true + } + } + } + return Credential{}, false +} + +func nonEmptyEnv(name string) (string, bool) { + v := strings.TrimSpace(os.Getenv(name)) + return v, v != "" +} + +// Export publishes the resolved keys under the upstream environment names, +// which is the only channel the enumeration engine reads. +// +// This mutates the process environment, so it is process-global: a build that +// runs several enumerations concurrently in one process must configure the +// keys once, up front, not per run. +func Export(creds map[string]Credential) error { + for source, c := range creds { + if err := os.Setenv(strings.ToUpper(source)+"_API_KEY", c.Value); err != nil { + return fmt.Errorf("export key for %s: %w", source, err) + } + } + return nil +} + +// Inventory reports which sources have a key and where it came from. It never +// includes a value, not even truncated. +type Inventory struct { + Configured map[string]string // source -> origin + Missing []string +} + +// Take builds the inventory for a set of sources. +func Take(sources []string, creds map[string]Credential) Inventory { + inv := Inventory{Configured: make(map[string]string, len(creds))} + for _, s := range sources { + if c, ok := creds[s]; ok { + inv.Configured[s] = c.Origin + continue + } + inv.Missing = append(inv.Missing, s) + } + return inv +} diff --git a/internal/secrets/secrets_test.go b/internal/secrets/secrets_test.go new file mode 100644 index 0000000..ddabcce --- /dev/null +++ b/internal/secrets/secrets_test.go @@ -0,0 +1,171 @@ +package secrets + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func writeFile(t *testing.T, name, body string) string { + t.Helper() + path := filepath.Join(t.TempDir(), name) + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + return path +} + +func TestResolvePrecedence(t *testing.T) { + cfg := writeFile(t, "provider-config.yaml", "chaos:\n - from-file\n") + keyFile := writeFile(t, "chaos.key", "from-secret-file\n") + + t.Setenv("FASTRECON_KEY_CHAOS_FILE", keyFile) + r, err := NewResolver(cfg) + if err != nil { + t.Fatal(err) + } + + // Lowest: the per-key secret file. + if got := r.Resolve([]string{"chaos"})["chaos"]; got.Value != "from-file" { + t.Errorf("value = %q, want the provider config to beat the secret file", got.Value) + } + + // The upstream environment variable beats the provider config. + t.Setenv("CHAOS_API_KEY", "from-upstream-env") + if got := r.Resolve([]string{"chaos"})["chaos"]; got.Value != "from-upstream-env" { + t.Errorf("value = %q, want the upstream environment variable", got.Value) + } + + // The namespaced form wins over everything. + t.Setenv("FASTRECON_KEY_CHAOS", "from-namespaced-env") + got := r.Resolve([]string{"chaos"})["chaos"] + if got.Value != "from-namespaced-env" { + t.Errorf("value = %q, want the namespaced environment variable", got.Value) + } + if !strings.Contains(got.Origin, "FASTRECON_KEY_CHAOS") { + t.Errorf("origin = %q, want it to name the winning channel", got.Origin) + } +} + +func TestSecretFileIsUsedWhenNothingElseIsSet(t *testing.T) { + keyFile := writeFile(t, "c99.key", " file-key \n") + t.Setenv("FASTRECON_KEY_C99_FILE", keyFile) + + r, err := NewResolver("") + if err != nil { + t.Fatal(err) + } + got := r.Resolve([]string{"c99"})["c99"] + if got.Value != "file-key" { + t.Errorf("value = %q, want the trimmed file contents", got.Value) + } +} + +// An empty variable is an absence, not a credential: a deployment that unsets +// a key must not end up sending an empty one. +func TestEmptyEnvValueIsNotACredential(t *testing.T) { + t.Setenv("CHAOS_API_KEY", " ") + r, err := NewResolver("") + if err != nil { + t.Fatal(err) + } + if _, ok := r.Resolve([]string{"chaos"})["chaos"]; ok { + t.Error("an empty environment value was treated as a credential") + } +} + +func TestUnreadableProviderConfigIsAnError(t *testing.T) { + if _, err := NewResolver(filepath.Join(t.TempDir(), "missing.yaml")); err == nil { + t.Error("a provider config that cannot be read must fail loudly") + } +} + +func TestTakeReportsMissingSourcesWithoutValues(t *testing.T) { + creds := map[string]Credential{"crt": {Source: "crt", Value: "secret-value", Origin: "env:CRT_API_KEY"}} + inv := Take([]string{"crt", "chaos"}, creds) + + if inv.Configured["crt"] != "env:CRT_API_KEY" { + t.Errorf("configured = %v, want the origin", inv.Configured) + } + for _, origin := range inv.Configured { + if strings.Contains(origin, "secret-value") { + t.Error("the inventory must never carry a credential value") + } + } + if len(inv.Missing) != 1 || inv.Missing[0] != "chaos" { + t.Errorf("missing = %v, want [chaos]", inv.Missing) + } +} + +func TestRedactorScrubsKnownValues(t *testing.T) { + r := NewRedactor(map[string]Credential{"chaos": {Value: "abcdef1234567890"}}) + got := r.Redact("request failed with key abcdef1234567890 attached") + if strings.Contains(got, "abcdef1234567890") { + t.Errorf("redacted = %q, the value survived", got) + } + if !strings.Contains(got, Placeholder) { + t.Errorf("redacted = %q, want the placeholder", got) + } +} + +// c99 authenticates through the query string, so an error quoting the request +// URL leaks the key verbatim unless the URL itself is scrubbed. +func TestRedactorScrubsCredentialsInQueryStrings(t *testing.T) { + r := NewRedactor(nil) + for _, in := range []string{ + "https://api.c99.nl/subdomainfinder?key=deadbeefcafe&domain=example.com", + "GET https://x.example/v1?api_key=sekrit-token-value returned 500", + "https://x.example/v1?foo=1&token=abc123&bar=2", + } { + got := r.Redact(in) + for _, leaked := range []string{"deadbeefcafe", "sekrit-token-value", "abc123"} { + if strings.Contains(got, leaked) { + t.Errorf("Redact(%q) = %q, leaked %q", in, got, leaked) + } + } + if !strings.Contains(got, Placeholder) { + t.Errorf("Redact(%q) = %q, want the placeholder", in, got) + } + } + // The rest of the URL must survive, or the message stops being useful. + got := r.Redact("https://api.c99.nl/subdomainfinder?key=deadbeefcafe&domain=example.com") + if !strings.Contains(got, "domain=example.com") { + t.Errorf("Redact stripped too much: %q", got) + } +} + +// Short values would match everywhere and mangle unrelated text. +func TestRedactorIgnoresVeryShortValues(t *testing.T) { + r := NewRedactor(map[string]Credential{"x": {Value: "ab"}}) + if got := r.Redact("a stable build"); got != "a stable build" { + t.Errorf("Redact mangled unrelated text: %q", got) + } +} + +func TestRedactErrorToleratesNil(t *testing.T) { + if got := NewRedactor(nil).RedactError(nil); got != "" { + t.Errorf("RedactError(nil) = %q, want empty", got) + } +} + +// The last line of defence before the report leaves the process. +func TestRedactBytesScrubsARenderedReport(t *testing.T) { + r := NewRedactor(map[string]Credential{"c99": {Value: "abcdef1234567890"}}) + in := []byte(`{"sources":[{"name":"c99","error":"GET https://api.c99.nl/x?key=abcdef1234567890 failed"}]}`) + + got := string(r.RedactBytes(in)) + if strings.Contains(got, "abcdef1234567890") { + t.Errorf("redacted = %q, the credential survived", got) + } + if !strings.Contains(got, `"name":"c99"`) { + t.Errorf("redacted = %q, the surrounding document was damaged", got) + } +} + +func TestRedactBytesIsAPassthroughWithoutCredentials(t *testing.T) { + in := []byte(`{"a":1}`) + if got := NewRedactor(nil).RedactBytes(in); string(got) != string(in) { + t.Errorf("RedactBytes = %q, want the input unchanged", got) + } +} diff --git a/internal/serve/serve.go b/internal/serve/serve.go new file mode 100644 index 0000000..32e2ba5 --- /dev/null +++ b/internal/serve/serve.go @@ -0,0 +1,279 @@ +// Package serve exposes the pipeline over HTTP, for the serverless function +// deployment. +// +// The handler runs synchronously and answers with the report. On most FaaS +// platforms the instance is frozen or reclaimed once the handler returns, so +// "accept, answer 202, finish in the background" loses runs intermittently: +// work that cannot fit in a function's timeout belongs in a job. +package serve + +import ( + "context" + "crypto/subtle" + "encoding/json" + "errors" + "fmt" + "log/slog" + "net/http" + "strings" + "sync" + "time" + + "github.com/JoshuaMart/FastRecon/internal/app" + "github.com/JoshuaMart/FastRecon/internal/config" + "github.com/JoshuaMart/FastRecon/internal/report" + "github.com/JoshuaMart/FastRecon/internal/secrets" + "github.com/JoshuaMart/FastRecon/internal/stage" +) + +const ( + // maxBodyBytes bounds a request body. The document is a handful of fields; + // anything larger is a mistake or an attempt to exhaust the instance. + maxBodyBytes = 64 << 10 + // busyRetryAfter is what a caller is told to wait when a run holds the + // instance. + busyRetryAfter = 30 * time.Second + // shutdownGrace bounds the drain of an in-flight run on SIGTERM. + shutdownGrace = 30 * time.Second +) + +// Runner is what the handler drives. It is an interface so the HTTP contract +// — auth, refusal, deadlines, error mapping — can be tested without a +// network. +type Runner interface { + Run(ctx context.Context, cfg *config.Config) (*report.Report, error) + Redactor() *secrets.Redactor + Config() *config.Config +} + +// Server answers run requests from one process. +type Server struct { + app Runner + token string + log *slog.Logger + + // running serializes runs. The enumeration engine keeps per-source state + // on globally shared instances — API keys and the per-run counters — so + // two runs in one process would overwrite each other's keys and report + // each other's statistics. + running sync.Mutex +} + +// New builds the server. +func New(a Runner, cfg *config.Config, log *slog.Logger) (*Server, error) { + if a == nil || cfg == nil || log == nil { + return nil, errors.New("serve: app, configuration and logger are required") + } + token := strings.TrimSpace(cfg.APIToken) + if token == "" { + // An open subdomain-enumeration endpoint is free reconnaissance for + // whoever finds it, charged to this deployment's API quotas. + return nil, errors.New("serve: --api-token is required; refusing to expose an unauthenticated endpoint") + } + return &Server{app: a, token: token, log: log}, nil +} + +// Handler returns the routes. +func (s *Server) Handler() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("POST /run", s.handleRun) + mux.HandleFunc("GET /healthz", s.handleHealth) + return mux +} + +// ListenAndServe runs until ctx ends, then drains the in-flight run. +func (s *Server) ListenAndServe(ctx context.Context, addr string) error { + srv := &http.Server{ + Addr: addr, + Handler: s.Handler(), + // A slow client must not hold an instance open indefinitely. + ReadHeaderTimeout: 10 * time.Second, + IdleTimeout: 60 * time.Second, + } + + errs := make(chan error, 1) + go func() { + s.log.Info("listening", "addr", addr) + if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + errs <- err + return + } + errs <- nil + }() + + select { + case err := <-errs: + return err + case <-ctx.Done(): + s.log.Info("shutting down", "grace", shutdownGrace.String()) + shutdownCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), shutdownGrace) + defer cancel() + return srv.Shutdown(shutdownCtx) + } +} + +// runRequest is what a caller may specify. +// +// It says what to scan, never how the deployment is wired. Credentials, the +// source selection and the webhook destination come from the function's +// environment: the engine holds credentials in process-global state, and a +// caller-supplied destination would make the function a request-forwarding +// gadget onto its own network. +type runRequest struct { + Domain string `json:"domain"` + Exclude []string `json:"exclude,omitempty"` + Stages string `json:"stages,omitempty"` + Ports string `json:"ports,omitempty"` + Timeout string `json:"timeout,omitempty"` +} + +// errorResponse is what a caller gets instead of a report. +type errorResponse struct { + Error string `json:"error"` +} + +func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) +} + +func (s *Server) handleRun(w http.ResponseWriter, r *http.Request) { + if !s.authorized(r) { + s.log.Warn("request rejected", "reason", "bad token", "remote", r.RemoteAddr) + writeJSON(w, http.StatusUnauthorized, errorResponse{Error: "unauthorized"}) + return + } + + req, err := decodeRequest(r) + if err != nil { + writeJSON(w, http.StatusBadRequest, errorResponse{Error: err.Error()}) + return + } + + runCfg, err := s.runConfig(req) + if err != nil { + writeJSON(w, http.StatusBadRequest, errorResponse{Error: err.Error()}) + return + } + + // Refused rather than queued: a queued request spends its own deadline + // waiting and then reports a timeout that explains nothing. + if !s.running.TryLock() { + w.Header().Set("Retry-After", fmt.Sprintf("%d", int(busyRetryAfter.Seconds()))) + writeJSON(w, http.StatusTooManyRequests, errorResponse{Error: "a run is already in progress on this instance"}) + return + } + defer s.running.Unlock() + + ctx, cancel := context.WithTimeout(r.Context(), budget(runCfg)) + defer cancel() + + rep, err := s.app.Run(ctx, runCfg) + if err != nil { + // The same split the exit codes make: a transient failure is ours, a + // configuration the stages reject is the caller's. Not every option + // can be checked before a stage is built — a port expression is + // parsed by the scanner — so this is where those surface. + if errors.Is(err, app.ErrRuntime) { + s.log.Error("run failed", "error", err) + writeJSON(w, http.StatusInternalServerError, errorResponse{Error: err.Error()}) + return + } + s.log.Warn("request rejected", "reason", "stage setup", "error", err) + writeJSON(w, http.StatusBadRequest, errorResponse{Error: err.Error()}) + return + } + + data, err := rep.Render(report.FormatJSON) + if err != nil { + s.log.Error("render report", "error", err) + writeJSON(w, http.StatusInternalServerError, errorResponse{Error: "could not render the report"}) + return + } + // The same last line of defence the one-shot path applies. + data = s.app.Redactor().RedactBytes(data) + + // A partial report is data, not an error: it says so itself, in + // completed and truncated_by_timeout. + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + if _, err := w.Write(append(data, '\n')); err != nil { + s.log.Warn("response not delivered", "error", err) + } +} + +// authorized compares the bearer token in constant time. +func (s *Server) authorized(r *http.Request) bool { + header := r.Header.Get("Authorization") + token, ok := strings.CutPrefix(header, "Bearer ") + if !ok { + return false + } + return subtle.ConstantTimeCompare([]byte(strings.TrimSpace(token)), []byte(s.token)) == 1 +} + +func decodeRequest(r *http.Request) (runRequest, error) { + var req runRequest + dec := json.NewDecoder(http.MaxBytesReader(nil, r.Body, maxBodyBytes)) + // An unknown field is a caller believing they configured something. Most + // of them name an option that is deliberately not caller-settable, and + // silently ignoring it would run a scan they did not ask for. + dec.DisallowUnknownFields() + if err := dec.Decode(&req); err != nil { + return runRequest{}, fmt.Errorf("invalid request body: %w", err) + } + return req, nil +} + +// runConfig overlays the request onto the process configuration and validates +// the result with the same rules the command line uses. +func (s *Server) runConfig(req runRequest) (*config.Config, error) { + cfg := s.app.Config().Clone() + + cfg.Domain = req.Domain + if len(req.Exclude) > 0 { + cfg.Exclude = req.Exclude + } + if req.Ports != "" { + cfg.Ports = req.Ports + } + if req.Stages != "" { + scope, err := stage.ParseScope(req.Stages) + if err != nil { + return nil, err + } + cfg.Scope = scope + } + if req.Timeout != "" { + d, err := time.ParseDuration(req.Timeout) + if err != nil { + return nil, fmt.Errorf("invalid timeout %q: %w", req.Timeout, err) + } + cfg.Timeout = d + } + + // The exclusion file was merged at startup; a request cannot name one. + cfg.ExcludeFile = "" + // The report is the response. Sinks belong to the deployment. + cfg.Output = config.StdoutPath + + if err := cfg.Validate(); err != nil { + return nil, err + } + return cfg, nil +} + +// budget reserves the output margin so the caller receives a well-formed +// truncated report rather than a platform-level timeout. +func budget(cfg *config.Config) time.Duration { + reserved := time.Duration(float64(cfg.Timeout) * cfg.OutputMargin) + if d := cfg.Timeout - reserved; d > 0 { + return d + } + return cfg.Timeout +} + +func writeJSON(w http.ResponseWriter, status int, body any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(body) +} diff --git a/internal/serve/serve_test.go b/internal/serve/serve_test.go new file mode 100644 index 0000000..63cb93f --- /dev/null +++ b/internal/serve/serve_test.go @@ -0,0 +1,314 @@ +package serve + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "strings" + "sync" + "testing" + "time" + + "github.com/spf13/pflag" + + "github.com/JoshuaMart/FastRecon/internal/app" + "github.com/JoshuaMart/FastRecon/internal/config" + "github.com/JoshuaMart/FastRecon/internal/report" + "github.com/JoshuaMart/FastRecon/internal/secrets" + "github.com/JoshuaMart/FastRecon/internal/stage" +) + +func discardLogger() *slog.Logger { + return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.Level(99)})) +} + +// fakeRunner records what it was asked to run and answers without a network. +type fakeRunner struct { + cfg *config.Config + + mu sync.Mutex + lastCfg *config.Config + calls int + + block chan struct{} + err error +} + +func (f *fakeRunner) Config() *config.Config { return f.cfg } +func (f *fakeRunner) Redactor() *secrets.Redactor { return secrets.NewRedactor(nil) } +func (f *fakeRunner) Run(ctx context.Context, cfg *config.Config) (*report.Report, error) { + f.mu.Lock() + f.calls++ + f.lastCfg = cfg + f.mu.Unlock() + + if f.block != nil { + select { + case <-f.block: + case <-ctx.Done(): + } + } + if f.err != nil { + return nil, f.err + } + rep := report.New("01TEST", cfg.Domain, cfg.Scope, "test", cfg.Environment, time.Now()) + rep.Finish(time.Now()) + return rep, nil +} + +func newTestServer(t *testing.T, runner *fakeRunner) *Server { + t.Helper() + if runner.cfg == nil { + runner.cfg = baseConfig(t) + } + s, err := New(runner, &config.Config{APIToken: "secret"}, discardLogger()) + if err != nil { + t.Fatalf("New: %v", err) + } + return s +} + +// baseConfig is the real default configuration, built the way the binary +// builds it. Hand-writing a valid Config here would drift the moment an +// option is added, and the test would then be validating a shape nothing +// else uses. +func baseConfig(t *testing.T) *config.Config { + t.Helper() + fs := pflag.NewFlagSet("test", pflag.ContinueOnError) + config.RegisterFlags(fs) + if err := fs.Parse(nil); err != nil { + t.Fatalf("parse defaults: %v", err) + } + cfg, err := config.LoadServe(fs) + if err != nil { + t.Fatalf("load defaults: %v", err) + } + cfg.Environment = config.EnvServerlessFunction + return cfg +} + +func post(t *testing.T, s *Server, token, body string) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(http.MethodPost, "/run", strings.NewReader(body)) + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + rec := httptest.NewRecorder() + s.Handler().ServeHTTP(rec, req) + return rec +} + +// An open subdomain-enumeration endpoint is free reconnaissance for whoever +// finds it, charged to this deployment's quotas. +func TestNewRefusesWithoutAToken(t *testing.T) { + if _, err := New(&fakeRunner{cfg: baseConfig(t)}, &config.Config{}, discardLogger()); err == nil { + t.Fatal("New accepted an empty token") + } + if _, err := New(&fakeRunner{cfg: baseConfig(t)}, &config.Config{APIToken: " "}, discardLogger()); err == nil { + t.Error("New accepted a blank token") + } +} + +func TestAuthorization(t *testing.T) { + runner := &fakeRunner{} + s := newTestServer(t, runner) + + for name, token := range map[string]string{ + "no token": "", + "wrong token": "nope", + } { + if got := post(t, s, token, `{"domain":"example.com"}`).Code; got != http.StatusUnauthorized { + t.Errorf("%s: status = %d, want 401", name, got) + } + } + if runner.calls != 0 { + t.Error("an unauthorized request reached the pipeline") + } + + // A malformed header must not be mistaken for a valid one. + req := httptest.NewRequest(http.MethodPost, "/run", strings.NewReader(`{"domain":"example.com"}`)) + req.Header.Set("Authorization", "secret") + rec := httptest.NewRecorder() + s.Handler().ServeHTTP(rec, req) + if rec.Code != http.StatusUnauthorized { + t.Errorf("bare token without the Bearer prefix: status = %d, want 401", rec.Code) + } + + if got := post(t, s, "secret", `{"domain":"example.com"}`).Code; got != http.StatusOK { + t.Errorf("valid token: status = %d, want 200", got) + } +} + +func TestHealthzNeedsNoToken(t *testing.T) { + s := newTestServer(t, &fakeRunner{}) + rec := httptest.NewRecorder() + s.Handler().ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/healthz", nil)) + s.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/healthz", nil)) + if rec.Code != http.StatusOK { + t.Errorf("status = %d, want 200", rec.Code) + } +} + +func TestRequestOverlaysOntoTheProcessConfiguration(t *testing.T) { + runner := &fakeRunner{} + s := newTestServer(t, runner) + + rec := post(t, s, "secret", `{"domain":"Example.COM","stages":"enum","ports":"80,443","exclude":["*.dev.example.com"],"timeout":"90s"}`) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, body %s", rec.Code, rec.Body) + } + + got := runner.lastCfg + if got.Domain != "example.com" { + t.Errorf("domain = %q, want it normalized", got.Domain) + } + if got.Scope != stage.ScopeEnum || got.Ports != "80,443" { + t.Errorf("scope/ports = %v/%q, want the request values", got.Scope, got.Ports) + } + if len(got.Exclude) != 1 || got.Exclude[0] != "*.dev.example.com" { + t.Errorf("exclude = %v, want the request value", got.Exclude) + } + if got.Timeout != 90*time.Second { + t.Errorf("timeout = %s, want 90s", got.Timeout) + } + // Untouched fields keep the deployment's configuration. + if got.ScanRate != runner.cfg.ScanRate { + t.Errorf("scan rate = %d, want the process value %d", got.ScanRate, runner.cfg.ScanRate) + } +} + +// A request must not be able to leave its exclusions behind in the process +// configuration for the next caller. +func TestRequestsDoNotLeakIntoEachOther(t *testing.T) { + runner := &fakeRunner{} + s := newTestServer(t, runner) + + post(t, s, "secret", `{"domain":"example.com","exclude":["admin.example.com"]}`) + post(t, s, "secret", `{"domain":"example.net"}`) + + if len(runner.lastCfg.Exclude) != 0 { + t.Errorf("second request inherited %v from the first", runner.lastCfg.Exclude) + } + if len(runner.cfg.Exclude) != 0 { + t.Errorf("the process configuration was mutated: %v", runner.cfg.Exclude) + } +} + +func TestInvalidRequests(t *testing.T) { + s := newTestServer(t, &fakeRunner{}) + + cases := map[string]string{ + "not json": `{`, + "empty domain": `{}`, + "domain is a url": `{"domain":"https://example.com/x"}`, + "unknown scope": `{"domain":"example.com","stages":"everything"}`, + "bad duration": `{"domain":"example.com","timeout":"soon"}`, + // A caller believing they configured a destination must be told they + // did not, rather than have it silently ignored. + "webhook url": `{"domain":"example.com","webhook_url":"http://elsewhere"}`, + "credentials": `{"domain":"example.com","sources":["chaos"]}`, + } + for name, body := range cases { + rec := post(t, s, "secret", body) + if rec.Code != http.StatusBadRequest { + t.Errorf("%s: status = %d, want 400 (body %s)", name, rec.Code, rec.Body) + } + } +} + +func TestOversizedBodyIsRejected(t *testing.T) { + s := newTestServer(t, &fakeRunner{}) + huge := `{"domain":"example.com","exclude":["` + strings.Repeat("a", maxBodyBytes+1) + `"]}` + if got := post(t, s, "secret", huge).Code; got != http.StatusBadRequest { + t.Errorf("status = %d, want 400", got) + } +} + +// Refused rather than queued: a queued request spends its own deadline +// waiting and then reports a timeout that explains nothing. +func TestSecondConcurrentRunIsRefused(t *testing.T) { + runner := &fakeRunner{block: make(chan struct{})} + s := newTestServer(t, runner) + + started := make(chan struct{}) + go func() { + close(started) + post(t, s, "secret", `{"domain":"example.com"}`) + }() + <-started + // Let the first request take the lock. + for range 100 { + runner.mu.Lock() + n := runner.calls + runner.mu.Unlock() + if n > 0 { + break + } + time.Sleep(time.Millisecond) + } + + rec := post(t, s, "secret", `{"domain":"example.net"}`) + if rec.Code != http.StatusTooManyRequests { + t.Fatalf("status = %d, want 429", rec.Code) + } + if rec.Header().Get("Retry-After") == "" { + t.Error("a refusal must tell the caller when to come back") + } + close(runner.block) +} + +// The same split the exit codes make: a transient failure is ours, a +// configuration the stages reject is the caller's. +func TestRunFailureStatusFollowsTheCause(t *testing.T) { + transient := &fakeRunner{err: fmt.Errorf("%w: every resolver failed the health check", app.ErrRuntime)} + if got := post(t, newTestServer(t, transient), "secret", `{"domain":"example.com"}`).Code; got != http.StatusInternalServerError { + t.Errorf("transient failure: status = %d, want 500", got) + } + + // A port expression is parsed by the scanner, so a typo in it surfaces + // here rather than at request validation. + caller := &fakeRunner{err: errors.New(`ports "http" contains "http", which is not a port number`)} + rec := post(t, newTestServer(t, caller), "secret", `{"domain":"example.com","ports":"http"}`) + if rec.Code != http.StatusBadRequest { + t.Errorf("caller mistake: status = %d, want 400", rec.Code) + } + if !strings.Contains(rec.Body.String(), "not a port number") { + t.Errorf("body = %s, want the offending part named", rec.Body) + } +} + +// A partial report is data, not an error: it says so itself. +func TestReportIsReturnedAsJSON(t *testing.T) { + s := newTestServer(t, &fakeRunner{}) + rec := post(t, s, "secret", `{"domain":"example.com","stages":"enum"}`) + + if ct := rec.Header().Get("Content-Type"); ct != "application/json" { + t.Errorf("content-type = %q", ct) + } + var got report.Report + if err := json.Unmarshal(bytes.TrimSpace(rec.Body.Bytes()), &got); err != nil { + t.Fatalf("response is not a report: %v", err) + } + if got.Run.Domain != "example.com" || got.SchemaVersion != report.SchemaVersion { + t.Errorf("report = %+v, want the run described", got.Run) + } +} + +// The margin is what buys the time to serialize a truncated report instead of +// being cut off by the platform. +func TestBudgetReservesTheOutputMargin(t *testing.T) { + cfg := &config.Config{Timeout: 100 * time.Second, OutputMargin: 0.1} + if got := budget(cfg); got != 90*time.Second { + t.Errorf("budget = %s, want 90s", got) + } + // A margin that would leave nothing must not produce a dead context. + if got := budget(&config.Config{Timeout: time.Second, OutputMargin: 1}); got <= 0 { + t.Errorf("budget = %s, want a positive window", got) + } +} diff --git a/internal/sink/file.go b/internal/sink/file.go new file mode 100644 index 0000000..c234e34 --- /dev/null +++ b/internal/sink/file.go @@ -0,0 +1,78 @@ +package sink + +import ( + "context" + "fmt" + "os" + "path/filepath" +) + +// File writes the report to a path, creating parent directories as needed. +// +// The write is atomic — a temporary file in the destination directory, then a +// rename — so a consumer watching the path never reads a half-written report. +type File struct { + path string +} + +// NewFile builds the file sink. +func NewFile(path string) *File { return &File{path: path} } + +func (f *File) Name() string { return "file" } + +// Path returns the destination, for logging. +func (f *File) Path() string { return f.path } + +func (f *File) Deliver(_ context.Context, data []byte) error { + // A character device or a pipe — /dev/stdout, /dev/null, a fifo — cannot + // be replaced by a rename, and there is nothing to make atomic: the write + // goes straight through. Treating these as regular files fails with a + // permission error that says nothing about the cause. + if info, err := os.Stat(f.path); err == nil && !info.Mode().IsRegular() { + return f.writeDirect(data) + } + + dir := filepath.Dir(f.path) + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("create %s: %w", dir, err) + } + + tmp, err := os.CreateTemp(dir, ".fastrecon-*") + if err != nil { + return fmt.Errorf("create temp file in %s: %w", dir, err) + } + tmpName := tmp.Name() + // No-op once the rename succeeded; on any earlier failure it is the cleanup. + defer func() { _ = os.Remove(tmpName) }() + + if _, err := tmp.Write(append(data, '\n')); err != nil { + _ = tmp.Close() + return fmt.Errorf("write %s: %w", tmpName, err) + } + if err := tmp.Close(); err != nil { + return fmt.Errorf("close %s: %w", tmpName, err) + } + if err := os.Chmod(tmpName, 0o644); err != nil { + return fmt.Errorf("chmod %s: %w", tmpName, err) + } + if err := os.Rename(tmpName, f.path); err != nil { + return fmt.Errorf("rename to %s: %w", f.path, err) + } + return nil +} + +// writeDirect appends to a destination that is not a regular file. +func (f *File) writeDirect(data []byte) error { + file, err := os.OpenFile(f.path, os.O_WRONLY|os.O_APPEND, 0) + if err != nil { + return fmt.Errorf("open %s: %w", f.path, err) + } + if _, err := file.Write(append(data, '\n')); err != nil { + _ = file.Close() + return fmt.Errorf("write %s: %w", f.path, err) + } + if err := file.Close(); err != nil { + return fmt.Errorf("close %s: %w", f.path, err) + } + return nil +} diff --git a/internal/sink/read.go b/internal/sink/read.go new file mode 100644 index 0000000..a6e5984 --- /dev/null +++ b/internal/sink/read.go @@ -0,0 +1,15 @@ +package sink + +import ( + "io" + "net/http" +) + +// maxDrain bounds how much of a webhook response is read before the +// connection is reused. The body is never reported: a webhook target may echo +// the payload, and re-logging it would defeat the redaction applied upstream. +const maxDrain = 4 << 10 + +func readAndDiscard(resp *http.Response) (int64, error) { + return io.Copy(io.Discard, io.LimitReader(resp.Body, maxDrain)) +} diff --git a/internal/sink/sink.go b/internal/sink/sink.go new file mode 100644 index 0000000..c64be83 --- /dev/null +++ b/internal/sink/sink.go @@ -0,0 +1,49 @@ +// Package sink delivers a rendered report to its destinations. +// +// Every sink receives the same bytes: the report is rendered once, so no +// consumer can end up with a different shape than another. +package sink + +import ( + "context" + "errors" + "fmt" +) + +// Sink is one report destination. +type Sink interface { + Name() string + Deliver(ctx context.Context, data []byte) error +} + +// Result is the outcome of one delivery. +type Result struct { + Sink string + Err error +} + +// DeliverAll writes to every sink and returns the failures. One failing sink +// never stops the others: a report that reached stdout is still a delivered +// report even if the webhook was down. +func DeliverAll(ctx context.Context, data []byte, sinks []Sink) []Result { + results := make([]Result, 0, len(sinks)) + for _, s := range sinks { + err := s.Deliver(ctx, data) + if err != nil { + err = fmt.Errorf("%s sink: %w", s.Name(), err) + } + results = append(results, Result{Sink: s.Name(), Err: err}) + } + return results +} + +// Errs collects the delivery failures. +func Errs(results []Result) error { + var errs []error + for _, r := range results { + if r.Err != nil { + errs = append(errs, r.Err) + } + } + return errors.Join(errs...) +} diff --git a/internal/sink/sink_test.go b/internal/sink/sink_test.go new file mode 100644 index 0000000..7b172ea --- /dev/null +++ b/internal/sink/sink_test.go @@ -0,0 +1,100 @@ +package sink + +import ( + "bytes" + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestStdoutWritesOneTrailingNewline(t *testing.T) { + var buf bytes.Buffer + if err := NewWriter(&buf).Deliver(context.Background(), []byte(`{"a":1}`)); err != nil { + t.Fatalf("Deliver: %v", err) + } + if got := buf.String(); got != "{\"a\":1}\n" { + t.Errorf("wrote %q", got) + } +} + +func TestFileCreatesParentDirectories(t *testing.T) { + path := filepath.Join(t.TempDir(), "nested", "deeper", "report.json") + if err := NewFile(path).Deliver(context.Background(), []byte("payload")); err != nil { + t.Fatalf("Deliver: %v", err) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read back: %v", err) + } + if strings.TrimSpace(string(data)) != "payload" { + t.Errorf("content = %q", data) + } +} + +// The write must be atomic: a reader watching the path sees either the old +// report or the new one, never a partial file, and no temp file is left over. +func TestFileWriteIsAtomic(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "report.json") + f := NewFile(path) + if err := f.Deliver(context.Background(), []byte("first")); err != nil { + t.Fatalf("first Deliver: %v", err) + } + if err := f.Deliver(context.Background(), []byte("second")); err != nil { + t.Fatalf("second Deliver: %v", err) + } + + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if strings.TrimSpace(string(data)) != "second" { + t.Errorf("content = %q, want the second write", data) + } + + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + for _, e := range entries { + if strings.HasPrefix(e.Name(), ".fastrecon-") { + t.Errorf("temp file %q left behind", e.Name()) + } + } +} + +type failing struct{} + +func (failing) Name() string { return "failing" } +func (failing) Deliver(context.Context, []byte) error { return errors.New("boom") } + +// One dead destination must not cost the others their report. +func TestDeliverAllContinuesPastAFailure(t *testing.T) { + var buf bytes.Buffer + results := DeliverAll(context.Background(), []byte("payload"), []Sink{failing{}, NewWriter(&buf)}) + + if len(results) != 2 { + t.Fatalf("got %d results, want 2", len(results)) + } + if buf.Len() == 0 { + t.Error("the healthy sink received nothing after the first one failed") + } + err := Errs(results) + if err == nil { + t.Fatal("Errs returned nil despite a failed delivery") + } + if !strings.Contains(err.Error(), "failing sink") { + t.Errorf("error %q does not name the sink that failed", err) + } +} + +// A destination that is not a regular file — /dev/stdout, /dev/null, a fifo — +// cannot be replaced by a rename, and there is nothing to make atomic. +func TestFileWritesDirectlyToCharacterDevices(t *testing.T) { + if err := NewFile("/dev/null").Deliver(context.Background(), []byte("payload")); err != nil { + t.Errorf("writing to /dev/null failed: %v", err) + } +} diff --git a/internal/sink/stdout.go b/internal/sink/stdout.go new file mode 100644 index 0000000..330f57e --- /dev/null +++ b/internal/sink/stdout.go @@ -0,0 +1,29 @@ +package sink + +import ( + "context" + "fmt" + "io" + "os" +) + +// Stdout writes the report to standard output. Logs never go here — that is +// what keeps `fastrecon ... | jq` working and the job's logs readable. +type Stdout struct { + w io.Writer +} + +// NewStdout builds the stdout sink. +func NewStdout() *Stdout { return &Stdout{w: os.Stdout} } + +// NewWriter builds a sink writing to an arbitrary writer, for tests. +func NewWriter(w io.Writer) *Stdout { return &Stdout{w: w} } + +func (s *Stdout) Name() string { return "stdout" } + +func (s *Stdout) Deliver(_ context.Context, data []byte) error { + if _, err := s.w.Write(append(data, '\n')); err != nil { + return fmt.Errorf("write: %w", err) + } + return nil +} diff --git a/internal/sink/webhook.go b/internal/sink/webhook.go new file mode 100644 index 0000000..4bd5c19 --- /dev/null +++ b/internal/sink/webhook.go @@ -0,0 +1,246 @@ +package sink + +import ( + "bytes" + "context" + "errors" + "fmt" + "log/slog" + "math/rand/v2" + "net/http" + "strconv" + "strings" + "time" +) + +// WebhookOptions configures the webhook sink. +type WebhookOptions struct { + URL string + Method string + Headers []string + Timeout time.Duration + // Retries is the number of extra attempts after the first. + Retries int + Logger *slog.Logger +} + +// Webhook POSTs the report to an HTTP endpoint. +// +// The payload is the report document exactly as the other sinks emit it: one +// shape for every consumer, no per-destination formatting. +type Webhook struct { + url string + method string + headers map[string][]string + retries int + log *slog.Logger + client *http.Client + // sleep is the backoff wait, replaced in tests. + sleep func(ctx context.Context, d time.Duration) bool +} + +const ( + // maxBackoff caps the exponential wait so a long retry chain cannot + // outlive the deadline it is running under. + maxBackoff = 30 * time.Second + // maxShift is the largest exponent worth computing; beyond it the cap + // applies and the shift would overflow. + maxShift = 16 +) + +// NewWebhook builds the sink. +func NewWebhook(opts WebhookOptions) (*Webhook, error) { + if opts.Logger == nil { + return nil, errors.New("webhook: logger is required") + } + if opts.URL == "" { + return nil, errors.New("webhook: url is required") + } + if opts.Timeout <= 0 { + return nil, errors.New("webhook: timeout must be positive") + } + if opts.Retries < 0 { + return nil, errors.New("webhook: retries must not be negative") + } + + headers, err := parseHeaders(opts.Headers) + if err != nil { + return nil, fmt.Errorf("webhook: %w", err) + } + + method := strings.ToUpper(strings.TrimSpace(opts.Method)) + if method == "" { + method = http.MethodPost + } + + return &Webhook{ + url: opts.URL, + method: method, + headers: headers, + retries: opts.Retries, + log: opts.Logger, + client: &http.Client{Timeout: opts.Timeout}, + sleep: sleepCtx, + }, nil +} + +func (w *Webhook) Name() string { return "webhook" } + +// Deliver POSTs the report, retrying the failures that can plausibly succeed +// on a second try. +func (w *Webhook) Deliver(ctx context.Context, data []byte) error { + var lastErr error + + for attempt := 0; attempt <= w.retries; attempt++ { + if attempt > 0 { + wait := backoff(attempt, retryAfter(lastErr)) + w.log.Debug("webhook retry", "attempt", attempt, "wait", wait.String()) + if !w.sleep(ctx, wait) { + return fmt.Errorf("delivery abandoned: %w", ctx.Err()) + } + } + + status, err := w.post(ctx, data) + if err == nil { + w.log.Debug("webhook delivered", "status", status, "attempt", attempt+1) + return nil + } + lastErr = err + + if !retryable(err) { + // A 4xx will not become valid on a retry; the request itself is + // wrong, most often the credentials or the URL. + return err + } + } + return fmt.Errorf("after %d attempt(s): %w", w.retries+1, lastErr) +} + +func (w *Webhook) post(ctx context.Context, data []byte) (int, error) { + req, err := http.NewRequestWithContext(ctx, w.method, w.url, bytes.NewReader(data)) + if err != nil { + return 0, fmt.Errorf("build request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + for name, values := range w.headers { + for _, v := range values { + req.Header.Add(name, v) + } + } + + resp, err := w.client.Do(req) + if err != nil { + return 0, transportError{err: err} + } + defer func() { _ = resp.Body.Close() }() + // The body is drained so the connection can be reused; its contents are + // not reported, since a webhook target may echo the payload back. + _, _ = readAndDiscard(resp) + + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + return resp.StatusCode, nil + } + return resp.StatusCode, statusError{ + status: resp.StatusCode, + retryAfter: parseRetryAfter(resp.Header.Get("Retry-After")), + } +} + +// transportError is a failure to reach the endpoint at all. +type transportError struct{ err error } + +func (e transportError) Error() string { return "transport: " + e.err.Error() } +func (e transportError) Unwrap() error { return e.err } + +// statusError is a non-2xx response. +type statusError struct { + status int + retryAfter time.Duration +} + +func (e statusError) Error() string { return "unexpected status " + strconv.Itoa(e.status) } + +// retryable reports whether another attempt could plausibly succeed. +func retryable(err error) bool { + var status statusError + if errors.As(err, &status) { + // 429 and 5xx are the server saying "not now"; every other 4xx is it + // saying "not like this". + return status.status == http.StatusTooManyRequests || status.status >= 500 + } + var transport transportError + return errors.As(err, &transport) +} + +// retryAfter extracts a server-requested wait, if the last failure carried one. +func retryAfter(err error) time.Duration { + var status statusError + if errors.As(err, &status) { + return status.retryAfter + } + return 0 +} + +// backoff grows exponentially with jitter, but honours a server-requested +// wait when there is one: the endpoint knows better than the schedule. +func backoff(attempt int, requested time.Duration) time.Duration { + if requested > 0 { + return min(requested, maxBackoff) + } + // The shift overflows int64 past ~35 attempts, and a negative duration + // then panics the jitter. The cap is reached long before that anyway. + wait := maxBackoff + if attempt <= maxShift { + wait = min(time.Second<<(attempt-1), maxBackoff) + } + // Jitter keeps several jobs retrying in lockstep from synchronising. + return wait/2 + time.Duration(rand.Int64N(int64(wait/2)+1)) +} + +// parseRetryAfter reads the header in both of its forms: seconds, or a date. +func parseRetryAfter(value string) time.Duration { + value = strings.TrimSpace(value) + if value == "" { + return 0 + } + if seconds, err := strconv.Atoi(value); err == nil { + if seconds < 0 { + return 0 + } + return time.Duration(seconds) * time.Second + } + if when, err := http.ParseTime(value); err == nil { + if d := time.Until(when); d > 0 { + return d + } + } + return 0 +} + +func sleepCtx(ctx context.Context, d time.Duration) bool { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-timer.C: + return true + case <-ctx.Done(): + return false + } +} + +// parseHeaders turns "Name: value" entries into a header map. +func parseHeaders(headers []string) (map[string][]string, error) { + if len(headers) == 0 { + return nil, nil + } + out := make(map[string][]string, len(headers)) + for _, h := range headers { + name, value, ok := strings.Cut(h, ":") + name, value = strings.TrimSpace(name), strings.TrimSpace(value) + if !ok || name == "" || value == "" { + return nil, fmt.Errorf("header %q must be in 'Name: value' form", h) + } + out[name] = append(out[name], value) + } + return out, nil +} diff --git a/internal/sink/webhook_test.go b/internal/sink/webhook_test.go new file mode 100644 index 0000000..7cfc977 --- /dev/null +++ b/internal/sink/webhook_test.go @@ -0,0 +1,229 @@ +package sink + +import ( + "context" + "errors" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "strings" + "sync/atomic" + "testing" + "time" +) + +func discardLogger() *slog.Logger { + return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.Level(99)})) +} + +// newTestWebhook builds a webhook whose backoff does not actually wait. +func newTestWebhook(t *testing.T, opts WebhookOptions) (*Webhook, *[]time.Duration) { + t.Helper() + if opts.Timeout == 0 { + opts.Timeout = 5 * time.Second + } + opts.Logger = discardLogger() + + w, err := NewWebhook(opts) + if err != nil { + t.Fatalf("NewWebhook: %v", err) + } + var waits []time.Duration + w.sleep = func(ctx context.Context, d time.Duration) bool { + waits = append(waits, d) + return ctx.Err() == nil + } + return w, &waits +} + +func TestWebhookSendsTheReportAsJSON(t *testing.T) { + var ( + gotBody string + gotMethod string + gotAuth string + gotType string + ) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + gotBody, gotMethod = string(body), r.Method + gotAuth = r.Header.Get("Authorization") + gotType = r.Header.Get("Content-Type") + w.WriteHeader(http.StatusAccepted) + })) + defer srv.Close() + + wh, _ := newTestWebhook(t, WebhookOptions{ + URL: srv.URL, + Headers: []string{"Authorization: Bearer token"}, + }) + if err := wh.Deliver(context.Background(), []byte(`{"schema_version":"1.0"}`)); err != nil { + t.Fatalf("Deliver: %v", err) + } + + if gotBody != `{"schema_version":"1.0"}` { + t.Errorf("body = %q, want the report unchanged", gotBody) + } + if gotMethod != http.MethodPost { + t.Errorf("method = %q, want POST", gotMethod) + } + if gotAuth != "Bearer token" { + t.Errorf("authorization = %q, want the configured header", gotAuth) + } + if gotType != "application/json" { + t.Errorf("content-type = %q, want application/json", gotType) + } +} + +func TestWebhookRetriesServerErrors(t *testing.T) { + var attempts atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if attempts.Add(1) < 3 { + w.WriteHeader(http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + wh, waits := newTestWebhook(t, WebhookOptions{URL: srv.URL, Retries: 3}) + if err := wh.Deliver(context.Background(), []byte("{}")); err != nil { + t.Fatalf("Deliver: %v", err) + } + if got := attempts.Load(); got != 3 { + t.Errorf("attempts = %d, want 3", got) + } + // Backoff must grow, or a struggling endpoint gets hammered. + if len(*waits) != 2 || (*waits)[1] <= (*waits)[0] { + t.Errorf("waits = %v, want an increasing backoff", *waits) + } +} + +// A 4xx will not become valid on a retry: the request itself is wrong. +func TestWebhookDoesNotRetryClientErrors(t *testing.T) { + var attempts atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + attempts.Add(1) + w.WriteHeader(http.StatusForbidden) + })) + defer srv.Close() + + wh, _ := newTestWebhook(t, WebhookOptions{URL: srv.URL, Retries: 5}) + err := wh.Deliver(context.Background(), []byte("{}")) + if err == nil { + t.Fatal("Deliver reported success on a 403") + } + if got := attempts.Load(); got != 1 { + t.Errorf("attempts = %d, want exactly one", got) + } + if !strings.Contains(err.Error(), "403") { + t.Errorf("error = %q, want the status named", err) + } +} + +// 429 is the server saying "not now", unlike every other 4xx. +func TestWebhookRetriesRateLimits(t *testing.T) { + var attempts atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if attempts.Add(1) == 1 { + w.Header().Set("Retry-After", "7") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + wh, waits := newTestWebhook(t, WebhookOptions{URL: srv.URL, Retries: 2}) + if err := wh.Deliver(context.Background(), []byte("{}")); err != nil { + t.Fatalf("Deliver: %v", err) + } + // The endpoint asked for a specific wait; it knows better than the schedule. + if len(*waits) != 1 || (*waits)[0] != 7*time.Second { + t.Errorf("waits = %v, want the requested 7s honoured", *waits) + } +} + +func TestWebhookRetriesTransportFailures(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + url := srv.URL + srv.Close() // nothing is listening any more + + wh, waits := newTestWebhook(t, WebhookOptions{URL: url, Retries: 2, Timeout: 500 * time.Millisecond}) + err := wh.Deliver(context.Background(), []byte("{}")) + if err == nil { + t.Fatal("Deliver reported success against a dead endpoint") + } + if len(*waits) != 2 { + t.Errorf("waits = %v, want both retries attempted", *waits) + } + if !strings.Contains(err.Error(), "attempt") { + t.Errorf("error = %q, want it to say how many attempts were made", err) + } +} + +func TestWebhookStopsWhenTheDeliveryBudgetEnds(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer srv.Close() + + wh, _ := newTestWebhook(t, WebhookOptions{URL: srv.URL, Retries: 10}) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + err := wh.Deliver(ctx, []byte("{}")) + if err == nil { + t.Fatal("Deliver reported success after its budget ended") + } + if !errors.Is(err, context.Canceled) { + t.Errorf("error = %v, want it to carry the cancellation", err) + } +} + +func TestParseRetryAfter(t *testing.T) { + if got := parseRetryAfter("12"); got != 12*time.Second { + t.Errorf("seconds form = %s, want 12s", got) + } + future := time.Now().Add(20 * time.Second).UTC().Format(http.TimeFormat) + if got := parseRetryAfter(future); got <= 0 || got > 21*time.Second { + t.Errorf("date form = %s, want roughly 20s", got) + } + for _, in := range []string{"", "soon", "-5", time.Now().Add(-time.Hour).UTC().Format(http.TimeFormat)} { + if got := parseRetryAfter(in); got != 0 { + t.Errorf("parseRetryAfter(%q) = %s, want 0", in, got) + } + } +} + +func TestBackoffIsCapped(t *testing.T) { + // The shift overflows int64 past ~35 attempts; a negative duration then + // panics the jitter, crashing the process after the report was produced. + for _, attempt := range []int{1, 2, 16, 20, 35, 64, 1000} { + got := backoff(attempt, 0) + if got <= 0 || got > maxBackoff { + t.Errorf("backoff(%d) = %s, want a positive wait within %s", attempt, got, maxBackoff) + } + } + if got := backoff(1, time.Hour); got != maxBackoff { + t.Errorf("a huge Retry-After gave %s, want it capped at %s", got, maxBackoff) + } +} + +func TestNewWebhookRejectsUnusableOptions(t *testing.T) { + base := WebhookOptions{URL: "https://example.net/hook", Timeout: time.Second, Logger: discardLogger()} + for name, mutate := range map[string]func(*WebhookOptions){ + "no logger": func(o *WebhookOptions) { o.Logger = nil }, + "no url": func(o *WebhookOptions) { o.URL = "" }, + "no timeout": func(o *WebhookOptions) { o.Timeout = 0 }, + "negative retries": func(o *WebhookOptions) { o.Retries = -1 }, + "bad header": func(o *WebhookOptions) { o.Headers = []string{"nope"} }, + } { + opts := base + mutate(&opts) + if _, err := NewWebhook(opts); err == nil { + t.Errorf("NewWebhook accepted options with %s", name) + } + } +} diff --git a/internal/stage/stage.go b/internal/stage/stage.go new file mode 100644 index 0000000..9a27491 --- /dev/null +++ b/internal/stage/stage.go @@ -0,0 +1,101 @@ +// Package stage defines the pipeline stages and the scope ladder that selects them. +package stage + +import "fmt" + +// Stage is a single step of the recon pipeline. +type Stage string + +const ( + Enumerate Stage = "enumerate" + Exclude Stage = "exclude" + Resolve Stage = "resolve" + PortScan Stage = "portscan" + HTTPProbe Stage = "httpprobe" +) + +// Scope selects how far up the ladder a run goes. Scopes form a strict ladder: +// each one runs every stage of the scope below it plus one more. Arbitrary +// combinations are deliberately not expressible — probing without a port scan +// would mean inventing a default port set, which looks like discovery but is +// really an assumption. +type Scope string + +const ( + ScopeEnum Scope = "enum" + ScopeResolve Scope = "resolve" + ScopePorts Scope = "ports" + ScopeFull Scope = "full" +) + +var ladder = []struct { + scope Scope + stages []Stage +}{ + {ScopeEnum, []Stage{Enumerate, Exclude}}, + {ScopeResolve, []Stage{Enumerate, Exclude, Resolve}}, + {ScopePorts, []Stage{Enumerate, Exclude, Resolve, PortScan}}, + {ScopeFull, []Stage{Enumerate, Exclude, Resolve, PortScan, HTTPProbe}}, +} + +// Scopes returns every valid scope, ordered from narrowest to widest. +func Scopes() []Scope { + out := make([]Scope, 0, len(ladder)) + for _, l := range ladder { + out = append(out, l.scope) + } + return out +} + +// ParseScope resolves a scope name, rejecting anything outside the ladder. +func ParseScope(s string) (Scope, error) { + for _, l := range ladder { + if string(l.scope) == s { + return l.scope, nil + } + } + return "", fmt.Errorf("unknown stage scope %q (valid: %s)", s, join(Scopes())) +} + +// Stages returns the ordered stages a scope runs. +func (s Scope) Stages() []Stage { + for _, l := range ladder { + if l.scope == s { + return l.stages + } + } + return nil +} + +// Includes reports whether the scope runs the given stage. +func (s Scope) Includes(st Stage) bool { + for _, have := range s.Stages() { + if have == st { + return true + } + } + return false +} + +func (s Scope) String() string { return string(s) } + +// StageNames renders a scope's stages as strings, for the run report. +func (s Scope) StageNames() []string { + stages := s.Stages() + out := make([]string, len(stages)) + for i, st := range stages { + out[i] = string(st) + } + return out +} + +func join(scopes []Scope) string { + out := "" + for i, s := range scopes { + if i > 0 { + out += ", " + } + out += string(s) + } + return out +} diff --git a/internal/stage/stage_test.go b/internal/stage/stage_test.go new file mode 100644 index 0000000..32a427b --- /dev/null +++ b/internal/stage/stage_test.go @@ -0,0 +1,40 @@ +package stage + +import "testing" + +func TestParseScopeRejectsArbitraryCombinations(t *testing.T) { + for _, in := range []string{"enumerate,httpprobe", "", "ENUM", "everything"} { + if _, err := ParseScope(in); err == nil { + t.Errorf("ParseScope(%q) succeeded, want an error", in) + } + } +} + +func TestScopeLadderIsCumulative(t *testing.T) { + prev := 0 + for _, s := range Scopes() { + stages := s.Stages() + if len(stages) <= prev { + t.Fatalf("scope %s has %d stages, want more than the scope below (%d)", s, len(stages), prev) + } + if stages[0] != Enumerate { + t.Errorf("scope %s starts at %s, want %s", s, stages[0], Enumerate) + } + prev = len(stages) + } +} + +func TestScopeIncludes(t *testing.T) { + if ScopeEnum.Includes(Resolve) { + t.Error("enum scope must not include the resolve stage") + } + if !ScopeFull.Includes(HTTPProbe) { + t.Error("full scope must include the httpprobe stage") + } + if !ScopePorts.Includes(Resolve) { + t.Error("ports scope must include resolve: the port scan needs addresses") + } + if ScopePorts.Includes(HTTPProbe) { + t.Error("ports scope must stop before httpprobe") + } +} diff --git a/internal/version/version.go b/internal/version/version.go new file mode 100644 index 0000000..a3c757e --- /dev/null +++ b/internal/version/version.go @@ -0,0 +1,55 @@ +// Package version carries the build identity, injected at link time. +package version + +import ( + "fmt" + "runtime/debug" +) + +// Set with -ldflags "-X github.com/JoshuaMart/FastRecon/internal/version.Version=..." +var ( + Version = "dev" + Commit = "" + Date = "" +) + +func init() { + if Commit != "" { + return + } + // Fall back to the VCS stamp the toolchain embeds for `go build` and + // `go install` outside the release pipeline. + info, ok := debug.ReadBuildInfo() + if !ok { + return + } + for _, s := range info.Settings { + switch s.Key { + case "vcs.revision": + Commit = s.Value + case "vcs.time": + Date = s.Value + } + } +} + +// String renders the full build identity. +func String() string { + s := Version + if Commit != "" { + short := Commit + if len(short) > 12 { + short = short[:12] + } + s += " (" + short + ")" + } + if Date != "" { + s += " built " + Date + } + return s +} + +// UserAgent is the default User-Agent for outbound requests. +func UserAgent() string { + return fmt.Sprintf("FastRecon/%s", Version) +} diff --git a/main.go b/main.go deleted file mode 100644 index 629ef89..0000000 --- a/main.go +++ /dev/null @@ -1,96 +0,0 @@ -package main - -import ( - "bufio" - "bytes" - "encoding/json" - "fmt" - "net/http" - "os/exec" - "strings" -) - -type HttpxResponse struct { - URL string `json:"url"` - Status int `json:"status_code"` - Size int `json:"content_length"` - Type string `json:"content_type"` - Title string `json:"title"` - IP []string `json:"a"` - CNAME []string `json:"cname"` - CDN bool `json:"cdn"` - Tech []string `json:"tech"` - Headers map[string]string `json:"header"` -} - -func handler(w http.ResponseWriter, r *http.Request) { - // Get query parameters - domain := r.URL.Query().Get("domain") - rawParam := r.URL.Query().Get("raw") - - if domain == "" { - http.Error(w, "Error: domain parameter is required", http.StatusBadRequest) - return - } - - // Build httpx command based on raw parameter - var httpxCmd string - if rawParam == "true" { - httpxCmd = "httpx -silent" - } else { - httpxCmd = "httpx -silent -sc -cl -ct -title -td -ip -cname -cdn -irh -j" - } - - // Run the command and capture the output - var output bytes.Buffer - command := fmt.Sprintf(`subfinder -pc subfinder.yaml -silent -d %s | - puredns resolve -q --resolvers resolvers.txt --resolvers-trusted resolvers-trusted.txt | - %s - `, domain, httpxCmd) - - exec := exec.Command("sh", "-c", command) - exec.Stdout = &output - err := exec.Run() - if err != nil { - http.Error(w, fmt.Sprintf("Error: %v", err), http.StatusInternalServerError) - return - } - - // If raw=true, return the domain list directly - if rawParam == "true" { - w.Header().Set("Content-Type", "text/plain") - w.Write(output.Bytes()) - return - } - - // Otherwise, parse JSON as before - scanner := bufio.NewScanner(strings.NewReader(output.String())) - var jsonArray []HttpxResponse - - for scanner.Scan() { - line := scanner.Text() - var httpxResponse HttpxResponse - - err := json.Unmarshal([]byte(line), &httpxResponse) - if err != nil { - fmt.Fprintf(w, "Error parsing JSON Lines output: %v", err) - return - } - jsonArray = append(jsonArray, httpxResponse) - } - - // Convert the slice of JSON objects to a single JSON array - jsonBytes, err := json.Marshal(jsonArray) - if err != nil { - fmt.Fprintf(w, "Error converting JSON array to bytes: %v", err) - return - } - - w.Header().Set("Content-Type", "application/json") - w.Write(jsonBytes) -} - -func main() { - http.HandleFunc("/", handler) - http.ListenAndServe(":8080", nil) -} \ No newline at end of file diff --git a/subfinder.yaml b/subfinder.yaml deleted file mode 100644 index 94bc26d..0000000 --- a/subfinder.yaml +++ /dev/null @@ -1,30 +0,0 @@ -bevigil: [] -binaryedge: [] -bufferover: [] -builtwith: [] -c99: [] -censys: [] -certspotter: [] -chaos: [] -chinaz: [] -digitalyama: [] -dnsdb: [] -dnsdumpster: [] -dnsrepo: [] -facebook: [] -fofa: [] -fullhunt: [] -github: [] -hunter: [] -intelx: [] -leakix: [] -netlas: [] -quake: [] -redhuntlabs: [] -robtex: [] -securitytrails: [] -shodan: [] -threatbook: [] -virustotal: [] -whoisxmlapi: [] -zoomeyeapi: []