Scanning Module Oslo Accords

CHANGELOG.md

@@ -15,6 +15,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Module dependency graph resolution, so modules can depend on other modules
   and are loaded in the right order.
 - PDF summary output for network scans.
+- `scan` command: enumerates the target's subdomains and runs every installed
+  module against each one, returning the whole run as JSON. The shell prints
+  it as an indented tree instead of raw JSON.
+- Live scan progress with the usual `[+]`, `[-]`, and `[!]` markers, so a run
+  reports as it goes rather than only at the end. Added `internal/logging` for
+  the concurrency-safe output behind it, and wired it into the DNS and HTTP
+  modules.
+- Scans skip subdomains that no longer resolve, so dead names left behind in
+  certificate transparency logs no longer cost a timeout per module.
 - Community and contribution scaffolding: `CONTRIBUTING.md`, issue templates,
   and a PR review request workflow.
 
@@ -26,6 +35,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   `internal/style`, unifying how commands and modules print to the terminal.
 - Reworked DNS, HTTP, and subdomain record handling in `pkg/records` and
   `pkg/subdomains`.
+- The HTTP module fetches its target once instead of twice: it checked the
+  site status and then refetched the same URL for the records.
+- Subdomain enumeration during a scan is bounded by a timeout, so an
+  unresponsive certificate transparency source can no longer hang the run.
 - Documentation updates to `README.md` covering installation, module
   management, and usage.
 

cmd/main.go

@@ -79,9 +79,6 @@ func main() {
 
 func run(target string) error {
 	printBanner()
-	if err := management.InstallModule(context.Background(), "https://github.com/GoScouter/nginx-module"); err != nil {
-		panic(err)
-	}
 
 	if err := versions.SuggestUpdate(VERSION); err != nil {
 		return fmt.Errorf("update check: %w", err)
@@ -130,6 +127,8 @@ func run(target string) error {
 		interrupted.Store(true)
 	}()
 
+	runner.CleanupState()
+
 	reader := bufio.NewReader(os.Stdin)
 	for !interrupted.Load() {
 		fmt.Print(style.Prompt())

internal/cmd/command.go

@@ -64,6 +64,11 @@ func NewManager(target string, manager *module.Manager) (*CommandManager, error)
 		})
 	}
 
+	cm.addCommand(&ScanCommand{
+		CmdManager:    cm,
+		ModuleManager: manager,
+	})
+
 	return cm, nil
 }
 

internal/cmd/help.go

@@ -54,6 +54,16 @@ func (cmd *HelpCommand) Exec(args []string) error {
 		style.Dim("Returns goscouter cli version"),
 	))
 
+	b.WriteString(fmt.Sprintf("  %s  %s\r\n",
+		style.Cyan("--install"),
+		style.Dim("Installs an external module"),
+	))
+
+	b.WriteString(fmt.Sprintf("  %s  %s\r\n",
+		style.Cyan("--uninstall"),
+		style.Dim("Uninstalls an external module"),
+	))
+
 	fmt.Printf("%s\r\n", b.String())
 	return nil
 }

internal/cmd/scan.go

@@ -0,0 +1,30 @@
+package cmd
+
+import (
+	"fmt"
+	"goscouter/internal/module"
+	"goscouter/internal/scanning"
+)
+
+type ScanCommand struct {
+	CmdManager    *CommandManager
+	ModuleManager *module.Manager
+}
+
+func (cmd *ScanCommand) Name() string {
+	return "scan"
+}
+
+func (cmd *ScanCommand) Description() string {
+	return "Makes a complete scan of the target"
+}
+
+func (cmd *ScanCommand) Exec(args []string) error {
+	output, err := scanning.Scan(cmd.CmdManager.Target, cmd.ModuleManager)
+	if err != nil {
+		return err
+	}
+
+	fmt.Print("\r\n" + scanning.Render(output))
+	return nil
+}

internal/dns/dns.go

@@ -11,6 +11,22 @@ import (
 
 const TIMEOUT time.Duration = 5 * time.Second
 
+const RESOLVE_TIMEOUT time.Duration = 2 * time.Second
+
+func Resolve(ctx context.Context, host string) ([]string, error) {
+	ctx, cancel := context.WithTimeout(ctx, RESOLVE_TIMEOUT)
+	defer cancel()
+
+	addrs, err := net.DefaultResolver.LookupHost(ctx, host)
+	if err != nil {
+		return nil, err
+	}
+	if len(addrs) == 0 {
+		return nil, fmt.Errorf("no addresses for %s", host)
+	}
+	return addrs, nil
+}
+
 func Lookup(host string) (*records.DNSRecords, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), TIMEOUT)
 	defer cancel()

internal/logging/log.go

@@ -0,0 +1,46 @@
+package logging
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"sync"
+
+	"goscouter/internal/style"
+)
+
+var (
+	mu      sync.Mutex
+	out     io.Writer = os.Stdout
+	enabled           = true
+)
+
+func SetOutput(w io.Writer) {
+	mu.Lock()
+	defer mu.Unlock()
+	out = w
+}
+
+func SetEnabled(on bool) {
+	mu.Lock()
+	defer mu.Unlock()
+	enabled = on
+}
+
+func line(s string) {
+	mu.Lock()
+	defer mu.Unlock()
+
+	if !enabled {
+		return
+	}
+	fmt.Fprint(out, s+"\r\n")
+}
+
+func Found(format string, a ...any) { line(style.Foundf(format, a...)) }
+
+func Failed(format string, a ...any) { line(style.Failuref(format, a...)) }
+
+func Alert(format string, a ...any) { line(style.Alertf(format, a...)) }
+
+func Step(format string, a ...any) { line(style.Infof(format, a...)) }

internal/module/dns.go

@@ -6,10 +6,12 @@ import (
 	"net/url"
 
 	"goscouter/internal/dns"
+	"goscouter/internal/logging"
 	"goscouter/internal/style"
 	"goscouter/pkg/records"
 
 	"github.com/GoScouter/sdk"
+	"golang.org/x/net/publicsuffix"
 )
 
 type DnsModule struct{}
@@ -24,16 +26,29 @@ func (m *DnsModule) Info() sdk.ModuleInfo {
 }
 
 func (m *DnsModule) Scout(target string, _ []string) (json.RawMessage, error) {
+	eTLDPlusOne, err := publicsuffix.EffectiveTLDPlusOne(target)
+	if err != nil {
+		return nil, fmt.Errorf("invalid domain")
+	}
+
+	if target != eTLDPlusOne {
+		return nil, nil
+	}
+
 	parsed, err := url.Parse(target)
 	if err != nil {
 		return nil, fmt.Errorf("invalid target %q: %w", target, err)
 	}
 
 	records, err := dns.Lookup(parsed.Path)
 	if err != nil {
+		logging.Failed("dns: %s: %v", target, err)
 		return nil, err
 	}
 
+	logging.Found("dns: %s (%d A, %d AAAA, %d MX, %d NS, %d TXT)",
+		target, len(records.A), len(records.AAAA), len(records.MX), len(records.NS), len(records.TXT))
+
 	data, err := json.Marshal(records)
 	if err != nil {
 		return nil, err

internal/module/http.go

@@ -7,6 +7,7 @@ import (
 	"net/url"
 	"strings"
 
+	"goscouter/internal/logging"
 	"goscouter/internal/style"
 	"goscouter/internal/web"
 	"goscouter/pkg/records"
@@ -53,16 +54,14 @@ func (m *HttpModule) Scout(target string, args []string) (json.RawMessage, error
 		target = forceScheme(target, "http")
 	}
 
-	_, err := web.CheckSiteStatus(target)
-	if err != nil {
-		return nil, err
-	}
-
 	records, err := web.FetchHTTPRecords(target, scheme)
 	if err != nil {
+		logging.Failed("http: %s: %v", target, err)
 		return nil, err
 	}
 
+	logging.Found("http: %s %s", target, records.Status)
+
 	data, err := json.Marshal(records)
 	if err != nil {
 		return nil, err

internal/module/runner.go

@@ -236,47 +236,53 @@ func RunInOrder(info sdk.ModuleInfo, target string, args []string, manager *Mana
 	viewOutput := make([]string, 0, len(plan))
 
 	for _, key := range plan {
-		author, name, _ := strings.Cut(key, ":")
-
-		ns := sdk.ModuleNamespace{Author: author, Name: name}
+		data, view, err := RunModule(target, args, key, manager, reporter)
+		if err != nil {
+			return nil, nil, err
+		}
 
-		if author != internalAuthor {
-			m := manager.GetExternal(key)
-			if m == nil {
-				return dataOutput, viewOutput, fmt.Errorf("unknown module %q", key)
-			}
+		dataOutput = append(dataOutput, data)
+		viewOutput = append(viewOutput, view)
+	}
 
-			data, view, err := m.Scout(context.Background(), target, args)
-			if err != nil {
-				return dataOutput, viewOutput, err
-			}
+	return dataOutput, viewOutput, nil
+}
 
-			if err := reporter.Report(ns, data); err != nil {
-				return dataOutput, viewOutput, err
-			}
+func RunModule(target string, args []string, key string, manager *Manager, reporter *Reporter) (json.RawMessage, string, error) {
+	author, name, _ := strings.Cut(key, ":")
+	ns := sdk.ModuleNamespace{Author: author, Name: name}
 
-			dataOutput = append(dataOutput, data)
-			viewOutput = append(viewOutput, view)
-			continue
-		}
-
-		m := manager.GetInternal(key)
+	if ns.Author != internalAuthor {
+		m := manager.GetExternal(key)
 		if m == nil {
-			return dataOutput, viewOutput, fmt.Errorf("unknown module %q", key)
+			return nil, "", fmt.Errorf("unknown module %q", key)
 		}
 
-		data, err := m.Scout(target, args)
+		data, view, err := m.Scout(context.Background(), target, args)
 		if err != nil {
-			return dataOutput, viewOutput, err
+			return nil, "", err
 		}
 
 		if err := reporter.Report(ns, data); err != nil {
-			return dataOutput, viewOutput, err
+			return nil, "", err
 		}
 
-		dataOutput = append(dataOutput, data)
-		viewOutput = append(viewOutput, m.Render(data))
+		return data, view, nil
 	}
 
-	return dataOutput, viewOutput, nil
+	m := manager.GetInternal(key)
+	if m == nil {
+		return nil, "", fmt.Errorf("unknown module %q", key)
+	}
+
+	data, err := m.Scout(target, args)
+	if err != nil {
+		return nil, "", err
+	}
+
+	if err := reporter.Report(ns, data); err != nil {
+		return nil, "", err
+	}
+
+	return data, m.Render(data), nil
 }

internal/net/subdomain/finder.go

@@ -22,8 +22,9 @@ type Finder struct {
 }
 
 var Finders = map[string]Finder{
-	"crtsh":       {Name: "crtsh", Fetch: fetchCrtSh},
-	"certspotter": {Name: "certspotter", Fetch: fetchCertSpotter},
+	"crtsh":        {Name: "crtsh", Fetch: fetchCrtSh},
+	"certspotter":  {Name: "certspotter", Fetch: fetchCertSpotter},
+	"hackertarget": {Name: "hackertarget", Fetch: fetchHackerTarget},
 }
 
 const TIMEOUT time.Duration = 5 * time.Second
@@ -147,6 +148,55 @@ func fetchCertSpotter(ctx context.Context, domain string) ([]subdomains.Subdomai
 	return flatten(latest), nil
 }
 
+func fetchHackerTarget(ctx context.Context, domain string) ([]subdomains.Subdomain, error) {
+	rawURL := fmt.Sprintf("https://api.hackertarget.com/hostsearch/?q=%s", url.QueryEscape(domain))
+
+	req, err := newRequest(ctx, rawURL)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("hackertarget returned status %d", resp.StatusCode)
+	}
+
+	suffix := "." + normalize(domain)
+
+	latest := make(map[string]time.Time)
+	scanner := bufio.NewScanner(resp.Body)
+
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if line == "" {
+			continue
+		}
+
+		host, _, ok := strings.Cut(line, ",")
+		if !ok {
+			return nil, fmt.Errorf("hackertarget: %s", line)
+		}
+
+		host = normalize(host)
+		if host != normalize(domain) && !strings.HasSuffix(host, suffix) {
+			continue
+		}
+
+		keepLatest(latest, host, time.Time{})
+	}
+
+	if err := scanner.Err(); err != nil {
+		return nil, fmt.Errorf("reading hackertarget response: %w", err)
+	}
+
+	return flatten(latest), nil
+}
+
 type finderResult struct {
 	source string
 	subs   []subdomains.Subdomain