Anatomy of a Scan

CLI invocation
  │
  ▼
┌─────────────────────┐
│  CLI Entry & Config  │  cmd/vigolium/main.go → pkg/cli/scan.go
│  Flag parsing, config│  Config loading, strategy/profile, DB init
│  loading, DB init    │
└────────┬────────────┘
         │
         ▼
┌─────────────────────┐
│   Input Parsing      │  pkg/input/source/
│   URL/file/stdin →   │  InputSource.Next() → WorkItem
│   WorkItem stream    │
└────────┬────────────┘
         │
         ▼
┌─────────────────────┐
│  Runner Orchestration│  internal/runner/runner.go
│  7-phase pipeline:   │  Heuristics → Harvest → Spider →
│  build infra, run    │  SAST → Discovery → KnownIssueScan →
│  phases in order     │  Audit
└────────┬────────────┘
         │
         ▼
┌─────────────────────┐
│     Executor         │  pkg/core/executor.go
│  Worker pool feeds   │  feedItems() → worker() → processItem()
│  items to modules    │
└────────┬────────────┘
         │
         ▼
┌─────────────────────┐
│  Module Dispatch     │  pkg/modules/
│  Passive (sequential)│  ScanPerHost → ScanPerRequest
│  Active (parallel)   │  ScanPerHost/Request/InsertionPoint
└────────┬────────────┘
         │
         ▼
┌─────────────────────┐
│   Result Emission    │  pkg/output/output.go
│  Post-hooks → DB     │  assignModuleInfo → emitResult →
│  save → output write │  SaveFinding → OnResult → Notify
└─────────────────────┘

Has --input file?  ──yes──▶  runScanWithIngest()    Parse file, create InputSource, run
       │ no
       ▼
Has targets?       ──no───▶  runDBScan()            Scan existing DB records (empty source)
       │ yes
       ▼
runner.New(scanOpts)         ──────────────────▶    Target-based: build source from CLI targets
  .SetSettings(settings)
  .SetRepository(repo)
  .RunNativeScan()
  .Close()

type InputSource interface {
    Next(ctx context.Context) (*work.WorkItem, error)
    Close() error
}

type WorkItem struct {
    Request       *httpmsg.HttpRequestResponse
    EnableModules []string   // per-item module selection (empty = all)
    RecordUUID    string     // pre-existing DB record UUID (skip store)
    onComplete    func()     // queue ack callback (unexported)
}

type HttpRequestResponse struct {
    request  *HttpRequest   // required
    response *HttpResponse  // optional, may be nil
}

type HttpRequest struct {
    raw     []byte     // source of truth
    service *Service   // host/port/protocol
    // lazy-parsed cache (populated by ensureParsed())
    method, path string
    headers      []HttpHeader
    bodyOffset   int
    parsed       bool
    mu           sync.RWMutex
}

type HttpResponse struct {
    raw        []byte
    statusCode int
    headers    []HttpHeader
    bodyOffset int
    parsed     bool
    mu         sync.RWMutex
}

type Service struct {
    host     string   // hostname only (no port)
    port     int
    protocol string   // "http" or "https"
}

type Runner struct {
    output            output.Writer
    options           *types.Options
    settings          *config.Settings
    inputSource       source.InputSource
    dedupManager      *dedup.Manager
    repository        *database.Repository
    heuristicsResults map[string]*HeuristicsResult
}

type phaseInfra struct {
    svc           *services.Services
    httpRequester *http.Requester
    scopeMatcher  *config.ScopeMatcher
    hostLimiter   *hostlimit.HostRateLimiter
    notifier      *notify.Manager
    hookChain     *jsext.HookChain
    jsEngine      *jsext.Engine
    scanUUID      string
}

RunNativeScan()
│
├── buildInfrastructure()
│
├─── Phase 0: Heuristics Check     [guard: heuristicsCheck != "none"]
│    Probes target root pages, detects blank/JSON/SPA responses.
│    Flags targets to skip spidering.
│
├─── Phase 1: External Harvest     [guard: ExternalHarvestEnabled]
│    Queries Wayback, CommonCrawl, AlienVault, URLScan, VirusTotal.
│    Ingests discovered URLs into DB (no modules, pure ingestion).
│
├─── Phase 2: Spidering            [guard: SpideringEnabled]
│    Browser-based crawling (Chromium). Applies heuristics filter.
│    Stores discovered pages in DB via repository.
│
├─── Phase 3: SAST                 [guard: SASTEnabled]
│    Static analysis via ast-grep. Extracts routes from source code.
│    Ingests routes as minimal HTTP requests into DB.
│
├─── Phase 4: Discovery            [guard: !SkipIngestion]
│    Content discovery (brute-force dirs/files via deparos engine)
│    + CLI input source. Both wrapped in MultiSource.
│    Ingests into DB (no modules, pure ingestion).
│    Fallback: seedCLITargets() if ingestion skipped but KnownIssueScan/DA need records.
│
├─── Phase 5: KnownIssueScan       [guard: KnownIssueScanEnabled]
│    Nuclei template scan + Kingfisher secret detection on stored
│    response bodies. Targets enriched with discovered paths
│    (enrich_targets). Filters out secret_detect passive module
│    to avoid duplicates in DA phase.
│    Post-phase: DeduplicateFindings() groups same-module/URL findings.
│
└─── Phase 6: Audit                [guard: !SkipAudit]
     THE CORE SCANNING PHASE. Reads records from DB, dispatches
     active + passive modules. Per-module finding cap suppresses
     noisy modules. Feedback loop (up to 3 rounds) re-scans
     newly discovered URLs.
     Post-phase: DeduplicateFindings() merges redundant findings.

type Executor struct {
    cfg            ExecutorConfig
    source         source.InputSource
    activeModules  []modules.ActiveModule
    passiveModules []modules.PassiveModule
    httpClient     *http.Requester
    scanCtx        *modules.ScanContext
    hooks          HookRunner

    // Pre-grouped by scan scope at init time
    perHostActive     []modules.ActiveModule
    perRequestActive  []modules.ActiveModule
    perIPActive       []modules.ActiveModule
    perHostPassive    []modules.PassiveModule
    perRequestPassive []modules.PassiveModule

    ipCache      *lru.Cache[string, []httpmsg.InsertionPoint]  // 4096-entry LRU
    requestUUIDs *shardedMap   // request hash → DB record UUID
}

func (e *Executor) Execute(ctx context.Context) (bool, error)

for item := range itemCh {
    e.processItem(item)
    item.Complete()
    e.statsTracker.Increment()
}

if SkipBaseline && response already attached:
    use existing response (DB-sourced items in audit phase)
else:
    httpClient.Execute(request) → response
    copy response bytes from pool before Close()
    attach response to request via WithResponse()

hooks.RunPreHooks(request)
  → error: log and skip item
  → nil return: hook filtered it out, skip item
  → modified request: continue with transformed request

if ScopeMatcher configured:
    check full scope (host, path, status, content types, body strings)
    if out-of-scope and ScopeOnIngest: drop entirely (no save, no scan)
    save to database
    if out-of-scope: saved but not scanned → return
else:
    save to database and continue

runPassivePerHost(request, filter)      sequential loop over perHostPassive
runPassivePerRequest(request, filter)   sequential loop over perRequestPassive

var g conc.WaitGroup
g.Go(func() { runActivePerHost(request, filter, eligibility) })
g.Go(func() { runActivePerRequest(request, filter, eligibility) })
g.Go(func() { runActivePerInsertionPoint(request, filter, eligibility) })
g.Wait()

insertion points = ipCache.GetOrCompute(requestHash)
for each insertionPoint:
    for each eligible module (parallel):
        module.ScanPerInsertionPoint(request, insertionPoint, httpClient, scanCtx)

Execute()
├── feedItems()                          [calling goroutine, producer]
└── Workers goroutines                   [consumer pool]
    └── processItem()
        ├── Passive modules              [sequential on worker goroutine]
        │   ├── runPassivePerHost
        │   └── runPassivePerRequest
        └── Active modules               [3-way parallel via conc.WaitGroup]
            ├── runActivePerHost          [inner parallel: all modules]
            ├── runActivePerRequest       [inner parallel: all modules]
            └── runActivePerInsertionPoint
                └── for each IP (serial)  [inner parallel: all modules]

type InsertionPoint interface {
    Name() string                        // parameter name (e.g. "id", "username")
    BaseValue() string                   // original value at this position
    Type() InsertionPointType            // one of INS_* constants
    BuildRequest(payload []byte) []byte  // new request bytes with payload injected
    PayloadOffsets(payload []byte) []int  // [startOffset, endOffset] in built request
}

Module (base)
├── ActiveModule
│   ├── ScanPerInsertionPoint(request, insertionPoint, httpClient, scanCtx)
│   ├── ScanPerRequest(request, httpClient, scanCtx)
│   ├── ScanPerHost(request, httpClient, scanCtx)
│   └── AllowedInsertionPointTypes() InsertionPointTypeSet
│
└── PassiveModule
    ├── ScanPerRequest(request, scanCtx)
    ├── ScanPerHost(request, scanCtx)
    ├── Scope() PassiveScanScope
    └── (optional) Flusher: Flush(scanCtx)

const (
    ScanScopeInsertionPoint ScanScope = 1 << iota  // = 1
    ScanScopeRequest                                // = 2
    ScanScopeHost                                   // = 4
)

module.AllowedInsertionPointTypes().Contains(ip.Type())

Per item:
  1. Passive per-host   → sequential loop, no goroutines
  2. Passive per-request → sequential loop, no goroutines
  3. Active per-host     → parallel: all eligible modules concurrently
  4. Active per-request  → parallel: all eligible modules concurrently
  5. Active per-IP       → for each insertion point (serial):
                             all eligible modules concurrently

type ScanContext struct {
    DedupManager        *dedup.Manager
    RiskScoreUpdater    RiskScoreUpdater
    RequestUUIDResolver RequestUUIDResolver
    OASTProvider        OASTProvider
    MutationGen         MutationGenerator
    baselineCache       sync.Map  // "METHOD:host/path" → *BaselineEntry
}

type Flusher interface {
    Flush(scanCtx *ScanContext)
}

type ResultEvent struct {
    ModuleID         string                 `json:"template-id"`
    Info             Info                   `json:"info,inline"`
    Type             string                 `json:"type"`
    Host             string                 `json:"host,omitempty"`
    URL              string                 `json:"url,omitempty"`
    Matched          string                 `json:"matched-at,omitempty"`
    ExtractedResults []string               `json:"extracted-results,omitempty"`
    Request          string                 `json:"request,omitempty"`
    Response         string                 `json:"response,omitempty"`
    Metadata         map[string]interface{} `json:"meta,omitempty"`
    Timestamp        time.Time              `json:"timestamp"`
    // ...
}

Module returns []*ResultEvent
  │
  ▼
processResults(results, module)
  │
  for each result:
  │
  ├── moduleFindingAllowed(module.ID())
  │     Per-module finding cap check (MaxFindingsPerModule).
  │     When > 0, suppresses results after the limit is reached.
  │     Logs a one-time warning when a module hits its cap.
  │
  ├── assignModuleInfo(result, module)
  │     Set ModuleID, Info.Name, Description, Severity, Confidence
  │     Default Type = "http"
  │     Derive Matched from URL if empty
  │     Derive URL from request bytes if empty
  │     Fill Host from URL
  │
  └── emitResult(result)
        │
        ├── 1. Post-hooks: RunPostHooks(result)
        │      nil return → drop result (hook filtered it out)
        │
        ├── 2. Set results flag: e.results.Store(true)
        │
        ├── 3. Database save:
        │      Build temp HttpRequest from result.Request
        │      Look up requestUUIDs[requestHash] → recordUUID
        │      repo.SaveFinding(result, [recordUUID], scanUUID)
        │      Uses INSERT ON CONFLICT (finding_hash) DO NOTHING
        │
        ├── 4. OnResult callback → output writer
        │
        └── 5. Notifier.Send(result) → Telegram/Discord

type Writer interface {
    Close()
    Write(*ResultEvent) error
    WriteFileOnly(*ResultEvent) error
}

[› phase │] [moduleType] [moduleName] [severity] matched-at [extracted-results] [fuzz-param]

type fileWriter struct {
    file *os.File
    mu   sync.Mutex
}

type RecordWriter struct {
    repo    *Repository
    cfg     RecordWriterConfig   // BufferSize=4096, BatchSize=128, FlushInterval=50ms
    ch      chan writeRequest     // backpressure via channel capacity
}

Phase completes (KnownIssueScan or audit)
  │
  ▼
DeduplicateFindings(projectUUID)
  │
  ├── GROUP BY (module_id, severity, matched_at[0])
  │     ORDER BY created_at ASC → survivor = row_number 1
  │
  ├── For each group with duplicates:
  │     Merge duplicate request/response → survivor.AdditionalEvidence
  │     Cap at 10 evidence entries
  │
  ├── DELETE duplicate findings + junction rows
  │
  └── Print feedback: "grouped N findings into M"

vigolium scan -t https://example.com
         │
         ▼
    ┌────────────┐
    │  CLI Parse  │  pkg/cli/scan.go: runScanCmd()
    │  + Config   │  Load settings, resolve strategy/profile
    │  + DB Init  │  database.NewDB() → CreateSchema()
    └─────┬──────┘
          │
          ▼
    ┌────────────┐
    │   Runner    │  internal/runner/runner.go
    │  Build Infra│  HTTP client, scope matcher, rate limiter, hooks
    └─────┬──────┘
          │
          ▼
    ┌────────────────────────────────────────────────────────┐
    │              RunNativeScan() — 7 Phases                │
    │                                                        │
    │  [Heuristics] → [Harvest] → [Spider] → [SAST]         │
    │       → [Discovery/Ingest] → [KnownIssueScan] → [Dynamic Assess] │
    │                                                        │
    │  Phases 0-5: populate DB with HTTP records             │
    │  Phase 5-6: DeduplicateFindings() after each          │
    │  Phase 6: scan records with modules                    │
    └───────────────────────┬────────────────────────────────┘
                            │
                            ▼  (Phase 6 detail)
    ┌───────────────────────────────────────────────────┐
    │                    Executor                        │
    │                                                   │
    │  feedItems():                                     │
    │    source.Next() → static filter → scope check    │
    │    → host error check → send to worker channel    │
    │                                                   │
    │  worker() → processItem():                        │
    │    1. Baseline HTTP fetch (or use DB response)    │
    │    2. Traffic callback                            │
    │    3. Pre-hooks (JS transform/filter)             │
    │    4. Body size enforcement                       │
    │    5. Scope check + DB save                       │
    │    6. Eligibility pre-computation                 │
    │    7. Passive modules (sequential)                │
    │    8. Active modules (parallel, 3-way)            │
    │       └── per insertion point: all modules        │
    │                                                   │
    │  Post-processing:                                 │
    │    Flush passive modules (Flusher interface)      │
    │    Flush OAST service (grace period)              │
    └───────────────────────┬───────────────────────────┘
                            │
                            ▼
    ┌───────────────────────────────────────────────────┐
    │              Result Emission                       │
    │                                                   │
    │  Per-module finding cap (suppress after limit)     │
    │  assignModuleInfo() → emitResult():               │
    │    1. Post-hooks (JS transform/filter)            │
    │    2. SaveFinding() to DB (dedup via finding_hash │
    │       + evidence append on conflict)              │
    │    3. OnResult → StandardWriter.Write()           │
    │    4. Notifier.Send() → Telegram/Discord          │
    └───────────────────────┬───────────────────────────┘
                            │
                            ▼
    ┌───────────────────────────────────────────────────┐
    │                   Output                           │
    │                                                   │
    │  Console: colored severity + module + matched URL │
    │  JSON:    JSONL via jsoniter                      │
    │  HTML:    embedded ag-grid template               │
    │  File:    append-only JSONL with mutex            │
    └───────────────────────────────────────────────────┘