diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 69407f5..915ce14 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -1,24 +1,42 @@ name: Go +# Runs on every branch, not just master: the previous filter meant a feature +# branch got no CI at all until it opened a PR. on: push: - branches: [ master ] pull_request: - branches: [ master ] jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: - go-version: 1.19 + # Track the version declared in go.mod rather than a second, drifting + # copy of it here. + go-version-file: go.mod + cache: true + + - name: Check formatting + run: | + unformatted=$(gofmt -l .) + if [ -n "$unformatted" ]; then + echo "These files are not gofmt'd:" + echo "$unformatted" + exit 1 + fi + + - name: Vet + run: go vet ./... - name: Build run: go build -v ./... - + # The suite was never run in CI, so every detection bug it now guards + # against could have been reintroduced silently. + - name: Test + run: go test -race ./... diff --git a/.gitignore b/.gitignore index fabfe2c..0fb998a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ targets.txt test.go wappaGo.exe +wappaGo_report.html diff --git a/README.md b/README.md index 22c1606..04b6292 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,13 @@ Usage of wappaGo: Number of chromes threads in each main threads total = option.threads*option.chrome-threads (Default 5) (default 5) -follow-redirect Follow redirect to detect technologie + -headless + Run Chrome without a display. Off by default: headless fakes an 800x600 + screen and brands its own User-Agent. Use xvfb-run on a server instead + -jitter int + Max random delay in ms added before each request (0 = none) + -no-progress + Disable the stderr progress bar -port-timeout int Timeout during port scanning in ms (default 2000) -ports string @@ -55,16 +62,139 @@ Usage of wappaGo: -proxy string Use http proxy -report - Generate HTML report + Generate HTML report (in addition to the JSON on stdout) -resolvers string Use specifique resolver separated by comma + -rps float + Max HTTP requests per second per host to stay under rate-based WAF rules (0 = unlimited) -screenshot string path to screenshot if empty no screenshot -threads int Number of threads to start recon in same time (default 5) + -user-agent string + User-Agent sent by both the HTTP probe and Chrome (blank keeps the built-in browser UA) ``` +Results are written to stdout as one JSON object per line, so `-report` can be +combined with a redirect: + +```bash +cat domain.txt | ./wappaGo -report > results.json # JSON on stdout + wappaGo_report.html +``` + +The technology fingerprints are downloaded at startup from +[Serizao/tech](https://github.com/Serizao/tech), parsed into memory, and the +on-disk copy is deleted before the scan begins. + +## Blending in + +wappaGo presents one consistent browser identity to every host it scans. The +User-Agent is read from the installed Chrome at startup (with the +`HeadlessChrome` token rewritten) and installed together with its matching +Client Hints metadata, so `navigator.userAgent`, `navigator.userAgentData` and +the `Sec-CH-UA-*` headers all report the same build. The raw HTTP probe derives +its `User-Agent`, `Sec-CH-UA-*`, `Accept` and `Accept-Language` from that same +identity. `--enable-automation` is not passed, and WebGL is left enabled. + +**Chrome runs with a display by default.** Headless is available behind +`-headless`, but it costs two things that then have to be papered over: it reports +an 800x600 screen whatever the window size, so a 1400x900 viewport claims to sit +on a smaller display, and it brands its own User-Agent. With a real display both +are simply correct — measured `screen=1920x1200` and a clean UA — and no screen +override is issued at all. On a server, run it under a virtual display: + +```bash +xvfb-run -a --server-args="-screen 0 1920x1080x24" ./wappaGo < domain.txt +``` + +Without a display and without `-headless`, Chrome will not start; wappaGo says so +and points at both options rather than failing with a bare error. + +`RTCPeerConnection` leaks the host's real public address when `-proxy` is in use: +WebRTC negotiates over UDP and does not traverse an HTTP proxy, so a scanned page +can reach a public STUN server and read straight past the proxy. That is +deanonymisation rather than fingerprinting, so `-proxy` also enables +`--webrtc-ip-handling-policy=disable_non_proxied_udp`, which drops both the +server-reflexive and host candidates while leaving `RTCPeerConnection` defined — +closing the leak without substituting a "browser with no WebRTC" tell. Chrome +already hides the LAN address behind an mDNS name by default. The +`--force-webrtc-ip-handling-policy` spelling was measured to have no effect at all +on Chrome 150; only the older flag name works. + +That identity is capped to the Chrome version the TLS handshake below can +actually imitate. uTLS ships no profile for the newest Chrome releases, and +claiming a browser newer than the handshake produces is a contradiction +bot-protection vendors correlate: against a DataDome-protected origin, a Chrome +133 handshake paired with a User-Agent claiming Chrome 150 was refused 4/4 with +`403` and a captcha page, while the same handshake with a matching User-Agent +passed 2/2. The cost is that the browser reports an older version than its own +engine, which a site could catch by comparing the User-Agent against JS features +only present in the real build — a rarer and more expensive check than the +TLS/User-Agent correlation, and claiming *older* is the safe direction. + +A measurement harness records what the browser actually gives away: + +```bash +WAPPAGO_STEALTH_CHECK=1 go test ./cmd/ -run Stealth -v +``` + +It serves its own page locally, so the verdicts are reproducible, and it also +drives the hosted detector at `bot-detector.rebrowser.net`: + +```bash +WAPPAGO_STEALTH_REMOTE=1 go test ./cmd/ -run RemoteDetector -v +``` + +Two known limits it reports rather than hides: + +- **`window.outerWidth` / `outerHeight` are 0.** This comes from the tab chromedp + attaches to through `Target.createTarget`, which has no window of its own — + not from headless, and not from the metrics override. Measured: every + combination of headless/display, override/no override and + `Browser.setWindowBounds` reports 0, while raw Chrome opening a URL itself + reports the window size. Nothing reachable over CDP changes it, and the usual + workaround redefines the accessor from an injected script, which leaves a getter + that no longer reports `[native code]` — a broader tell than the one it fixes. +- **WebGL quality depends on the host.** With a GPU, Chrome reports the real + adapter, the most plausible fingerprint available. On a GPU-less server it falls + back to software rendering and says so (`SwiftShader`, `llvmpipe`, + `Microsoft Basic Render Driver`), and the canvas hash shifts with it — a stable + value that is a *known* value is still identifying. There is no CDP override for + WebGL, so masking it would mean patching JS. Measured against the hosted + detector, forcing SwiftShader changed no verdict at all: the same greens as a + real GPU. That is a negative result from one detector rather than proof of + safety — CreepJS and commercial vendors do fingerprint WebGL — so nothing is + patched for now and the harness logs a `NOTE:` when software rendering is in use. + +Canvas, WebGL and audio fingerprints are deliberately **not** randomised. The +goal is to look ordinary, not unique: a fingerprint that changes between two +reads is itself a signature of an anti-fingerprinting tool. The harness asserts +those surfaces are present and stable instead. + +The raw HTTP probe also handshakes like a browser. It runs before Chrome on every +target, so its TLS ClientHello is the first thing a host sees, and Go's +`crypto/tls` hello does not resemble one: different cipher list and ordering, and +no ALPN at all — which is what JA3/JA4 hashes key on. The probe now dials https +through [uTLS](https://github.com/refraction-networking/utls) with a Chrome +ClientHello, and carries the request over HTTP/2 when ALPN settles on it, so the +HTTP/2 `SETTINGS`/`WINDOW_UPDATE` fingerprint is a real h2 client's rather than a +downgrade to HTTP/1.1. Measured against a local server: + +| | cipher suites | first suite | ALPN offered | +|---|---|---|---| +| `crypto/tls` | 13 | `0xc02b` | *(none)* | +| uTLS Chrome | 16 | `0x5a5a` (GREASE) | `h2`, `http/1.1` | + +Two cases keep the standard transport on purpose: plain `http`, which has no +handshake to disguise, and `-proxy`, where the origin handshake happens inside a +`CONNECT` tunnel wappaGo does not drive. + +Still outstanding: `chromedp` issues `Runtime.enable` on every target, which is +a reliable CDP detection signal. The usual fix routes evaluation through an +isolated world, but wappaGo's `js` fingerprints read page globals +(`jQuery.fn.jquery`, `Stimulus.Application`) and so require the main world. + You can either use wappaGo from a file containing a list of domains ```bash cat domain.txt | ./wappaGo @@ -94,16 +224,30 @@ type WrapperOptions struct { ChromeTimeout int ChromeThreads int Proxy string + UserAgent string + Rps float64 + Jitter int + Headless bool } ``` +Any field left at its zero value falls back to the same default as the +corresponding CLI flag (`Ports: "80,443"`, `Threads: 5`, `ChromeThreads: 5`, +`Porttimeout: 2000`), so you only have to set what you want to change. + ## Example +Both entry points return an `error`, which is non-nil when the setup itself +failed — most often because the technology fingerprints could not be downloaded. +Ignoring it means scanning every host with an empty database and reporting zero +technologies everywhere. + ```go package main import ( "fmt" + "log" "github.com/EasyRecon/wappaGo/structure" "github.com/EasyRecon/wappaGo/wrapper" @@ -117,26 +261,35 @@ func main() { Screenshot: "screenshots", } - // Async mode + // Sync mode: blocks, then returns everything. + results, err := wrapper.StartReconSync(input, options) + if err != nil { + log.Fatal(err) + } + for _, result := range results { + fmt.Println(result) + } +} +``` +```go + // Async mode: results are streamed on the channel, which wrapper closes + // when the scan is done. StartReconAsync blocks until then, so run the + // consumer in its own goroutine and wait for it to drain. results := make(chan structure.Data) + done := make(chan struct{}) go func() { + defer close(done) for result := range results { fmt.Println(result) } }() - wrapper.StartReconAsync(input, options, results) - - // Sync mode - - results := wrapper.StartReconSync(input, options) - - for _, result := range results { - fmt.Println(result) + if err := wrapper.StartReconAsync(input, options, results); err != nil { + log.Fatal(err) } -} + <-done ``` For each url, you will receive a structure.Data which contains all the information about the target. diff --git a/analyze/analyze_cert.go b/analyze/analyze_cert.go index ffa04b0..72b4230 100644 --- a/analyze/analyze_cert.go +++ b/analyze/analyze_cert.go @@ -4,10 +4,39 @@ import ( "github.com/EasyRecon/wappaGo/technologies" ) -func (a *Analyze)analyze_cert_main(technoName string,key string){ - if a.CertIssuer == a.ResultGlobal[technoName].(map[string]interface{})[key].(string) { +// analyze_cert_main matches the TLS issuer CN. The fingerprint value used to be +// asserted to string unconditionally, panicking on any technology that ships +// certIssuer as an array. +func (a *Analyze) analyze_cert_main(technoName string, key string) { + entry, ok := a.ResultGlobal[technoName].(map[string]interface{}) + if !ok || a.CertIssuer == "" { + return + } + for _, issuer := range namesOfValue(entry[key]) { + if a.CertIssuer != issuer { + continue + } technoTemp := a.NewTechno(technoName) a.Technos = append(a.Technos, technoTemp) a.Technos = technologies.CheckRequired(technoTemp.Name, a.ResultGlobal, a.Technos) + return + } +} + +// namesOfValue normalises a fingerprint value that may be a bare string or an +// array of strings into a slice. +func namesOfValue(v interface{}) []string { + switch t := v.(type) { + case string: + return []string{t} + case []interface{}: + out := make([]string, 0, len(t)) + for _, item := range t { + if s, ok := item.(string); ok { + out = append(out, s) + } + } + return out } -} \ No newline at end of file + return nil +} diff --git a/analyze/analyze_cookies.go b/analyze/analyze_cookies.go index b2c87ca..6febc3c 100644 --- a/analyze/analyze_cookies.go +++ b/analyze/analyze_cookies.go @@ -4,7 +4,7 @@ import ( "github.com/EasyRecon/wappaGo/technologies" ) -func (a *Analyze) analyze_cookies_main(technoName string,key string){ +func (a *Analyze) analyze_cookies_main(technoName string, key string) { for cookieTechno, _ := range a.ResultGlobal[technoName].(map[string]interface{})[key].(map[string]interface{}) { for _, cookie := range a.CookiesList { if cookieTechno == cookie.Name { @@ -14,4 +14,4 @@ func (a *Analyze) analyze_cookies_main(technoName string,key string){ } } } -} \ No newline at end of file +} diff --git a/analyze/analyze_dns.go b/analyze/analyze_dns.go index cfff180..6664e79 100644 --- a/analyze/analyze_dns.go +++ b/analyze/analyze_dns.go @@ -1,54 +1,82 @@ package analyze - -import( - - "fmt" +import ( "github.com/EasyRecon/wappaGo/technologies" - "regexp" + "github.com/projectdiscovery/retryabledns" ) -func (a *Analyze) analyze_dns_main(technoName string,key string){ - for key,value := range a.ResultGlobal[technoName].(map[string]interface{})[key].(map[string]interface{}) { + +func (a *Analyze) analyze_dns_main(technoName string, key string) { + // DnsData is nil whenever resolution failed — fastdialer returns (nil, err) + // on NXDOMAIN or a resolver hiccup, and launchChrome assigned it anyway. + // Dereferencing it panicked, and the per-port recover() then dropped the + // whole target from the output without a word. + if a.DnsData == nil { + return + } + entry, ok := a.ResultGlobal[technoName].(map[string]interface{}) + if !ok { + return + } + records, ok := entry[key].(map[string]interface{}) + if !ok { + return + } + for key, value := range records { var resultDNS []string switch key { - case "TXT": - resultDNS = a.DnsData.TXT - case "SOA": - resultDNS = a.DnsData.SOA - case "NS": - resultDNS = a.DnsData.NS - case "CNAME": - resultDNS = a.DnsData.CNAME - case "MX": - resultDNS = a.DnsData.MX + case "TXT": + resultDNS = a.DnsData.TXT + case "SOA": + resultDNS = soaStrings(a.DnsData.SOA) + case "NS": + resultDNS = a.DnsData.NS + case "CNAME": + resultDNS = a.DnsData.CNAME + case "MX": + resultDNS = a.DnsData.MX } - if fmt.Sprintf("%T",value) == "string" { - found := a.analyze_dns_regex(value.(string),resultDNS) - if found { + for _, regex := range namesOfValue(value) { + if a.analyze_dns_regex(regex, resultDNS) { technoTemp := a.NewTechno(technoName) a.Technos = append(a.Technos, technoTemp) a.Technos = technologies.CheckRequired(technoTemp.Name, a.ResultGlobal, a.Technos) } - } else { - for _,regex:= range value.([]interface{}) { - found := a.analyze_dns_regex(regex.(string),resultDNS) - if found { - technoTemp := a.NewTechno(technoName) - a.Technos = append(a.Technos, technoTemp) - a.Technos = technologies.CheckRequired(technoTemp.Name, a.ResultGlobal, a.Technos) - } - } } - } + } } -func (a *Analyze) analyze_dns_regex(regex string,resultsDNS []string)(bool){ +// soaStrings flattens SOA records into the strings the fingerprints actually +// match against. +// +// retryabledns used to hand back []string here and now returns a parsed struct, +// so the fields have to be picked deliberately. All 43 SOA patterns in the +// database target a hostname: either the primary nameserver +// (`\.cloudflare\.com`, `\.domaincontrol\.com`, `\.vercel-dns\.com`) or the +// responsible mailbox (`tech\.sakura\.ad\.jp`). The numeric serial/refresh/TTL +// fields are never matched, so they are left out rather than concatenated into +// noise a pattern could accidentally hit. +func soaStrings(records []retryabledns.SOA) []string { + out := make([]string, 0, len(records)*2) + for _, r := range records { + if r.NS != "" { + out = append(out, r.NS) + } + if r.Mbox != "" { + out = append(out, r.Mbox) + } + } + return out +} - for _,resultDNS:= range resultsDNS { - findregex, _ := regexp.MatchString("(?i)"+regex, resultDNS) - if findregex { +func (a *Analyze) analyze_dns_regex(regex string, resultsDNS []string) bool { + re, ok := compileCI(regex) + if !ok { + return false + } + for _, resultDNS := range resultsDNS { + if re.MatchString(resultDNS) { return true } } return false -} \ No newline at end of file +} diff --git a/analyze/analyze_dom.go b/analyze/analyze_dom.go index a915ffa..54dbb87 100644 --- a/analyze/analyze_dom.go +++ b/analyze/analyze_dom.go @@ -1,125 +1,148 @@ package analyze - import ( - "fmt" - "regexp" - "strconv" "strings" + "github.com/EasyRecon/wappaGo/technologies" "github.com/PuerkitoBio/goquery" "github.com/chromedp/chromedp" ) +// analyze_dom_main matches the fingerprint's "dom" family. The value is either +// a selector (string), a list of selectors, or a map of selector -> spec where +// the spec describes an attribute, a property, or the element's text. +func (a *Analyze) analyze_dom_main(technoName string, key string, doc *goquery.Document) { + if doc == nil { + return + } + entry, ok := a.ResultGlobal[technoName].(map[string]interface{}) + if !ok { + return + } + + switch spec := entry[key].(type) { + case string: + a.domSelectorPresent(technoName, spec, doc) + case []interface{}: + for _, selector := range spec { + if s, ok := selector.(string); ok { + a.domSelectorPresent(technoName, s, doc) + } + } -func (a *Analyze)analyze_dom_main(technoName string,key string, doc *goquery.Document){ - if fmt.Sprintf("%T", a.ResultGlobal[technoName].(map[string]interface{})[key]) == "string" { - doc.Find(a.ResultGlobal[technoName].(map[string]interface{})[key].(string)).Each(func(i int, s *goquery.Selection) { - technoTemp := a.NewTechno(technoName) - a.Technos = append(a.Technos, technoTemp) - a.Technos = technologies.CheckRequired(technoTemp.Name, a.ResultGlobal, a.Technos) - }) - } else if fmt.Sprintf("%T", a.ResultGlobal[technoName].(map[string]interface{})[key]) == "map[string]interface {}" { - for domKey, domArray := range a.ResultGlobal[technoName].(map[string]interface{})[key].(map[string]interface{}) { - for domKeyElement, domElement := range domArray.(map[string]interface{}) { - if fmt.Sprintf("%T", domElement) == "string" { - doc.Find(domKey).Each(func(i int, s *goquery.Selection) { - a.analyze_dom_valued(technoName,domElement) + case map[string]interface{}: + for domKey, domArray := range spec { + domSpec, ok := domArray.(map[string]interface{}) + if !ok { + continue + } + for domKeyElement, domElement := range domSpec { + switch element := domElement.(type) { + case string: + // Match the selected element's own text. This used to + // re-scan the ENTIRE page body once per matched element, so + // a bare "div" selector turned a single fingerprint into + // thousands of full-document regex scans (~10 s on a normal + // page). + doc.Find(domKey).EachWithBreak(func(i int, s *goquery.Selection) bool { + return !a.analyze_dom_valued(technoName, element, s) }) - } else if fmt.Sprintf("%T", domElement) == "map[string]interface {}" { - for domKeyElement2, domElement2 := range domElement.(map[string]interface{}) { + case map[string]interface{}: + for domKeyElement2, domElement2 := range element { if domKeyElement == "attributes" { - doc.Find(domKey).Each(func(i int, s *goquery.Selection) { - a.analyze_dom_attribute(technoName,domKeyElement2,domElement2,s) + doc.Find(domKey).EachWithBreak(func(i int, s *goquery.Selection) bool { + return !a.analyze_dom_attribute(technoName, domKeyElement2, domElement2, s) }) } else { - a.analyze_dom_exist(technoName,domKeyElement2,domKey) + a.analyze_dom_exist(technoName, domKeyElement2, domKey) } } } } } - } else { - for _, domArray := range a.ResultGlobal[technoName].(map[string]interface{})[key].([]interface{}) { - doc.Find(domArray.(string)).Each(func(i int, s *goquery.Selection) { - technoTemp := a.NewTechno(technoName) - a.Technos = append(a.Technos, technoTemp) - a.Technos = technologies.CheckRequired(technoTemp.Name, a.ResultGlobal, a.Technos) - }) - } - } + } } -func (a *Analyze) analyze_dom_exist(technoName string,domKeyElement2 string,domKey string ){ + +// domSelectorPresent records the technology when the selector matches anything. +func (a *Analyze) domSelectorPresent(technoName string, selector string, doc *goquery.Document) { + if doc.Find(selector).Length() == 0 { + return + } + technoTemp := a.NewTechno(technoName) + a.Technos = append(a.Technos, technoTemp) + a.Technos = technologies.CheckRequired(technoTemp.Name, a.ResultGlobal, a.Technos) +} + +func (a *Analyze) analyze_dom_exist(technoName string, domKeyElement2 string, domKey string) { var res interface{} + if a.Ctx == nil { + return + } chromedp.Evaluate("(()=>{a=false;document.querySelectorAll('"+domKey+"').forEach(element=>{if(element."+domKeyElement2+"!=undefined){a=true}});return a})()", &res).Do(a.Ctx) - //fmt.Println(res, "(()=>{a=false;document.querySelectorAll('"+domKey+"').forEach(element=>{if(element."+domKeyElement2+"!=undefined){a=true}});return a})()") if res == true { - technoTemp := a.NewTechno(technoName) + technoTemp := a.NewTechno(technoName) a.Technos = append(a.Technos, technoTemp) a.Technos = technologies.CheckRequired(technoTemp.Name, a.ResultGlobal, a.Technos) } } - -func (a *Analyze) analyze_dom_attribute(technoName string,domKeyElement2 string,domElement2 interface{},s *goquery.Selection){ - dommAttr, _ := s.Attr(domKeyElement2) - if dommAttr != "" { - if domKeyElement2 != "" { - regex := strings.Split(domElement2.(string), "\\;") - findRegex, _ := regexp.MatchString("(?i)"+regex[0], dommAttr) - if findRegex { - technoTemp := a.NewTechno(technoName) - compiledregex := regexp.MustCompile("(?i)" + regex[0]) - regexGroup := compiledregex.FindAllStringSubmatch(dommAttr, -1) - - if len(regex) > 1 && strings.HasPrefix(regex[1], "version") { - versionGrp := strings.Split(regex[1], "\\") - - if len(versionGrp) > 1 { - offset, _ := strconv.Atoi(versionGrp[1]) - //fmt.Println(versionGrp) - technoTemp.Version = regexGroup[0][offset] - } - } - a.Technos = append(a.Technos, technoTemp) - a.Technos = technologies.CheckRequired(technoTemp.Name, a.ResultGlobal, a.Technos) - } - } else { - technoTemp := a.NewTechno(technoName) - a.Technos = append(a.Technos, technoTemp) - a.Technos = technologies.CheckRequired(technoTemp.Name, a.ResultGlobal, a.Technos) - } +func (a *Analyze) analyze_dom_attribute(technoName string, attrName string, pattern interface{}, s *goquery.Selection) bool { + if attrName == "" { + return false + } + attrValue, exists := s.Attr(attrName) + if !exists || attrValue == "" { + return false + } + patternStr, ok := pattern.(string) + if !ok { + return false + } + // An empty pattern means the attribute only has to be present. + if patternStr == "" { + technoTemp := a.NewTechno(technoName) + a.Technos = append(a.Technos, technoTemp) + a.Technos = technologies.CheckRequired(technoTemp.Name, a.ResultGlobal, a.Technos) + return true + } + regex := strings.Split(patternStr, "\\;") + re, ok := compileCI(regex[0]) + if !ok { + return false } + groups := re.FindStringSubmatch(attrValue) + if groups == nil { + return false + } + technoTemp := a.NewTechno(technoName) + technoTemp.Version = versionFromMarker(regex, [][]string{groups}) + a.Technos = append(a.Technos, technoTemp) + a.Technos = technologies.CheckRequired(technoTemp.Name, a.ResultGlobal, a.Technos) + return true } - -func (a *Analyze) analyze_dom_valued(technoName string,domElement interface{}){ - if domElement == "" { +// analyze_dom_valued matches a dom spec's text/value pattern against the +// selected element. +func (a *Analyze) analyze_dom_valued(technoName string, pattern string, s *goquery.Selection) bool { + if pattern == "" { technoTemp := a.NewTechno(technoName) a.Technos = append(a.Technos, technoTemp) a.Technos = technologies.CheckRequired(technoTemp.Name, a.ResultGlobal, a.Technos) - } else { - regex := strings.Split(domElement.(string), "\\;") - - findregex, _ := regexp.MatchString("(?i)"+regex[0], a.Body) - if findregex { - //fmt.Println(technoName) - technoTemp := a.NewTechno(technoName) - compiledregex := regexp.MustCompile("(?i)" + regex[0]) - regexGroup := compiledregex.FindAllStringSubmatch(a.Body, -1) - - if len(regex) > 1 && strings.HasPrefix(regex[1], "version") { - versionGrp := strings.Split(regex[1], "\\") - - if len(versionGrp) > 1 { - offset, _ := strconv.Atoi(versionGrp[1]) - //fmt.Println(regexGroup[0][offset]) - technoTemp.Version = regexGroup[0][offset] - } - } - a.Technos = append(a.Technos, technoTemp) - a.Technos = technologies.CheckRequired(technoTemp.Name, a.ResultGlobal, a.Technos) - } + return true + } + regex := strings.Split(pattern, "\\;") + re, ok := compileCI(regex[0]) + if !ok { + return false + } + groups := re.FindStringSubmatch(s.Text()) + if groups == nil { + return false } + technoTemp := a.NewTechno(technoName) + technoTemp.Version = versionFromMarker(regex, [][]string{groups}) + a.Technos = append(a.Technos, technoTemp) + a.Technos = technologies.CheckRequired(technoTemp.Name, a.ResultGlobal, a.Technos) + return true } diff --git a/analyze/analyze_golden_test.go b/analyze/analyze_golden_test.go new file mode 100644 index 0000000..a8188ac --- /dev/null +++ b/analyze/analyze_golden_test.go @@ -0,0 +1,632 @@ +package analyze + +import ( + "sort" + "testing" + + "github.com/EasyRecon/wappaGo/structure" + "github.com/EasyRecon/wappaGo/technologies" + "github.com/projectdiscovery/retryabledns" +) + +// fixtureResultGlobal mimics the shape of the merged Wappalyzer fingerprint DB +// for a handful of technologies that can be detected without a live browser: +// only the html / headers / scriptSrc / meta signal sources are used, plus an +// "implies" edge. This lets Run() execute end-to-end (goquery parse + the +// regex matchers + CheckRequired + DedupTechno) with no chromedp context. +// +// It is the regression guard for the regex-precompilation refactor: the exact +// set of detected technologies and their versions must not change. +func fixtureResultGlobal() map[string]interface{} { + return map[string]interface{}{ + "Nginx": map[string]interface{}{ + "cpe": "cpe:2.3:a:nginx:nginx:*:*:*:*:*:*:*:*", + "headers": map[string]interface{}{ + "Server": "nginx(?:/([\\d.]+))?\\;version:\\1", + }, + "implies": "Reverse proxy", + }, + "Reverse proxy": map[string]interface{}{}, // referenced only via implies + "jQuery": map[string]interface{}{ + "scriptSrc": "jquery-([\\d.]+)\\.js\\;version:\\1", + }, + "WordPress": map[string]interface{}{ + "meta": map[string]interface{}{ + "generator": "WordPress ?([\\d.]+)?\\;version:\\1", + }, + "html": "wp-content", + }, + } +} + +func technologyVersions(techs []structure.Technologie) map[string]string { + out := map[string]string{} + for _, te := range techs { + // keep the first non-empty version seen for a name + if prev, ok := out[te.Name]; ok && prev != "" { + continue + } + out[te.Name] = te.Version + } + return out +} + +func technologyNames(techs []structure.Technologie) []string { + var out []string + for _, te := range techs { + out = append(out, te.Name+"@"+te.Version) + } + sort.Strings(out) + return out +} + +func TestRunGoldenDetection(t *testing.T) { + a := Analyze{ + ResultGlobal: fixtureResultGlobal(), + Body: `
` + + `` + + `hello world`, + Resp: structure.Response{ + Headers: map[string][]string{"Server": {"nginx/1.21.0"}}, + }, + SrcList: []string{"https://code.jquery.com/jquery-3.6.0.js"}, + } + + got := technologies.DedupTechno(a.Run()) + gotVersions := technologyVersions(got) + + want := map[string]string{ + "Nginx": "1.21.0", // version from header capture group + "Reverse proxy": "", // implied by Nginx, no version + "jQuery": "3.6.0", // version from scriptSrc capture group + "WordPress": "6.2", // version from meta generator capture group + } + + if len(gotVersions) != len(want) { + t.Fatalf("detected %d technologies, want %d: got %v", len(gotVersions), len(want), technologyNames(got)) + } + for name, ver := range want { + if gotVersions[name] != ver { + t.Errorf("technology %q version = %q, want %q (all: %v)", name, gotVersions[name], ver, technologyNames(got)) + } + } + + // The CPE attached to a fingerprint must survive detection. + for _, te := range got { + if te.Name == "Nginx" && te.Cpe == "" { + t.Errorf("Nginx detected without its CPE") + } + } +} + +// TestIconIsCarriedThrough pins the icon field: it is copied from the database +// for detected and implied technologies alike, and omitted (rather than +// invented) for fingerprints that declare none. +func TestIconIsCarriedThrough(t *testing.T) { + rg := map[string]interface{}{ + "Nginx": map[string]interface{}{ + "icon": "Nginx.svg", + "headers": map[string]interface{}{"Server": "nginx"}, + "implies": "Reverse proxy", + }, + "Reverse proxy": map[string]interface{}{"icon": "ReverseProxy.svg"}, + "NoIcon": map[string]interface{}{ + "html": "no-icon-marker", + }, + } + a := Analyze{ + ResultGlobal: rg, + Body: `no-icon-marker`, + Resp: structure.Response{Headers: map[string][]string{"Server": {"nginx"}}}, + } + + icons := map[string]string{} + for _, te := range technologies.DedupTechno(a.Run()) { + icons[te.Name] = te.Icon + } + for name, want := range map[string]string{ + "Nginx": "Nginx.svg", + "Reverse proxy": "ReverseProxy.svg", // reached through implies + "NoIcon": "", + } { + got, ok := icons[name] + if !ok { + t.Errorf("%q not detected (got %v)", name, icons) + continue + } + if got != want { + t.Errorf("%q icon = %q, want %q", name, got, want) + } + } +} + +// TestRunGoldenURLAndDNS covers the matchers converted from regexp.MatchString +// to the cached compileCI (url, dns) and pins the panic-safety contract: a +// PCRE-only pattern that Go's RE2 engine rejects must be skipped silently +// rather than aborting the scan. +func TestRunGoldenURLAndDNS(t *testing.T) { + rg := map[string]interface{}{ + "AdminPanel": map[string]interface{}{ + "url": "example\\.com/admin", + }, + "GoogleWorkspace": map[string]interface{}{ + "dns": map[string]interface{}{ + "TXT": "include:_spf\\.google\\.com", + }, + }, + "BrokenPCRE": map[string]interface{}{ + // Go RE2 rejects look-ahead; must be skipped, not matched, not panic. + "html": "foo(?=bar)", + }, + } + a := Analyze{ + ResultGlobal: rg, + Body: `foobar example`, + Hote: structure.Host{Location: "https://example.com/admin"}, + DnsData: &retryabledns.DNSData{TXT: []string{"v=spf1 include:_spf.google.com ~all"}}, + } + + got := technologies.DedupTechno(a.Run()) + gotVersions := technologyVersions(got) + + want := []string{"AdminPanel", "GoogleWorkspace"} + if len(gotVersions) != len(want) { + t.Fatalf("detected %v, want exactly %v", technologyNames(got), want) + } + for _, name := range want { + if _, ok := gotVersions[name]; !ok { + t.Errorf("missing detection %q (got %v)", name, technologyNames(got)) + } + } + if _, ok := gotVersions["BrokenPCRE"]; ok { + t.Errorf("BrokenPCRE matched; an invalid RE2 pattern must be skipped") + } +} + +// TestImpliesMissingTechnoNoPanic pins the crash-containment hardening: an +// "implies" edge pointing at a technology absent from the DB must add that +// technology without panicking (the original AddTechno/NewTechno asserted the +// missing map entry and crashed the whole scan). +func TestImpliesMissingTechnoNoPanic(t *testing.T) { + rg := map[string]interface{}{ + "Thing": map[string]interface{}{ + "html": "thing-marker", + "implies": "GhostTech", // intentionally not present in the DB + }, + } + a := Analyze{ResultGlobal: rg, Body: `thing-marker`} + + got := technologies.DedupTechno(a.Run()) // must not panic + names := technologyVersions(got) + if _, ok := names["Thing"]; !ok { + t.Errorf("Thing not detected (got %v)", technologyNames(got)) + } + if _, ok := names["GhostTech"]; !ok { + t.Errorf("implied GhostTech not added (got %v)", technologyNames(got)) + } +} + +// TestRequiresGate pins the requires-vs-implies behaviour change: a technology +// with a "requires" precondition is kept only when the required technology is +// independently detected, and is dropped otherwise (the old code added the +// required techno unconditionally, producing false positives). +func TestRequiresGate(t *testing.T) { + rg := map[string]interface{}{ + "WPPlugin": map[string]interface{}{ + "html": "wp-plugin-marker", + "requires": "WordPress", + }, + "WordPress": map[string]interface{}{ + "meta": map[string]interface{}{"generator": "WordPress"}, + }, + } + + // WordPress present -> the plugin survives the gate. + a := Analyze{ + ResultGlobal: rg, + Body: `` + + `wp-plugin-marker`, + } + got := technologies.FilterRequired(technologies.DedupTechno(a.Run()), rg) + names := technologyVersions(got) + for _, want := range []string{"WordPress", "WPPlugin"} { + if _, ok := names[want]; !ok { + t.Errorf("expected %q kept when WordPress is present (got %v)", want, technologyNames(got)) + } + } + + // WordPress absent -> the plugin is dropped, and nothing is fabricated. + b := Analyze{ + ResultGlobal: rg, + Body: `wp-plugin-marker`, + } + got2 := technologies.FilterRequired(technologies.DedupTechno(b.Run()), rg) + names2 := technologyVersions(got2) + if _, ok := names2["WPPlugin"]; ok { + t.Errorf("WPPlugin should be dropped when required WordPress is absent (got %v)", technologyNames(got2)) + } + if _, ok := names2["WordPress"]; ok { + t.Errorf("WordPress must not be fabricated by a requires edge (got %v)", technologyNames(got2)) + } +} + +// TestHeaderKeyCasing pins the header-lookup fix. Wappalyzer keys its header +// fingerprints however upstream wrote them ("x-powered-by", "server"), while +// a.Resp.Headers is an http.Header clone and is therefore canonical +// ("X-Powered-By", "Server"). Indexing the fingerprint with the response's key +// silently disabled 43% of the header database, so every one of these forms +// must now match. +func TestHeaderKeyCasing(t *testing.T) { + rg := map[string]interface{}{ + // lowercase key + version marker, as Vercel/Ktor ship it + "LowerWithVersion": map[string]interface{}{ + "headers": map[string]interface{}{"x-engine": `Ktor/([\d.]+)\;version:\1`}, + }, + // lowercase key, presence only + "LowerPresence": map[string]interface{}{ + "headers": map[string]interface{}{"x-vercel-cache": ""}, + }, + // mixed case that canonicalises differently ("X-Aspnet-Version") + "MixedCase": map[string]interface{}{ + "headers": map[string]interface{}{"X-AspNet-Version": `([\d.]+)`}, + }, + // already canonical: the one spelling that worked before + "Canonical": map[string]interface{}{ + "headers": map[string]interface{}{"Server": `nginx/([\d.]+)\;version:\1`}, + }, + } + a := Analyze{ + ResultGlobal: rg, + Resp: structure.Response{ + Headers: map[string][]string{ + "X-Engine": {"Ktor/2.3.4"}, + "X-Vercel-Cache": {"HIT"}, + "X-Aspnet-Version": {"4.0.30319"}, + "Server": {"nginx/1.21.0"}, + "X-Unrelated-Header": {"noise"}, + }, + }, + } + + got := technologyVersions(technologies.DedupTechno(a.Run())) + for name, wantVer := range map[string]string{ + "LowerWithVersion": "2.3.4", + "LowerPresence": "", + "MixedCase": "", + "Canonical": "1.21.0", + } { + ver, ok := got[name] + if !ok { + t.Errorf("%q not detected — header lookup missed its fingerprint key (got %v)", name, got) + continue + } + if ver != wantVer { + t.Errorf("%q version = %q, want %q", name, ver, wantVer) + } + } + if len(got) != 4 { + t.Errorf("detected %d technologies, want 4: %v", len(got), got) + } +} + +// TestHeaderRepeatedValues covers the second half of the same fix: a header can +// repeat, and only values[0] used to be examined (which also panicked on an +// empty value slice). +func TestHeaderRepeatedValues(t *testing.T) { + rg := map[string]interface{}{ + "Varnish": map[string]interface{}{ + "headers": map[string]interface{}{"via": `varnish \(Varnish/([\d.]+)\)\;version:\1`}, + }, + } + a := Analyze{ + ResultGlobal: rg, + Resp: structure.Response{ + Headers: map[string][]string{ + "Via": {"1.1 google", "1.1 varnish (Varnish/7.1)"}, + "Empty": {}, + "Server": {"nginx"}, + }, + }, + } + + got := technologyVersions(technologies.DedupTechno(a.Run())) // must not panic + if got["Varnish"] != "7.1" { + t.Errorf("Varnish version = %q, want %q (only the first header value was inspected)", got["Varnish"], "7.1") + } +} + +// TestFinalHeadersPreferredOverProbe pins the redirect fix on the header side: +// with -follow-redirect off the raw probe stops at the 30x, so its headers +// describe the redirector, not the page Chrome renders and analyses. A host +// redirecting http->https therefore reported none of its header-based +// technologies. CDP also newline-joins repeated headers, which must be split +// back apart. +func TestFinalHeadersPreferredOverProbe(t *testing.T) { + rg := map[string]interface{}{ + "Nginx": map[string]interface{}{"headers": map[string]interface{}{"Server": `nginx/([\d.]+)\;version:\1`}}, + "Varnish": map[string]interface{}{"headers": map[string]interface{}{"via": `(\d+(?:\.\d+)?)\s+varnish\;version:\1`}}, + } + + a := Analyze{ + ResultGlobal: rg, + Body: "", + // The probe only saw the redirector: no Server, no Via. + Resp: structure.Response{Headers: map[string][]string{"Location": {"https://example.com/"}}}, + } + // What Chrome reported for the final document, in CDP's shape. + a.SetFinalHeaders("https://example.com/", map[string]interface{}{ + "server": "nginx/1.25.3", + "via": "1.1 google\n1.1 varnish", + }) + + got := technologyVersions(technologies.DedupTechno(a.Run())) + if got["Nginx"] != "1.25.3" { + t.Errorf("Nginx = %q, want %q — the final page's headers must be used", got["Nginx"], "1.25.3") + } + if got["Varnish"] != "1.1" { + t.Errorf("Varnish = %q, want %q — repeated headers must be split on newline", got["Varnish"], "1.1") + } + + // With no Chrome headers at all, the probe's are still used. + b := Analyze{ + ResultGlobal: rg, + Body: "", + Resp: structure.Response{Headers: map[string][]string{"Server": {"nginx/1.21.0"}}}, + } + if got := technologyVersions(technologies.DedupTechno(b.Run())); got["Nginx"] != "1.21.0" { + t.Errorf("fallback to probe headers broken: %v", got) + } +} + +// TestHeadersAreMergedNotReplaced pins the union: Chrome never reports Alt-Svc +// (it consumes it), and that header is the only signal for HTTP/3, so preferring +// Chrome's set outright silently dropped it. Chrome must still win on headers +// present in both, since its view is the page that was actually analysed. +func TestHeadersAreMergedNotReplaced(t *testing.T) { + rg := map[string]interface{}{ + "HTTP/3": map[string]interface{}{"headers": map[string]interface{}{"Alt-Svc": "h3"}}, + "Nginx": map[string]interface{}{"headers": map[string]interface{}{"Server": `nginx/([\d.]+)\;version:\1`}}, + } + a := Analyze{ + ResultGlobal: rg, + Body: "", + Resp: structure.Response{Headers: map[string][]string{ + "Alt-Svc": {`h3=":443"; ma=86400`}, // only the probe sees this + "Server": {"nginx/1.21.0"}, // stale: this was the redirector + }}, + } + a.SetFinalHeaders("https://example.com/", map[string]interface{}{ + "server": "nginx/1.25.3", // the page actually analysed + }) + + got := technologyVersions(technologies.DedupTechno(a.Run())) + if _, ok := got["HTTP/3"]; !ok { + t.Errorf("probe-only header lost: %v", got) + } + if got["Nginx"] != "1.25.3" { + t.Errorf("Nginx = %q, want %q — Chrome must win where both have the header", got["Nginx"], "1.25.3") + } +} + +// TestUrlMatchesScannedURL pins the url-family fix: patterns are matched +// against the URL that was scanned, not against the Location header, and no +// longer require the host to have redirected. +func TestUrlMatchesScannedURL(t *testing.T) { + rg := map[string]interface{}{ + "AdminPanel": map[string]interface{}{"url": `example\.com/admin`}, + "Ionos": map[string]interface{}{"url": `^https?://[^/]+\.ionos\.(space|live)`}, + } + + // No redirect at all: both fingerprints used to be skipped outright. + a := Analyze{ResultGlobal: rg, Url: "https://example.com/admin", Body: ""} + if got := technologyVersions(technologies.DedupTechno(a.Run())); len(got) != 1 || got["AdminPanel"] != "" { + t.Errorf("AdminPanel not detected on a non-redirecting host: %v", got) + } + + b := Analyze{ResultGlobal: rg, Url: "https://foo.ionos.space", Body: ""} + if got := technologyVersions(technologies.DedupTechno(b.Run())); len(got) != 1 || got["Ionos"] != "" { + t.Errorf("Ionos not detected: %v", got) + } + + // The redirect target is still considered. + c := Analyze{ + ResultGlobal: rg, + Url: "https://other.test", + Hote: structure.Host{Location: "https://example.com/admin"}, + Body: "", + } + if got := technologyVersions(technologies.DedupTechno(c.Run())); len(got) != 1 { + t.Errorf("redirect target ignored: %v", got) + } + + // Neither matches -> nothing detected. + d := Analyze{ResultGlobal: rg, Url: "https://unrelated.test/", Body: ""} + if got := technologyVersions(technologies.DedupTechno(d.Run())); len(got) != 0 { + t.Errorf("false positive: %v", got) + } +} + +// TestMetaMatchesPropertyAttribute pins the meta fix: Wappalyzer keys meta +// fingerprints on either attribute, and only meta[name=] used to be searched, so +// every og:/twitter:/fb: fingerprint was unreachable. +func TestMetaMatchesPropertyAttribute(t *testing.T) { + rg := map[string]interface{}{ + "OpenGraph": map[string]interface{}{ + "meta": map[string]interface{}{"og:site_name": ""}, + }, + "Generator": map[string]interface{}{ + "meta": map[string]interface{}{"generator": `Hugo ([\d.]+)\;version:\1`}, + }, + } + a := Analyze{ + ResultGlobal: rg, + Body: `` + + `` + + `` + + ``, + } + + got := technologyVersions(technologies.DedupTechno(a.Run())) + if _, ok := got["OpenGraph"]; !ok { + t.Errorf("meta[property] fingerprint not matched: %v", got) + } + if got["Generator"] != "0.157.0" { + t.Errorf("meta[name] version = %q, want %q", got["Generator"], "0.157.0") + } +} + +// TestTextMatchesRenderedTextOnly pins the html-vs-text split: a "text" pattern +// must match the page's rendered text, not its markup, or it fires on attribute +// values and URLs that are invisible to a visitor. +func TestTextMatchesRenderedTextOnly(t *testing.T) { + rg := map[string]interface{}{ + "MarkupOnly": map[string]interface{}{"text": "data-vendor"}, + "RealText": map[string]interface{}{"text": "Powered by Acme"}, + "HtmlFamily": map[string]interface{}{"html": "data-vendor"}, + } + a := Analyze{ + ResultGlobal: rg, + Body: `Powered by Acme
`, + } + + got := technologyVersions(technologies.DedupTechno(a.Run())) + if _, ok := got["RealText"]; !ok { + t.Errorf("visible text not matched: %v", got) + } + if _, ok := got["HtmlFamily"]; !ok { + t.Errorf("html family must still see the markup: %v", got) + } + if _, ok := got["MarkupOnly"]; ok { + t.Errorf("a text pattern matched an attribute value: %v", got) + } +} + +// TestDomTextUsesElementText pins the dom fix: the pattern is matched against +// the selected element, not against the whole document. +func TestDomTextUsesElementText(t *testing.T) { + rg := map[string]interface{}{ + "Widget": map[string]interface{}{ + "dom": map[string]interface{}{ + ".version": map[string]interface{}{"text": `v([\d.]+)\;version:\1`}, + }, + }, + } + a := Analyze{ + ResultGlobal: rg, + Body: `v9.9.9` + + `v1.2.3`, + } + + got := technologyVersions(technologies.DedupTechno(a.Run())) + if got["Widget"] != "1.2.3" { + t.Errorf("version = %q, want %q — the pattern must read the selected element, not the page", got["Widget"], "1.2.3") + } +} + +// TestBadFingerprintShapesDoNotPanic feeds every matcher a value of the wrong +// JSON type. The database is third-party data fetched at startup, so a shape +// change must degrade detection, not crash the scan. +func TestBadFingerprintShapesDoNotPanic(t *testing.T) { + rg := map[string]interface{}{ + "BadHeaders": map[string]interface{}{"headers": "should-be-an-object"}, + "BadMeta": map[string]interface{}{"meta": []interface{}{"should-be-an-object"}}, + "BadDom": map[string]interface{}{"dom": map[string]interface{}{"div": "should-be-an-object"}}, + "BadCert": map[string]interface{}{"certIssuer": []interface{}{1, 2, 3}}, + "BadDns": map[string]interface{}{"dns": "should-be-an-object"}, + "BadUrl": map[string]interface{}{"url": 42}, + "BadScriptSrc": map[string]interface{}{"scriptSrc": map[string]interface{}{"a": "b"}}, + "BadHtml": map[string]interface{}{"html": map[string]interface{}{"a": "b"}}, + "BadXhr": map[string]interface{}{"xhr": 3.14}, + "NotAnObject": "this technology is not a map at all", + "NilEverything": map[string]interface{}{"headers": nil, "html": nil, "meta": nil}, + } + a := Analyze{ + ResultGlobal: rg, + Body: `