diff --git a/filters/antivirus/deceptive-bytes.yml b/filters/antivirus/deceptive-bytes.yml index c44723e0c..74590629a 100644 --- a/filters/antivirus/deceptive-bytes.yml +++ b/filters/antivirus/deceptive-bytes.yml @@ -1,4 +1,4 @@ -# Deceptive Bytes filter, version 3.0.3 +# Deceptive Bytes filter, version 3.0.4 # Based on previous version of the same filter pipeline: @@ -184,7 +184,7 @@ pipeline: pattern: '\,{{.data}}\,' - fieldName: origin.path pattern: '{{.greedy}}\,' - - fieldName: command + - fieldName: origin.command pattern: '{{.greedy}}' source: log.restMessage @@ -302,6 +302,7 @@ pipeline: fieldSplit: " " valueSplit: "=" source: log.restMessageToKv + where: exists("log.restMessageToKv") # Using grok to analyze the rest of the data - grok: @@ -412,13 +413,13 @@ pipeline: function: prefix substring: '"' fields: - - command + - origin.command - trim: function: suffix substring: '"' fields: - - command + - origin.command - trim: function: prefix @@ -443,19 +444,21 @@ pipeline: fieldSplit: " " valueSplit: "=" source: log.restData + where: exists("log.restData") # Using the kv filter with other config, usefull in key-value logs - kv: fieldSplit: ", " valueSplit: "=" source: log.pidStatusToKv + where: exists("log.pidStatusToKv") # Adding action result - add: function: string params: key: actionResult - value: "blocked" + value: "denied" where: 'exists("log.action") && oneOf("log.action", ["blocked", "prevented"])' # Adding severity based on log.severityLabelCharacter @@ -493,4 +496,4 @@ pipeline: - log.restMessageToKv - log.pidStatusToKv - log.userWithTrash - - log.severityLabelCharacter \ No newline at end of file + - log.severityLabelCharacter diff --git a/filters/audits/deceptive-bytes.md b/filters/audits/deceptive-bytes.md new file mode 100644 index 000000000..129250e95 --- /dev/null +++ b/filters/audits/deceptive-bytes.md @@ -0,0 +1,151 @@ +# Deceptive Bytes v11 filter review + +The filter previously extracted a command into a root `command` field. The v11 alert +module pins go-sdk v1.1.36 (v1.1.33 when this was first reviewed); neither version's +`Event` has such a field. Finalization discards it. +The corrected grok and both quote trims use `origin.command`, the documented +`Side.command` field. Three KV inputs are optional products of different grok +branches; the public KV plugin returns an error when a source is absent. Each KV +step now runs only when its source exists. Present-source separators and outputs +are unchanged. The filter's explicit `log.action=blocked` or `prevented` case +now yields the documented `actionResult=denied` value. It retains the vendor +action in `log.action`, and no shipped Deceptive Bytes rule reads root +`actionResult`. + +The following raw inputs are **fabricated parser fixtures**, not captured Deceptive +Bytes records: + +| Case | Raw input | Original output | Corrected output | +|---|---|---|---| +| Command | `<14>2026-09-23T12:00:00Z,123,-,45,source,67,path,platform,/tmp/fake.bin,"synthetic --flag"` | `origin.path=/tmp/fake.bin`; command absent; three missing-source KV errors | same path; `origin.command=synthetic --flag`; zero errors | +| Present KV source | `<14>1 2026-09-23T12:00:00Z host 2 foo:1 sampleKey=sampleValue` | `log.sampleKey=sampleValue`; two missing-source KV errors | same vendor fields; zero errors | +| Blocked action | `<14>1 2026-09-23T12:00:00Z host 2 foo:1 action=blocked` | `actionResult=blocked`; two missing-source KV errors | `actionResult=denied`, `log.action=blocked`; zero errors | +| Unrelated raw | `not a Deceptive Bytes message` | three missing-source KV errors | zero errors; no command | + +These four input pairs first produced eight events with the public EventProcessor +playground at commit `497bf53dbd1ae096f7b2dbc7bce77a6bf9f22ce1`, whose parser and +writer link go-sdk v1.1.26. On 2026-09-24 they were run again with EventProcessor +`main` at `8a3ade72bd9d12db21f6b273200588fb49540f14`, whose playground and plugins all +link go-sdk v1.1.36, the version the v11 alerts module now pins: every original and +corrected output in the table is unchanged. The common regex patterns +were read from a deployed UTMStack configuration. The complete staged inputs, +binary hashes and resulting events are retained in the private Data Engine review +run. SDK CEL evaluated the diagnostic predicate +`equals("dataType", "deceptive-bytes") && equals("origin.command", "synthetic --flag")` +on all eight resulting events: only the corrected command case matched. The focused +contract test and full `plugins/alerts` Go suite pass with go-sdk v1.1.36 (48 tests +pass, 11 skip because they need other technologies' private evidence, none fail). +The builds are not asserted to be identical to a customer deployment. + +The newest published engine image, `ghcr.io/utmstack/utmstack/eventprocessor:v11.2.14` +(built 2026-09-24 19:13 UTC on base image `eventprocessor/base:1.1.7`), embeds Go build +information showing that its playground and plugin binaries come from the same +EventProcessor revision `8a3ade7` with go-sdk v1.1.36, built with go1.26.8 for +linux/amd64. The local build used here is that source revision compiled natively for +darwin/arm64 with go1.25.7; only the Go toolchain and platform differ. + +All 16 shipped Deceptive Bytes rules were checked for consumers of `command`, +`origin.command` and the three temporary KV inputs. None reads them, so the +command fix itself needs no rule rewrite. + +Six shipped rules read 18 vendor keys that contain an underscore and reach the +event through KV, for example `log.event_type`, `log.decoy_sensitivity` and +`log.source_ip`, in predicates, history fields and placeholders, and grouping +paths. The KV parser passes every key through go-sdk `utils.SanitizeField`. Up to +v1.1.34 that function removed underscores, so KV stored `log.eventtype` and these +rules could not match; an earlier revision of this draft therefore respelled the 18 +names without underscores. Since v1.1.35 the function keeps underscores, and `v11` +now pins v1.1.36, so KV stores the vendor's own spelling and those respelled names +would never match. This revision restores the original names. Three of the six rules +are again identical to `v11`; `data_theft_attempt_indicators` keeps only its +`origin.ip` guard, `ransomware_behavior_patterns` only its placeholder guard (below) and +`nation_state_tactic_detection` only its `"true"` comparisons. +KV also stores values as strings, so ten boolean comparisons across five rules use +`"true"`; go-sdk v1.1.36 CEL still accepts a native boolean `true` for those +comparisons. Literal event labels, thresholds and source-IP requirements are +unchanged. Six source rules needed neither field nor boolean changes. + +Three rules run a history search on `{{.origin.ip}}` (and `{{.log.tacticName}}` or +`{{.log.processName}}`), but this filter never writes `origin.ip`. A missing +placeholder makes the search fail, and five failures switch a rule off with a +Circuit Breaker alert, so the data theft, advanced threat tactic and zero-day +conditions now also require those fields, as the other seven `origin.ip` rules of +this source already do. For the same reason `ransomware_behavior_patterns`, which +searches on `{{.log.process}}` and `{{.log.source_ip}}`, now requires both fields. The +history-guard test checks each of the four rules: no match without the fields it needs or +without any one of them, a match with them, and every placeholder resolved. Without the +ransomware guard it fails; the go-sdk v1.1.36 replay over the playground events plus two +copies of the ransomware line that each lack one of those fields then matched both copies +with unresolved placeholders. With the guard the same replay passes 65 of 65 checks: each +rule matches only its intended case, and the ransomware rule only the line that carries +both fields. + +The committed fabricated lines carry all 18 keys. On EventProcessor `8a3ade7` the KV +plugin stored every one with its underscore (for example `log.event_type`, +`log.process_name` and `log.deceptive_target`) and none without it. With the +respelled rules the Living Off The Land positive yielded no alert. With the restored +rules the playground raised exactly one alert for each of the Living Off The Land, +nation-state and privilege-escalation positives, containing only that line's event +ID, and none for the near-miss negative or any other line; these three rules need no +history search. The go-sdk v1.1.36 rule replay over the same events, plus copies +given an `origin.ip` and synthetic events for the boolean rules, matched each of the +six restored rules and the four other rules this draft edits only on its intended +case, resolved every history placeholder on those matches, and matched nothing with +the six other rules or with the respelled names (89 of 89 checks). This proves the +local parser and rule contract for these fabricated cases, not the frequency or +semantics of real vendor detections. + +No retained Deceptive Bytes documents were found in 29 successful source-index +discovery queries across the accessible v11 estate; two discovery attempts failed. +Thus raw vendor syntax, deployed parser version, production rule coverage, alert +grouping and notification remain unverified. The available official product pages +do not define the severity-letter crosswalk, actor roles, or the vendor event +labels assumed by these rules. Those semantic mappings are left for +a separate review with relevant source logs or a technical export specification. + +Sources: [SDK Event schema](https://github.com/threatwinds/go-sdk/blob/v1.1.36/plugins/plugins.proto), +[standard field meanings](https://github.com/threatwinds/go-sdk/wiki/Standard-Event-Schema), +[filter steps](https://github.com/threatwinds/go-sdk/wiki/Filter-Steps-Reference), +[public KV parser](https://github.com/utmstack/EventProcessor/blob/8a3ade72bd9d12db21f6b273200588fb49540f14/plugins/kv/main.go), +[SDK field sanitizer](https://github.com/threatwinds/go-sdk/blob/v1.1.36/utils/fields.go) (keeps letters, +digits, dots and underscores since v1.1.35). + +## Reproduce the raw parser and local alert checks + +`plugins/alerts/testdata/deceptive-bytes/` contains eleven fabricated raw inputs, +the twelve common regex definitions needed by this filter, and `replay.py`. +It stages the current filter and the three shipped rules that need no history search +(Living Off The Land, nation-state, privilege escalation), runs the actual +playground, and requires all eleven finalized events, zero parser errors, every +vendor key stored with its underscore, and exactly one alert from each rule, on its +positive line. The unrelated, near-miss and other cases must not alert. It records +the binary build information, input/configuration hashes and outputs in a fresh +local directory. It uses no customer connection or index writer. On EventProcessor +`8a3ade7` it passes: 11 events and 3 alerts. + +Build a separately checked-out EventProcessor at `8a3ade72bd9d12db21f6b273200588fb49540f14`, +preserving each module's checked-in dependencies. With `EP` set to that checkout's +absolute path: + +```sh +mkdir -p "$EP/test-bin" "$EP/test-plugins" +(cd "$EP" && go build -mod=readonly -o "$EP/test-bin/playground" ./cmd/playground) +for plugin in add grok delete sew kv trim cel saw; do + (cd "$EP/plugins/$plugin" && go build -mod=readonly -o "$EP/test-plugins/$plugin.plugin" .) +done +``` + +From this UTMStack checkout, use a Python environment with PyYAML installed: + +```sh +python3 plugins/alerts/testdata/deceptive-bytes/replay.py \ + --playground "$EP/test-bin/playground" --plugins "$EP/test-plugins" +(cd plugins/alerts && go test ./... -count=1) +``` + +The Python replay and Go suite are separate checks. The Go suite alone does not +execute raw extraction. Playground startup can take several minutes. At `8a3ade7` +the CEL plugin reads its OpenSearch address from separate `host`, `port`, `user` and +`password` settings, so the single loopback URL in `replay.py` leaves it an empty +address; the staged rules have no history request. This does not validate any other +rule's history, production grouping or notification. diff --git a/plugins/alerts/deceptive_bytes_filter_test.go b/plugins/alerts/deceptive_bytes_filter_test.go new file mode 100644 index 000000000..450b10fb7 --- /dev/null +++ b/plugins/alerts/deceptive_bytes_filter_test.go @@ -0,0 +1,232 @@ +package main + +import ( + "os" + "path/filepath" + "regexp" + "strings" + "testing" + + "github.com/threatwinds/go-sdk/plugins" + "github.com/threatwinds/go-sdk/utils" + "google.golang.org/protobuf/encoding/protojson" +) + +func TestDeceptiveBytesCommandAndKVContract(t *testing.T) { + path := filepath.Join("..", "..", "filters", "antivirus", "deceptive-bytes.yml") + encoded, err := utils.ReadPbYaml(path) + if err != nil { + t.Fatal(err) + } + config := new(plugins.Config) + if err := protojson.Unmarshal(encoded, config); err != nil { + t.Fatal(err) + } + if len(config.Pipeline) != 1 { + t.Fatalf("pipeline stages: %d, want 1", len(config.Pipeline)) + } + commandCaptures, commandTrims, deniedResults := 0, 0, 0 + kvSources := map[string]bool{ + "log.restMessageToKv": false, + "log.restData": false, + "log.pidStatusToKv": false, + } + for _, step := range config.Pipeline[0].Steps { + if grok := step.Grok; grok != nil && grok.Source == "log.restMessage" { + for _, pattern := range grok.Patterns { + if pattern.FieldName == "command" { + t.Fatal("root command is dropped during Event finalization") + } + if pattern.FieldName == "origin.command" { + commandCaptures++ + } + } + } + if trim := step.Trim; trim != nil { + for _, field := range trim.Fields { + if field == "command" { + t.Fatal("quote trim still targets the discarded command field") + } + if field == "origin.command" && trim.Substring == `"` { + commandTrims++ + } + } + } + if kv := step.Kv; kv != nil { + if _, ok := kvSources[kv.Source]; !ok { + t.Fatalf("unexpected KV source %q", kv.Source) + } + if kv.Where != `exists("`+kv.Source+`")` { + t.Errorf("optional KV source %q has guard %q", kv.Source, kv.Where) + } + kvSources[kv.Source] = true + } + if add := step.Add; add != nil && add.Params["key"].GetStringValue() == "actionResult" { + if add.Params["value"].GetStringValue() != "denied" { + t.Errorf("blocked/prevented must map to standard denied, got %q", add.Params["value"].GetStringValue()) + } + deniedResults++ + } + } + if commandCaptures != 1 || commandTrims != 2 { + t.Errorf("command captures=%d trims=%d, want 1 and 2", commandCaptures, commandTrims) + } + for source, seen := range kvSources { + if !seen { + t.Errorf("KV source %q missing", source) + } + } + if deniedResults != 1 { + t.Errorf("actionResult mappings: %d, want 1", deniedResults) + } + + // The reviewed SDK silently drops unknown root fields in ordinary + // finalization. The standard Side.command survives and can be read by CEL. + predicate := `equals("dataType", "deceptive-bytes") && equals("origin.command", "synthetic --flag")` + cache := plugins.NewCELCache("deceptive-bytes-command-contract") + for _, tc := range []struct { + name, input string + wantStored, wantMatch bool + }{ + {"standard command", `{"dataType":"deceptive-bytes","origin":{"command":"synthetic --flag"}}`, true, true}, + {"old root command", `{"dataType":"deceptive-bytes","command":"synthetic --flag"}`, false, false}, + {"different command", `{"dataType":"deceptive-bytes","origin":{"command":"other"}}`, true, false}, + {"different source", `{"dataType":"other","origin":{"command":"synthetic --flag"}}`, true, false}, + } { + t.Run(tc.name, func(t *testing.T) { + input := tc.input + event := new(plugins.Event) + if err := utils.StringToProtoMessage(&input, event); err != nil { + t.Fatal(err) + } + output, err := utils.ProtoMessageToString(event) + if err != nil { + t.Fatal(err) + } + if strings.Contains(*output, `"command":`) != tc.wantStored { + t.Fatalf("command finalization: %s", *output) + } + match, err := cache.Eval(predicate, *output) + if err != nil { + t.Fatal(err) + } + if match != tc.wantMatch { + t.Fatalf("diagnostic predicate match=%t want=%t", match, tc.wantMatch) + } + }) + } +} + +// Deceptive Bytes parses its dynamic vendor keys through KV, which calls the +// pinned SDK sanitizer before storing them under log. A rule spelling that the +// sanitizer removes cannot read the value produced by this filter. +func TestDeceptiveBytesRuleFieldSanitization(t *testing.T) { + files, err := filepath.Glob(filepath.Join("..", "..", "rules", "antivirus", "deceptive-bytes", "*.yml")) + if err != nil || len(files) != 16 { + t.Fatalf("source rules: %d files, error %v", len(files), err) + } + field := regexp.MustCompile(`(?:lastEvent\.)?log\.([A-Za-z][A-Za-z0-9_]*)`) + for _, path := range files { + contents, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + for _, match := range field.FindAllStringSubmatch(string(contents), -1) { + name := match[1] + utils.SanitizeField(&name) + if name != match[1] { + t.Errorf("%s reads %q; the KV producer writes log.%s", path, match[0], name) + } + } + } +} + +func TestDeceptiveBytesKVBooleanRuleCompatibility(t *testing.T) { + path := filepath.Join("..", "..", "rules", "antivirus", "deceptive-bytes", "nation_state_tactic_detection.yml") + encoded, err := utils.ReadPbYaml(path) + if err != nil { + t.Fatal(err) + } + rule := new(plugins.Rule) + if err := protojson.Unmarshal(encoded, rule); err != nil { + t.Fatal(err) + } + cache := plugins.NewCELCache("deceptive-bytes-kv-boolean") + // Build the vendor keys the way KV stores them, with the linked SDK sanitizer. + keys := []string{"event_type", "threat_level", "attack_sophistication", "apt_indicators"} + for i := range keys { + utils.SanitizeField(&keys[i]) + } + for _, tc := range []struct { + name, value string + want bool + }{ + {"KV string true", `"true"`, true}, + {"native boolean true", `true`, true}, + {"KV string false", `"false"`, false}, + {"native boolean false", `false`, false}, + {"numeric one", `1`, false}, + } { + t.Run(tc.name, func(t *testing.T) { + input := `{"dataType":"deceptive-bytes","log":{"` + keys[0] + `":"decoy_interaction","` + keys[1] + `":"critical","` + keys[2] + `":"advanced","` + keys[3] + `":` + tc.value + `}}` + event := new(plugins.Event) + if err := utils.StringToProtoMessage(&input, event); err != nil { + t.Fatal(err) + } + output, err := utils.ProtoMessageToString(event) + if err != nil { + t.Fatal(err) + } + match, err := cache.Eval(rule.Where, *output) + if err != nil { + t.Fatal(err) + } + if match != tc.want { + t.Fatalf("shipped rule predicate match=%t want=%t", match, tc.want) + } + }) + } +} + +func TestDeceptiveBytesExistingRulePredicate(t *testing.T) { + path := filepath.Join("..", "..", "rules", "antivirus", "deceptive-bytes", "living_off_the_land_detection.yml") + encoded, err := utils.ReadPbYaml(path) + if err != nil { + t.Fatal(err) + } + rule := new(plugins.Rule) + if err := protojson.Unmarshal(encoded, rule); err != nil { + t.Fatal(err) + } + cache := plugins.NewCELCache("deceptive-bytes-existing-rule") + eventTypeField, processNameField, targetField := "event_type", "process_name", "deceptive_target" + utils.SanitizeField(&eventTypeField) + utils.SanitizeField(&processNameField) + utils.SanitizeField(&targetField) + for _, tc := range []struct { + name, eventType string + want bool + }{ + {"matching decoy process event", "lolbin_trap", true}, + {"ordinary process event", "ordinary", false}, + } { + t.Run(tc.name, func(t *testing.T) { + input := `{"dataType":"deceptive-bytes","log":{"` + eventTypeField + `":"` + tc.eventType + `","` + processNameField + `":"cmd.exe","` + targetField + `":"decoy"}}` + event := new(plugins.Event) + if err := utils.StringToProtoMessage(&input, event); err != nil { + t.Fatal(err) + } + output, err := utils.ProtoMessageToString(event) + if err != nil { + t.Fatal(err) + } + match, err := cache.Eval(rule.Where, *output) + if err != nil { + t.Fatal(err) + } + if match != tc.want { + t.Fatalf("shipped rule predicate match=%t want=%t", match, tc.want) + } + }) + } +} diff --git a/plugins/alerts/deceptive_bytes_history_guard_test.go b/plugins/alerts/deceptive_bytes_history_guard_test.go new file mode 100644 index 000000000..02e357e84 --- /dev/null +++ b/plugins/alerts/deceptive_bytes_history_guard_test.go @@ -0,0 +1,109 @@ +package main + +import ( + "encoding/json" + "path/filepath" + "strings" + "testing" + + "github.com/threatwinds/go-sdk/plugins" + "github.com/threatwinds/go-sdk/utils" + "github.com/tidwall/gjson" + "google.golang.org/protobuf/encoding/protojson" +) + +// deceptiveBytesEvent builds an event from a log object plus dotted fields. +func deceptiveBytesEvent(t *testing.T, logFields map[string]any, extra map[string]string) string { + t.Helper() + logCopy := map[string]any{} + for k, v := range logFields { + logCopy[k] = v + } + event := map[string]any{"dataType": "deceptive-bytes", "log": logCopy} + for path, value := range extra { + parts := strings.Split(path, ".") + node := event + for _, part := range parts[:len(parts)-1] { + next, ok := node[part].(map[string]any) + if !ok { + next = map[string]any{} + node[part] = next + } + node = next + } + node[parts[len(parts)-1]] = value + } + b, err := json.Marshal(event) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +// A history search whose placeholder is missing returns an error, and five errors switch the +// rule off with a Circuit Breaker alert. Each rule must only match when its placeholders exist. +func TestDeceptiveBytesHistoryPlaceholdersGuarded(t *testing.T) { + cache := plugins.NewCELCache("deceptive-bytes-history-guard") + cases := []struct { + rule string + log map[string]any // matches the condition apart from the placeholder fields + needed map[string]string // the fields the history placeholders read + }{ + {"data_theft_attempt_indicators", + map[string]any{"event_type": "decoy_accessed", "action": "file_copy", "decoy_sensitivity": "high", "decoy_file": "f"}, + map[string]string{"origin.ip": "192.0.2.10"}}, + {"advanced_threat_tactic_identification", + map[string]any{"eventType": "advanced_threat_detected", "threatLevel": "critical", "tacticName": "execution", "deceptionTriggered": "true", "behaviorScore": 95}, + map[string]string{"origin.ip": "192.0.2.10"}}, + {"zero_day_behavior_patterns", + map[string]any{"eventType": "zero_day_suspect", "threatSignature": "unknown", "deceptionEnvironment": "true", "memoryAnomalyScore": 95, "knownMalwareFamily": "", "exploitTechnique": "t", "processName": "p.exe"}, + map[string]string{"origin.ip": "192.0.2.10"}}, + {"ransomware_behavior_patterns", + map[string]any{"event_type": "ransomware_behavior", "behavior_pattern": "mass_encryption"}, + map[string]string{"log.process": "example.exe", "log.source_ip": "192.0.2.10"}}, + } + for _, tc := range cases { + b, err := utils.ReadPbYaml(filepath.Join("../..", "rules/antivirus/deceptive-bytes", tc.rule+".yml")) + if err != nil { + t.Fatal(err) + } + rule := new(plugins.Rule) + if err := protojson.Unmarshal(b, rule); err != nil { + t.Fatal(err) + } + rule.Normalize() + t.Run(tc.rule, func(t *testing.T) { + // Without the placeholder fields, and without any one of them, the rule must not match. + if got, err := cache.Eval(rule.Where, deceptiveBytesEvent(t, tc.log, nil)); err != nil || got { + t.Errorf("matched without %v: %v (%v)", tc.needed, got, err) + } + for missing := range tc.needed { + partial := map[string]string{} + for path, value := range tc.needed { + if path != missing { + partial[path] = value + } + } + if got, err := cache.Eval(rule.Where, deceptiveBytesEvent(t, tc.log, partial)); err != nil || got { + t.Errorf("matched without %s: %v (%v)", missing, got, err) + } + } + with := deceptiveBytesEvent(t, tc.log, tc.needed) + got, err := cache.Eval(rule.Where, with) + if err != nil || !got { + t.Fatalf("did not match with %v: %v (%v)", tc.needed, got, err) + } + for _, block := range rule.Correlation { + for _, expr := range block.With { + value := expr.Value.GetStringValue() + if strings.HasPrefix(value, "{{.") && strings.HasSuffix(value, "}}") { + field := strings.TrimSuffix(strings.TrimPrefix(value, "{{."), "}}") + if !gjson.Get(with, field).Exists() { + t.Errorf("placeholder %s unresolved on a matching event", field) + } + } + } + } + }) + } +} diff --git a/plugins/alerts/testdata/deceptive-bytes/patterns.yaml b/plugins/alerts/testdata/deceptive-bytes/patterns.yaml new file mode 100644 index 000000000..50823d738 --- /dev/null +++ b/plugins/alerts/testdata/deceptive-bytes/patterns.yaml @@ -0,0 +1,14 @@ +# Common regex definitions needed by these fabricated parser cases. +patterns: + data: (.*?) + greedy: .* + hour: (([01][0-9])|2[0-4]) + integer: (?:[+-]?(?:[0-9]+)) + minute: (?:[0-5][0-9]) + monthDay: (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9]) + monthNumber: (?:0[1-9]|1[0-2]) + seconds: (?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?) + space: \s+ + time: ((([01][0-9])|2[0-4]):(?:[0-5][0-9])(?::(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?))) + word: \b\w+\b + year: (([1-9])[0-9]{1,3}) diff --git a/plugins/alerts/testdata/deceptive-bytes/raw.json b/plugins/alerts/testdata/deceptive-bytes/raw.json new file mode 100644 index 000000000..c8ba9f14a --- /dev/null +++ b/plugins/alerts/testdata/deceptive-bytes/raw.json @@ -0,0 +1,13 @@ +{ + "command": "<14>2026-09-23T12:00:00Z,123,-,45,source,67,path,platform,/tmp/fake.bin,\"synthetic --flag\"", + "rest-data": "<14>1 2026-09-23T12:00:00Z host 2 foo:1 sampleKey=sampleValue", + "action-blocked": "<14>1 2026-09-23T12:00:00Z host 2 foo:1 action=blocked", + "unrelated": "not a Deceptive Bytes message", + "rule-positive": "<14>1 2026-09-23T12:00:00Z host 2 foo:1 event_type=lolbin_trap process_name=cmd.exe deceptive_target=decoy", + "rule-negative": "<14>1 2026-09-23T12:00:00Z host 2 foo:1 event_type=ordinary process_name=cmd.exe deceptive_target=decoy", + "theft-names": "<14>1 2026-09-23T12:00:00Z host 2 foo:1 event_type=decoy_accessed action=file_copy decoy_sensitivity=high decoy_file=finance-decoy.xlsx", + "lateral-names": "<14>1 2026-09-23T12:00:00Z host 2 foo:1 event_type=trap_triggered trap_type=lateral_movement", + "nation-state-positive": "<14>1 2026-09-23T12:00:00Z host 2 foo:1 event_type=decoy_interaction threat_level=critical attack_sophistication=advanced threat_score=90 apt_indicators=true custom_malware=false advanced_ttps=false targeted_decoys=1 persistence_attempt=false", + "privilege-positive": "<14>1 2026-09-23T12:00:00Z host 2 foo:1 event_type=bait_accessed bait_type=privileged_account target_privilege=admin", + "ransomware-names": "<14>1 2026-09-23T12:00:00Z host 2 foo:1 event_type=ransomware_behavior behavior_pattern=mass_encryption process=example.exe source_ip=192.0.2.10" +} diff --git a/plugins/alerts/testdata/deceptive-bytes/replay.py b/plugins/alerts/testdata/deceptive-bytes/replay.py new file mode 100644 index 000000000..3e950eba4 --- /dev/null +++ b/plugins/alerts/testdata/deceptive-bytes/replay.py @@ -0,0 +1,163 @@ +"""Replay fabricated raw cases through separately built EventProcessor binaries. + +Requires PyYAML. This does not build or deploy anything and uses only local file +writers. See filters/audits/deceptive-bytes.md for the tested version boundary. +""" +import argparse +import errno +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile + +import yaml + +# Rules without history searches, staged together; each must alert on exactly one case. +RULES = { + "living_off_the_land_detection": "rule-positive", + "nation_state_tactic_detection": "nation-state-positive", + "privilege_escalation_bait_detection": "privilege-positive", +} +# Vendor keys that KV stores under log. Since go-sdk v1.1.35 the field-name sanitizer +# keeps underscores, so each key keeps its underscore; the six rules read these names. +KV_FIELDS = { + "rule-positive": {"event_type": "lolbin_trap", "process_name": "cmd.exe", "deceptive_target": "decoy"}, + "rule-negative": {"event_type": "ordinary", "process_name": "cmd.exe", "deceptive_target": "decoy"}, + "theft-names": {"event_type": "decoy_accessed", "action": "file_copy", "decoy_sensitivity": "high", + "decoy_file": "finance-decoy.xlsx"}, + "lateral-names": {"event_type": "trap_triggered", "trap_type": "lateral_movement"}, + "nation-state-positive": {"event_type": "decoy_interaction", "threat_level": "critical", + "attack_sophistication": "advanced", "threat_score": "90", "apt_indicators": "true", + "custom_malware": "false", "advanced_ttps": "false", "targeted_decoys": "1", + "persistence_attempt": "false"}, + "privilege-positive": {"event_type": "bait_accessed", "bait_type": "privileged_account", "target_privilege": "admin"}, + "ransomware-names": {"event_type": "ransomware_behavior", "behavior_pattern": "mass_encryption", + "process": "example.exe", "source_ip": "192.0.2.10"}, +} + + +def records(path): + # The playground writers can append adjacent JSON objects before newlines. + content = path.read_text() if path.exists() else "" + decoder = json.JSONDecoder() + result, offset = [], 0 + while offset < len(content): + while offset < len(content) and content[offset].isspace(): + offset += 1 + if offset < len(content): + record, offset = decoder.raw_decode(content, offset) + result.append(record) + return result + + +def require(condition, message): + if not condition: + raise RuntimeError(message) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--playground", required=True, type=Path) + parser.add_argument("--plugins", required=True, type=Path) + args = parser.parse_args() + fixture_dir = Path(__file__).resolve().parent + root = fixture_dir.parents[3] + os.umask(0o077) + work = Path(tempfile.mkdtemp(prefix="db-pg-", dir="/tmp")) + print(f"Local evidence directory: {work}", flush=True) + for part in ("input", "output", "pipeline/filters", "rules", "plugins", "sockets", "geolocation"): + (work / part).mkdir(parents=True, exist_ok=True) + binaries = {"playground": args.playground.resolve()} + for name in ("add", "grok", "delete", "sew", "kv", "trim", "cel", "saw"): + source = (args.plugins / f"{name}.plugin").resolve() + require(source.is_file(), f"Missing binary: {source}") + try: + os.link(source, work / "plugins" / source.name) + except OSError as error: + if error.errno != errno.EXDEV: + raise + shutil.copy2(source, work / "plugins" / source.name) + binaries[name] = source + config = { + "tenants": [{"id": "00000000-0000-4000-8000-000000000001", "name": "fixture"}], + "plugins": { + "analysis": {"order": ["sew", "cel"]}, + "correlation": {"order": ["saw"]}, + "notification": {"order": []}, + # CEL initializes a client; this selected rule has no history. + "org.opensearch": {"opensearch": "http://127.0.0.1:19200"}, + }, + } + (work / "pipeline/config.yaml").write_text(yaml.safe_dump(config)) + shutil.copy2(fixture_dir / "patterns.yaml", work / "pipeline/patterns.yaml") + filter_path = root / "filters/antivirus/deceptive-bytes.yml" + shutil.copy2(filter_path, work / "pipeline/filters/deceptive-bytes.yaml") + rule_paths, names = [], {} + for offset, stem in enumerate(RULES): + rule_path = root / f"rules/antivirus/deceptive-bytes/{stem}.yml" + rule = yaml.safe_load(rule_path.read_text()) + require(not rule.get("afterEvents") and not rule.get("correlation"), f"{stem} needs history") + rule["id"] = 9001 + offset # the playground loader needs unique non-zero ids + (work / f"rules/{stem}.yaml").write_text(yaml.safe_dump([rule])) + rule_paths.append(rule_path) + names[rule["name"]] = stem + cases = json.loads((fixture_dir / "raw.json").read_text()) + for name, raw in cases.items(): + event = { + "id": f"deceptive-bytes-{name}", "dataType": "deceptive-bytes", + "dataSource": "synthetic-device", "@timestamp": "2026-09-23T12:00:00Z", + "tenantId": config["tenants"][0]["id"], "raw": raw, + } + (work / "input" / f"{name}.json").write_text(json.dumps(event)) + manifest = { + "provenance": "fabricated raw inputs; no customer data", + "sourceHashes": {str(p.relative_to(root)): hashlib.sha256(p.read_bytes()).hexdigest() + for p in (filter_path, *rule_paths, fixture_dir / "patterns.yaml", fixture_dir / "raw.json")}, + "binaries": {name: { + "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + "buildInfo": subprocess.check_output(["go", "version", "-m", str(path)], text=True), + } for name, path in binaries.items()}, + } + (work / "manifest.json").write_text(json.dumps(manifest, indent=2)) + env = dict(os.environ, WORK_DIR=str(work), MODE="playground") + with (work / "execution.log").open("w") as log: + subprocess.run([str(binaries["playground"])], env=env, stdout=log, + stderr=subprocess.STDOUT, check=True, timeout=360) + parsed = records(work / "output/resulting_log.json") + events = {r.get("id"): r for r in parsed} + require(len(parsed) == len(cases) == len(events), "Missing or duplicate output events") + for name, raw in cases.items(): + event = events[f"deceptive-bytes-{name}"] + require(event.get("raw") == raw, f"Raw input mismatch: {name}") + require(not event.get("errors"), f"Parser errors: {name}") + command = events["deceptive-bytes-command"] + require(command.get("origin", {}).get("command") == "synthetic --flag", "Command not preserved") + require(command.get("origin", {}).get("path") == "/tmp/fake.bin", "Path changed") + require(events["deceptive-bytes-rest-data"].get("log", {}).get("sampleKey") == "sampleValue", "Present KV changed") + blocked = events["deceptive-bytes-action-blocked"] + require(blocked.get("actionResult") == "denied", "Blocked outcome not normalized") + require(blocked.get("log", {}).get("action") == "blocked", "Vendor action lost") + for name, fields in KV_FIELDS.items(): + vendor = events[f"deceptive-bytes-{name}"].get("log", {}) + for key, value in fields.items(): + require(vendor.get(key) == value, f"{name}: log.{key}={vendor.get(key)!r}, want {value!r}") + require(key.replace("_", "") == key or key.replace("_", "") not in vendor, + f"{name}: log.{key} was stored without its underscore") + alerts = records(work / "output/resulting_alert.json") + require(len(alerts) == len(RULES), f"Expected {len(RULES)} local alerts, got {len(alerts)}") + fired = {} + for alert in alerts: + require(alert.get("name") in names and not alert.get("errors"), "Unexpected alert or evaluation error") + fired[names[alert["name"]]] = [e.get("id") for e in alert.get("events", [])] + for stem, case in RULES.items(): + require(fired.get(stem) == [f"deceptive-bytes-{case}"], f"{stem}: alert events {fired.get(stem)}") + (work / "assertions.json").write_text(json.dumps({"passed": True, "events": len(parsed), "alerts": len(alerts)})) + print(f"PASS: {len(parsed)} raw events, zero parser errors, every vendor key stored with its underscore, " + f"{len(alerts)} local alerts, each from its intended rule") + + +if __name__ == "__main__": + main() diff --git a/rules/antivirus/deceptive-bytes/advanced_threat_tactic_identification.yml b/rules/antivirus/deceptive-bytes/advanced_threat_tactic_identification.yml index 981c1d205..8876767d3 100644 --- a/rules/antivirus/deceptive-bytes/advanced_threat_tactic_identification.yml +++ b/rules/antivirus/deceptive-bytes/advanced_threat_tactic_identification.yml @@ -1,4 +1,4 @@ -# Rule version v1.0.0 +# Rule version v1.0.1 dataTypes: - deceptive-bytes @@ -32,8 +32,9 @@ where: | equals("log.eventType", "advanced_threat_detected") && equals("log.threatLevel", "critical") && (oneOf("log.tacticName", ["initial_access", "execution", "persistence", "privilege_escalation", "defense_evasion"])) && - equals("log.deceptionTriggered", true) && - greaterOrEqual("log.behaviorScore", 80) + equals("log.deceptionTriggered", "true") && + greaterOrEqual("log.behaviorScore", 80) && + exists("origin.ip") afterEvents: - indexPattern: v11-log-deceptive-bytes-* with: diff --git a/rules/antivirus/deceptive-bytes/data_theft_attempt_indicators.yml b/rules/antivirus/deceptive-bytes/data_theft_attempt_indicators.yml index 135e8de27..63ab5b0eb 100644 --- a/rules/antivirus/deceptive-bytes/data_theft_attempt_indicators.yml +++ b/rules/antivirus/deceptive-bytes/data_theft_attempt_indicators.yml @@ -1,4 +1,4 @@ -# Rule version v1.0.0 +# Rule version v1.0.1 dataTypes: - deceptive-bytes @@ -28,7 +28,8 @@ description: | where: | equals("log.event_type", "decoy_accessed") && oneOf("log.action", ["file_read", "file_copy", "file_download"]) && - equals("log.decoy_sensitivity", "high") + equals("log.decoy_sensitivity", "high") && + exists("origin.ip") afterEvents: - indexPattern: v11-log-deceptive-bytes-* with: diff --git a/rules/antivirus/deceptive-bytes/fake_user_authentication_attempts.yml b/rules/antivirus/deceptive-bytes/fake_user_authentication_attempts.yml index 28460bf55..0dd26c442 100644 --- a/rules/antivirus/deceptive-bytes/fake_user_authentication_attempts.yml +++ b/rules/antivirus/deceptive-bytes/fake_user_authentication_attempts.yml @@ -25,7 +25,7 @@ description: | - Check for any legitimate user accounts that may have been compromised - Review authentication logs for attempts using real credentials from the same source - Notify the security team for potential active breach investigation -where: equals("log.eventType", "authentication") && equals("log.isDecoyUser", true) && exists("log.authResult") && exists("origin.ip") +where: equals("log.eventType", "authentication") && equals("log.isDecoyUser", "true") && exists("log.authResult") && exists("origin.ip") groupBy: - lastEvent.log.username - adversary.ip diff --git a/rules/antivirus/deceptive-bytes/nation_state_tactic_detection.yml b/rules/antivirus/deceptive-bytes/nation_state_tactic_detection.yml index c6ea53f34..69403d552 100644 --- a/rules/antivirus/deceptive-bytes/nation_state_tactic_detection.yml +++ b/rules/antivirus/deceptive-bytes/nation_state_tactic_detection.yml @@ -28,11 +28,11 @@ where: | oneOf("log.event_type", ["decoy_interaction", "honeypot_access", "deception_triggered"]) && equals("log.threat_level", "critical") && (equals("log.attack_sophistication", "advanced") || greaterOrEqual("log.threat_score", 85)) && - (equals("log.apt_indicators", true) || - equals("log.custom_malware", true) || - equals("log.advanced_ttps", true) || + (equals("log.apt_indicators", "true") || + equals("log.custom_malware", "true") || + equals("log.advanced_ttps", "true") || greaterThan("log.targeted_decoys", 1) || - equals("log.persistence_attempt", true)) + equals("log.persistence_attempt", "true")) groupBy: - adversary.host - adversary.ip diff --git a/rules/antivirus/deceptive-bytes/ransomware_behavior_patterns.yml b/rules/antivirus/deceptive-bytes/ransomware_behavior_patterns.yml index 01ccabb37..52ed40cec 100644 --- a/rules/antivirus/deceptive-bytes/ransomware_behavior_patterns.yml +++ b/rules/antivirus/deceptive-bytes/ransomware_behavior_patterns.yml @@ -1,4 +1,4 @@ -# Rule version v1.0.0 +# Rule version v1.0.1 dataTypes: - deceptive-bytes @@ -29,7 +29,9 @@ description: | 9. Review backup integrity and availability before any restoration attempts where: | equals("log.event_type", "ransomware_behavior") && - oneOf("log.behavior_pattern", ["mass_encryption", "file_enumeration", "ransom_note_drop"]) + oneOf("log.behavior_pattern", ["mass_encryption", "file_enumeration", "ransom_note_drop"]) && + exists("log.process") && + exists("log.source_ip") afterEvents: - indexPattern: v11-log-deceptive-bytes-* with: diff --git a/rules/antivirus/deceptive-bytes/threat_actor_attribution.yml b/rules/antivirus/deceptive-bytes/threat_actor_attribution.yml index 4bd0f5ec5..7c27df534 100644 --- a/rules/antivirus/deceptive-bytes/threat_actor_attribution.yml +++ b/rules/antivirus/deceptive-bytes/threat_actor_attribution.yml @@ -29,11 +29,11 @@ where: | equals("log.eventType", "threat_attribution") && greaterOrEqual("log.attributionConfidence", 70) && exists("log.actorProfile") && - equals("log.deceptionTriggered", true) && + equals("log.deceptionTriggered", "true") && (greaterOrEqual("log.ttpsMatched", 3) || - equals("log.infrastructureMatch", true) || + equals("log.infrastructureMatch", "true") || exists("log.toolingFingerprint")) && - equals("log.historicalCampaignMatch", true) + equals("log.historicalCampaignMatch", "true") groupBy: - lastEvent.log.actorProfile - lastEvent.log.campaignId diff --git a/rules/antivirus/deceptive-bytes/zero_day_behavior_patterns.yml b/rules/antivirus/deceptive-bytes/zero_day_behavior_patterns.yml index 2c7feb007..874ac0e7b 100644 --- a/rules/antivirus/deceptive-bytes/zero_day_behavior_patterns.yml +++ b/rules/antivirus/deceptive-bytes/zero_day_behavior_patterns.yml @@ -1,4 +1,4 @@ -# Rule version v1.0.0 +# Rule version v1.0.1 dataTypes: - deceptive-bytes @@ -28,7 +28,7 @@ description: | where: | oneOf("log.eventType", ["unknown_threat", "behavioral_anomaly", "zero_day_suspect"]) && equals("log.threatSignature", "unknown") && - equals("log.deceptionEnvironment", true) && + equals("log.deceptionEnvironment", "true") && ( (greaterOrEqual("log.memoryAnomalyScore", 90)) || (greaterOrEqual("log.processChainAnomalyScore", 85)) || @@ -36,7 +36,9 @@ where: | (greaterOrEqual("log.fileSystemAnomalyScore", 92)) ) && equals("log.knownMalwareFamily", "") && - exists("log.exploitTechnique") + exists("log.exploitTechnique") && + exists("origin.ip") && + exists("log.processName") afterEvents: - indexPattern: v11-log-deceptive-bytes-* with: