From 067aac11356f33cc3c9b496c9a7b0005deb58de7 Mon Sep 17 00:00:00 2001 From: Ricardo Valdes Date: Wed, 23 Sep 2026 09:16:31 -0400 Subject: [PATCH 1/2] fix(aws): keep MFA requirement checks out of sign-in success --- filters/aws/aws.yml | 7 +++-- plugins/alerts/aws_action_result_test.go | 40 ++++++++++++++++++++++++ plugins/alerts/testdata/aws_raw.json | 7 +++-- 3 files changed, 49 insertions(+), 5 deletions(-) create mode 100644 plugins/alerts/aws_action_result_test.go diff --git a/filters/aws/aws.yml b/filters/aws/aws.yml index 44a4f8128..ef0aacdad 100644 --- a/filters/aws/aws.yml +++ b/filters/aws/aws.yml @@ -1,4 +1,4 @@ -# AWS filter v1.1.0: CloudTrail JSON and identified GuardDuty EventBridge findings. +# AWS filter v1.1.1: CloudTrail JSON and identified GuardDuty EventBridge findings. # Original vendor objects and legacy aliases are retained. API success is request success. # Correlation candidates are generated here, never trusted from input. pipeline: @@ -282,13 +282,14 @@ pipeline: params: key: actionResult value: success + # CheckMfa only checks whether MFA is required; it does not finish a sign-in. + # Preserve its vendor response without promoting the intermediate success. where: equals("log.awsRecordType","cloudtrail") && !((exists("log.errorCode") && !equals("log.errorCode","")) || (exists("log.errorMessage") && !equals("log.errorMessage","")) || (exists("log.responseElements.errorCode") && !equals("log.responseElements.errorCode","")) || (exists("log.responseElements.errorMessage") && !equals("log.responseElements.errorMessage","")) || equalsIgnoreCase("log.responseElements.ConsoleLogin","Failure") || equalsIgnoreCase("log.responseElements.GetSigninToken","Failure") || equalsIgnoreCase("log.responseElements.CheckMfa","Failure")) && (equals("log.eventType","AwsApiCall") || (equals("log.eventName","ConsoleLogin") && equals("log.responseElements.ConsoleLogin","Success")) - || (equals("log.eventName","GetSigninToken") && equals("log.responseElements.GetSigninToken","Success")) || (equals("log.eventName","CheckMfa") - && equals("log.responseElements.CheckMfa","Success"))) + || (equals("log.eventName","GetSigninToken") && equals("log.responseElements.GetSigninToken","Success"))) - grok: source: log.recipientAccountId patterns: diff --git a/plugins/alerts/aws_action_result_test.go b/plugins/alerts/aws_action_result_test.go new file mode 100644 index 000000000..e31209322 --- /dev/null +++ b/plugins/alerts/aws_action_result_test.go @@ -0,0 +1,40 @@ +package main + +import ( + "testing" + + "github.com/threatwinds/go-sdk/plugins" + "github.com/tidwall/gjson" +) + +// These are fabricated raw CloudTrail records exercised by the existing offline +// extraction model and the pinned SDK CEL evaluator. They do not run the closed +// EventProcessor or feeds plugin and do not establish that a TI alert was created. +func TestAWSAuthenticationOutcome(t *testing.T) { + cfg, cache := awsConfig(t), plugins.NewCELCache("aws-authentication-outcome") + for _, tc := range []struct { + name, raw, result string + }{ + {"MFA requirement check", `{"eventVersion":"1.09","eventSource":"signin.amazonaws.com","eventType":"AwsConsoleSignIn","eventName":"CheckMfa","sourceIPAddress":"192.0.2.10","responseElements":{"CheckMfa":"Success"}}`, ""}, + {"failed check", `{"eventVersion":"1.09","eventSource":"signin.amazonaws.com","eventType":"AwsConsoleSignIn","eventName":"CheckMfa","sourceIPAddress":"192.0.2.10","responseElements":{"CheckMfa":"Failure"}}`, "failure"}, + {"completed sign-in", `{"eventVersion":"1.09","eventSource":"signin.amazonaws.com","eventType":"AwsConsoleSignIn","eventName":"ConsoleLogin","sourceIPAddress":"192.0.2.10","responseElements":{"ConsoleLogin":"Success"}}`, "success"}, + {"failed sign-in", `{"eventVersion":"1.09","eventSource":"signin.amazonaws.com","eventType":"AwsConsoleSignIn","eventName":"ConsoleLogin","sourceIPAddress":"192.0.2.10","responseElements":{"ConsoleLogin":"Failure"}}`, "failure"}, + {"missing sign-in outcome", `{"eventVersion":"1.09","eventSource":"signin.amazonaws.com","eventType":"AwsConsoleSignIn","eventName":"ConsoleLogin","sourceIPAddress":"192.0.2.10","responseElements":null}`, ""}, + {"denial overrides success", `{"eventVersion":"1.09","eventSource":"signin.amazonaws.com","eventType":"AwsConsoleSignIn","eventName":"ConsoleLogin","sourceIPAddress":"192.0.2.10","errorCode":"AccessDenied","responseElements":{"ConsoleLogin":"Success"}}`, "denied"}, + } { + t.Run(tc.name, func(t *testing.T) { + out := awsParse(t, cfg, tc.raw, "synthetic-collector", cache) + result := gjson.Get(out, "actionResult") + if result.String() != tc.result || (tc.result == "" && result.Exists()) { + t.Fatalf("actionResult = %s, want %q", result.Raw, tc.result) + } + eligible, err := cache.Eval(`equals("actionResult","success") && inCIDR("origin.ip","0.0.0.0/0")`, out) + if err != nil || eligible != (tc.result == "success") { + t.Fatalf("SDK success-and-IP predicate = %v, %v", eligible, err) + } + if tc.name == "MFA requirement check" && gjson.Get(out, "log.responseElements.CheckMfa").String() != "Success" { + t.Fatal("vendor check result was lost") + } + }) + } +} diff --git a/plugins/alerts/testdata/aws_raw.json b/plugins/alerts/testdata/aws_raw.json index c02e8cb40..ccaf78538 100644 --- a/plugins/alerts/testdata/aws_raw.json +++ b/plugins/alerts/testdata/aws_raw.json @@ -1676,9 +1676,12 @@ "dataSource": "collector-test", "raw": "{\"eventVersion\":\"1.09\",\"eventTime\":\"2026-09-17T12:00:00Z\",\"eventSource\":\"signin.amazonaws.com\",\"eventName\":\"CheckMfa\",\"eventType\":\"AwsConsoleSignIn\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"198.51.100.10\",\"recipientAccountId\":\"123456789012\",\"userIdentity\":{\"type\":\"IAMUser\",\"accountId\":\"123456789012\",\"arn\":\"arn:aws:iam::123456789012:user/reviewer\",\"principalId\":\"TESTPRINCIPAL\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\"}}},\"requestParameters\":{},\"responseElements\":{\"CheckMfa\":\"Success\"}}", "expected": { - "actionResult": "success" + "action": "CheckMfa", + "log.responseElements.CheckMfa": "Success" }, - "absent": [], + "absent": [ + "actionResult" + ], "matches": [] }, { From 88b530b46858ca6016767ca76c1153f61a269100 Mon Sep 17 00:00:00 2001 From: Ricardo Valdes Date: Thu, 24 Sep 2026 10:06:03 -0400 Subject: [PATCH 2/2] fix(aws): name correlation markers with characters the parser keeps The parser plugins remove every character except letters, digits and dots from the field names they write. The AWS filter added its 22 correlation markers with underscores (for example log.correlationCandidate.mass_resource_deletion), so they were stored without them (log.correlationCandidate.massresourcedeletion) and the 22 history rules that count them never matched. Rename the markers to camelCase (for example log.correlationCandidate.massResourceDeletion) in the filter and in every rule that reads them. Rule conditions, thresholds and windows are unchanged. Tests: the AWS parse model now stores grok, rename and add targets the way the parser does, the history test counts each marker by its new name, and a new test checks that every marker the filter adds and every marker an AWS rule reads are the same name, made only of letters, digits and dots. Fixtures that forge a marker in the input now use the new name. Co-Authored-By: Claude Opus 5.5 --- filters/aws/aws.yml | 44 ++--- plugins/alerts/aws_contract_test.go | 14 +- plugins/alerts/aws_history_test.go | 13 +- plugins/alerts/aws_marker_names_test.go | 184 ++++++++++++++++++ plugins/alerts/testdata/aws_raw.json | 6 +- .../alerts/testdata/filter-contracts/aws.json | 2 +- .../aws/aws/aws_ecs_credential_theft.yml | 2 +- .../cloud/aws/aws/aws_golden_saml_attack.yml | 2 +- .../aws/aws_securityhub_finding_evasion.yml | 2 +- .../aws/aws/aws_ssm_sendcommand_abuse.yml | 2 +- .../aws/aws/aws_sso_suspicious_activities.yml | 2 +- .../aws/aws/cloudformation_stack_deletion.yml | 2 +- .../aws/console_login_impossible_travel.yml | 2 +- .../aws/cross_account_access_anomalies.yml | 2 +- .../aws/iam_backdoor_creation_attempts.yml | 2 +- .../aws/iam_privilege_escalation_paths.yml | 2 +- .../aws/aws/lambda_privilege_escalation.yml | 2 +- .../cloud/aws/aws/mass_resource_deletion.yml | 2 +- rules/cloud/aws/aws/route53_dns_hijacking.yml | 2 +- .../aws/aws/s3_bulk_data_exfiltration.yml | 2 +- .../aws/secrets_manager_suspicious_access.yml | 4 +- .../aws/aws/security_group_modifications.yml | 2 +- rules/cloud/aws/aws/ssm_session_abuse.yml | 2 +- rules/cloud/aws/aws/sts_token_abuse.yml | 2 +- .../aws/aws/unusual_api_call_patterns.yml | 2 +- .../cloud/aws/aws/vpc_flow_log_anomalies.yml | 2 +- ...access_aws_iam_assume_role_brute_force.yml | 2 +- ...ccess_root_console_failure_brute_force.yml | 2 +- 28 files changed, 251 insertions(+), 58 deletions(-) create mode 100644 plugins/alerts/aws_marker_names_test.go diff --git a/filters/aws/aws.yml b/filters/aws/aws.yml index ef0aacdad..16243039b 100644 --- a/filters/aws/aws.yml +++ b/filters/aws/aws.yml @@ -388,7 +388,7 @@ pipeline: - add: function: string params: - key: log.correlationCandidate.aws_ecs_credential_theft + key: log.correlationCandidate.awsEcsCredentialTheft value: match where: |- equals("log.awsRecordType","cloudtrail") && (equals("log.eventSource", "ecs.amazonaws.com") && @@ -399,14 +399,14 @@ pipeline: - add: function: string params: - key: log.correlationCandidate.saml_provider_change + key: log.correlationCandidate.samlProviderChange value: match where: equals("log.awsRecordType","cloudtrail") && equals("log.eventSource","iam.amazonaws.com") && oneOf("log.eventName",["CreateSAMLProvider", "UpdateSAMLProvider"]) && equals("actionResult","success") - add: function: string params: - key: log.correlationCandidate.aws_securityhub_finding_evasion + key: log.correlationCandidate.awsSecurityhubFindingEvasion value: match where: |- equals("log.awsRecordType","cloudtrail") && (equals("log.eventSource", "securityhub.amazonaws.com") && @@ -416,7 +416,7 @@ pipeline: - add: function: string params: - key: log.correlationCandidate.aws_ssm_sendcommand_abuse + key: log.correlationCandidate.awsSsmSendcommandAbuse value: match where: |- equals("log.awsRecordType","cloudtrail") && (equals("log.eventSource", "ssm.amazonaws.com") && @@ -425,28 +425,28 @@ pipeline: - add: function: string params: - key: log.correlationCandidate.aws_sso_suspicious_activities + key: log.correlationCandidate.awsSsoSuspiciousActivities value: match where: equals("log.awsRecordType","cloudtrail") && (equals("log.eventSource","sso.amazonaws.com") && oneOf("log.eventName",["CreatePermissionSet", "AttachManagedPolicyToPermissionSet", "DeletePermissionSet", "PutInlinePolicyToPermissionSet"]) && equals("actionResult","success")) - add: function: string params: - key: log.correlationCandidate.cloudformation_stack_deletion + key: log.correlationCandidate.cloudformationStackDeletion value: match where: equals("log.awsRecordType","cloudtrail") && (equals("log.eventSource", "cloudformation.amazonaws.com") && equals("log.eventName", "DeleteStack") && equals("actionResult","success")) - add: function: string params: - key: log.correlationCandidate.console_login_impossible_travel + key: log.correlationCandidate.consoleLoginImpossibleTravel value: match where: equals("log.awsRecordType","cloudtrail") && (equals("log.eventSource","signin.amazonaws.com") && oneOf("log.eventName",["ConsoleLogin"]) && equals("actionResult","success") && regexMatch("origin.geolocation.countryCode","^[A-Z]{2}$")) - add: function: string params: - key: log.correlationCandidate.cross_account_access_anomalies + key: log.correlationCandidate.crossAccountAccessAnomalies value: match where: equals("log.awsRecordType","cloudtrail") && (equals("log.eventSource","sts.amazonaws.com") && oneOf("log.eventName",["AssumeRole"]) && equals("actionResult","success") && regexMatch("log.userIdentity.accountId","^[0-9]{12}$") && regexMatch("log.responseElements.assumedRoleUser.arn","^arn:[^:]+:sts::[0-9]{12}:assumed-role/.+$") @@ -454,14 +454,14 @@ pipeline: - add: function: string params: - key: log.correlationCandidate.iam_backdoor_creation_attempts + key: log.correlationCandidate.iamBackdoorCreationAttempts value: match where: equals("log.awsRecordType","cloudtrail") && (equals("log.eventSource", "iam.amazonaws.com") && oneOf("log.eventName", ["CreateUser", "CreateAccessKey", "AttachUserPolicy", "PutUserPolicy", "CreateLoginProfile"]) && equals("actionResult","success")) - add: function: string params: - key: log.correlationCandidate.iam_privilege_escalation_paths + key: log.correlationCandidate.iamPrivilegeEscalationPaths value: match where: |- equals("log.awsRecordType","cloudtrail") && (equals("log.eventSource", "iam.amazonaws.com") && @@ -470,28 +470,28 @@ pipeline: - add: function: string params: - key: log.correlationCandidate.lambda_privilege_escalation + key: log.correlationCandidate.lambdaPrivilegeEscalation value: match where: equals("log.awsRecordType","cloudtrail") && (equals("log.eventSource","iam.amazonaws.com") && oneOf("log.eventName",["AttachRolePolicy", "AttachUserPolicy"]) && equals("actionResult","success") && regexMatch("log.requestParameters.policyArn","^arn:(aws|aws-cn|aws-us-gov):iam::aws:policy/(AdministratorAccess|IAMFullAccess)$")) - add: function: string params: - key: log.correlationCandidate.mass_resource_deletion + key: log.correlationCandidate.massResourceDeletion value: match where: equals("log.awsRecordType","cloudtrail") && (regexMatch("log.eventName","^(Delete|Terminate|Remove)[A-Z]") && equals("actionResult","success") && !equals("log.eventSource","s3.amazonaws.com")) - add: function: string params: - key: log.correlationCandidate.route53_dns_hijacking + key: log.correlationCandidate.route53DnsHijacking value: match where: equals("log.awsRecordType","cloudtrail") && (equals("log.eventSource", "route53.amazonaws.com") && equals("log.eventName", "ChangeResourceRecordSets") && equals("actionResult","success")) - add: function: string params: - key: log.correlationCandidate.s3_bulk_data_exfiltration + key: log.correlationCandidate.s3BulkDataExfiltration value: match where: |- equals("log.awsRecordType","cloudtrail") && (equals("log.eventSource", "s3.amazonaws.com") && @@ -500,14 +500,14 @@ pipeline: - add: function: string params: - key: log.correlationCandidate.secrets_manager_suspicious_access + key: log.correlationCandidate.secretsManagerSuspiciousAccess value: match where: equals("log.awsRecordType","cloudtrail") && (equals("log.eventSource", "secretsmanager.amazonaws.com") && (equals("log.eventName", "GetSecretValue") || equals("log.eventName", "BatchGetSecretValue")) && equals("actionResult","success")) - add: function: string params: - key: log.correlationCandidate.security_group_modifications + key: log.correlationCandidate.securityGroupModifications value: match where: equals("log.awsRecordType","cloudtrail") && (equals("log.eventSource","ec2.amazonaws.com") && oneOf("log.eventName",["AuthorizeSecurityGroupIngress", "AuthorizeSecurityGroupEgress", "ModifySecurityGroupRules"]) && equals("actionResult","success") && (exists('log.requestParameters.ipPermissions.items.#(ipRanges.items.#(cidrIp=="0.0.0.0/0"))') @@ -516,7 +516,7 @@ pipeline: - add: function: string params: - key: log.correlationCandidate.ssm_session_abuse + key: log.correlationCandidate.ssmSessionAbuse value: match where: |- equals("log.awsRecordType","cloudtrail") && (equals("log.eventSource", "ssm.amazonaws.com") && @@ -525,7 +525,7 @@ pipeline: - add: function: string params: - key: log.correlationCandidate.sts_token_abuse + key: log.correlationCandidate.stsTokenAbuse value: match where: |- equals("log.awsRecordType","cloudtrail") && (equals("log.eventSource", "sts.amazonaws.com") && @@ -538,7 +538,7 @@ pipeline: - add: function: string params: - key: log.correlationCandidate.unusual_api_call_patterns + key: log.correlationCandidate.unusualApiCallPatterns value: match where: "equals(\"log.awsRecordType\",\"cloudtrail\") && (exists(\"log.eventSource\") && \nexists(\"log.sourceIPAddress\") &&\n\ exists(\"log.eventName\") &&\n(\n oneOf(\"log.eventName\", [\"DescribeSecurityGroups\", \"DescribeNetworkAcls\", \"DescribeVpcs\"\ @@ -549,21 +549,21 @@ pipeline: - add: function: string params: - key: log.correlationCandidate.vpc_flow_log_anomalies + key: log.correlationCandidate.vpcFlowLogAnomalies value: match where: equals("log.awsRecordType","cloudtrail") && (equals("log.eventSource","ec2.amazonaws.com") && oneOf("log.eventName",["DeleteFlowLogs"]) && equals("actionResult","success")) - add: function: string params: - key: log.correlationCandidate.credential_access_aws_iam_assume_role_brute_force + key: log.correlationCandidate.credentialAccessAwsIamAssumeRoleBruteForce value: match where: equals("log.awsRecordType","cloudtrail") && (equals("log.eventSource","iam.amazonaws.com") && oneOf("log.eventName",["UpdateAssumeRolePolicy"]) && oneOf("log.errorCode",["MalformedPolicyDocument", "MalformedPolicyDocumentException"]) && equals("actionResult","failure")) - add: function: string params: - key: log.correlationCandidate.credential_access_root_console_failure_brute_force + key: log.correlationCandidate.credentialAccessRootConsoleFailureBruteForce value: match where: equals("log.awsRecordType","cloudtrail") && (equals("log.eventSource","signin.amazonaws.com") && oneOf("log.eventName",["ConsoleLogin"]) && equals("log.userIdentity.type","Root") && oneOf("actionResult",["failure", "denied"])) diff --git a/plugins/alerts/aws_contract_test.go b/plugins/alerts/aws_contract_test.go index bc7d27847..9f2c3eed9 100644 --- a/plugins/alerts/aws_contract_test.go +++ b/plugins/alerts/aws_contract_test.go @@ -106,6 +106,14 @@ func awsRegex(t *testing.T, g *plugins.Grok, cfg *plugins.Config) *regexp.Regexp } return r } + +// awsStoredName is the name the parser plugins store for a grok, rename or add target: +// utils.SanitizeField keeps only letters, digits and dots. +func awsStoredName(name string) string { + utils.SanitizeField(&name) + return name +} + func awsParse(t *testing.T, cfg *plugins.Config, raw string, dataSource string, cache *plugins.CELCache, enrichment ...map[string]any) string { return awsParseMode(t, cfg, raw, dataSource, cache, false, enrichment...) } @@ -169,13 +177,13 @@ func awsParseMode(t *testing.T, cfg *plugins.Config, raw string, dataSource stri } for i, p := range g.Patterns { if p.FieldName != "" { - awsPut(draft, p.FieldName, m[r.SubexpIndex(fmt.Sprintf("f%d", i))], false) + awsPut(draft, awsStoredName(p.FieldName), m[r.SubexpIndex(fmt.Sprintf("f%d", i))], false) } } case "rename": for _, p := range s.Rename.From { if v, ok := awsGet(draft, p); ok { - awsPut(draft, s.Rename.To, v, false) + awsPut(draft, awsStoredName(s.Rename.To), v, false) awsPut(draft, p, nil, true) break } @@ -184,7 +192,7 @@ func awsParseMode(t *testing.T, cfg *plugins.Config, raw string, dataSource stri if s.Add.Function != "string" { t.Fatalf("unsupported add function %s", s.Add.Function) } - awsPut(draft, s.Add.Params["key"].GetStringValue(), s.Add.Params["value"].AsInterface(), false) + awsPut(draft, awsStoredName(s.Add.Params["key"].GetStringValue()), s.Add.Params["value"].AsInterface(), false) case "delete": for _, p := range s.Delete.Fields { awsPut(draft, p, nil, true) diff --git a/plugins/alerts/aws_history_test.go b/plugins/alerts/aws_history_test.go index e48aec0aa..9b8614759 100644 --- a/plugins/alerts/aws_history_test.go +++ b/plugins/alerts/aws_history_test.go @@ -35,10 +35,13 @@ func TestAWSSDKHistory(t *testing.T) { queries := 0 mapping := map[string]any{"properties": map[string]any{}} props := mapping["properties"].(map[string]any) - paths := []string{"dataSource", "log.awsAccountKeyType", "log.awsAccountKey", "log.awsActorKeyType", "log.awsActorKey", "origin.ip", "origin.geolocation.countryCode", "log.eventName", "log.correlationCandidate.saml_provider_change"} + paths := []string{"dataSource", "log.awsAccountKeyType", "log.awsAccountKey", "log.awsActorKeyType", "log.awsActorKey", "origin.ip", "origin.geolocation.countryCode", "log.eventName"} for name, r := range rules { if len(r.Correlation) > 0 { - paths = append(paths, "log.correlationCandidate."+name) + if awsHistoryMarkers[name] == "" { + t.Fatalf("history rule %s has no correlation marker", name) + } + paths = append(paths, "log.correlationCandidate."+awsHistoryMarkers[name]) } } for _, path := range paths { @@ -225,10 +228,8 @@ func TestAWSSDKHistory(t *testing.T) { if yes, e := cache.Eval(r.Where, out); e != nil || !yes { t.Fatalf("raw trigger failed: %v %v", yes, e) } - marker := "log.correlationCandidate." + tc.rule - if tc.rule == "aws_golden_saml_attack" { - marker = "log.correlationCandidate.saml_provider_change" - } + // The filter stores this marker and the rule counts it under the same name. + marker := "log.correlationCandidate." + awsHistoryMarkers[tc.rule] terms = map[string]string{"dataSource": "collector-test", "log.awsAccountKeyType": "recipient", "log.awsAccountKey": "123456789012", marker: "match"} notTerms = map[string]string{} if tc.mode == "ip" { diff --git a/plugins/alerts/aws_marker_names_test.go b/plugins/alerts/aws_marker_names_test.go new file mode 100644 index 000000000..21c6fe045 --- /dev/null +++ b/plugins/alerts/aws_marker_names_test.go @@ -0,0 +1,184 @@ +package main + +// The parser plugins keep only letters, digits and dots in the field names they write +// (go-sdk utils.SanitizeField), while rules look names up exactly as written. A marker +// added as log.correlationCandidate.mass_resource_deletion is stored as +// ...massresourcedeletion, so a history rule counting the underscored name never fires. +import ( + "encoding/json" + "os" + "path/filepath" + "regexp" + "slices" + "strings" + "testing" + + "github.com/threatwinds/go-sdk/plugins" + "github.com/threatwinds/go-sdk/utils" + "google.golang.org/protobuf/encoding/protojson" +) + +// awsHistoryMarkers maps each AWS history rule (file name without extension) to the +// correlation marker that filters/aws/aws.yml adds and the rule counts. +var awsHistoryMarkers = map[string]string{ + "aws_ecs_credential_theft": "awsEcsCredentialTheft", + "aws_golden_saml_attack": "samlProviderChange", + "aws_securityhub_finding_evasion": "awsSecurityhubFindingEvasion", + "aws_ssm_sendcommand_abuse": "awsSsmSendcommandAbuse", + "aws_sso_suspicious_activities": "awsSsoSuspiciousActivities", + "cloudformation_stack_deletion": "cloudformationStackDeletion", + "console_login_impossible_travel": "consoleLoginImpossibleTravel", + "credential_access_aws_iam_assume_role_brute_force": "credentialAccessAwsIamAssumeRoleBruteForce", + "credential_access_root_console_failure_brute_force": "credentialAccessRootConsoleFailureBruteForce", + "cross_account_access_anomalies": "crossAccountAccessAnomalies", + "iam_backdoor_creation_attempts": "iamBackdoorCreationAttempts", + "iam_privilege_escalation_paths": "iamPrivilegeEscalationPaths", + "lambda_privilege_escalation": "lambdaPrivilegeEscalation", + "mass_resource_deletion": "massResourceDeletion", + "route53_dns_hijacking": "route53DnsHijacking", + "s3_bulk_data_exfiltration": "s3BulkDataExfiltration", + "secrets_manager_suspicious_access": "secretsManagerSuspiciousAccess", + "security_group_modifications": "securityGroupModifications", + "ssm_session_abuse": "ssmSessionAbuse", + "sts_token_abuse": "stsTokenAbuse", + "unusual_api_call_patterns": "unusualApiCallPatterns", + "vpc_flow_log_anomalies": "vpcFlowLogAnomalies", +} + +// Every marker the AWS filter adds and every marker an AWS rule reads (where, history +// fields and placeholders, groupBy, deduplicateBy) must be the same string, made only of +// letters, digits and dots, so the stored name is the name the rule looks up. +func TestAWSCorrelationMarkerNames(t *testing.T) { + const prefix = "log.correlationCandidate." + clean := regexp.MustCompile(`^[A-Za-z0-9.]+$`) + kept := func(name string) bool { + stored := name + utils.SanitizeField(&stored) + return stored == name && clean.MatchString(name) + } + + written := map[string]string{} + for _, stage := range awsConfig(t).Pipeline { + if !slices.Contains(stage.DataTypes, "aws") { + continue + } + for _, step := range stage.Steps { + names := []string{} + if s := step.Grok; s != nil { + for _, p := range s.Patterns { + if p.FieldName != "" { + names = append(names, p.FieldName) + } + } + } + if s := step.Rename; s != nil { + names = append(names, s.To) + } + if s := step.Csv; s != nil { + names = append(names, s.Headers...) + } + if s := step.Add; s != nil { + key := s.Params["key"].GetStringValue() + names = append(names, key) + if strings.HasPrefix(key, prefix) { + value := s.Params["value"].GetStringValue() + if previous, ok := written[key]; ok && previous != value { + t.Errorf("filter adds %s as %q and %q", key, previous, value) + } + written[key] = value + } + } + for _, name := range names { + if !kept(name) { + t.Errorf("filter writes %q, which the parser stores under another name", name) + } + } + } + } + + reference := regexp.MustCompile(`log\.correlationCandidate\.[^"'\s,()\[\]{}]*`) + read := map[string]map[string]bool{} + err := filepath.WalkDir("../../rules", func(path string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() || (filepath.Ext(path) != ".yml" && filepath.Ext(path) != ".yaml") { + return err + } + b, err := utils.ReadPbYaml(path) + if err != nil { + return err + } + var head struct { + DataTypes []string `json:"dataTypes"` + } + if err = json.Unmarshal(b, &head); err != nil || !slices.Contains(head.DataTypes, "aws") { + return err + } + rule := new(plugins.Rule) + if err = protojson.Unmarshal(b, rule); err != nil { + return err + } + name := strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)) + texts := []string{rule.Where} + texts = append(texts, rule.GroupBy...) + texts = append(texts, rule.DeduplicateBy...) + var searches func([]*plugins.SearchRequest) + searches = func(list []*plugins.SearchRequest) { + for _, search := range list { + for _, term := range search.With { + value := term.Value.GetStringValue() + texts = append(texts, term.Field, value) + if strings.HasPrefix(term.Field, prefix) && written[term.Field] != value { + t.Errorf("%s counts %s = %q; the filter adds %q", path, term.Field, value, written[term.Field]) + } + } + searches(search.Or) + } + } + searches(rule.AfterEvents) + searches(rule.Correlation) + for _, text := range texts { + for _, marker := range reference.FindAllString(text, -1) { + marker = strings.TrimSuffix(marker, ".keyword") + if read[marker] == nil { + read[marker] = map[string]bool{} + } + read[marker][name] = true + if !kept(marker) { + t.Errorf("%s reads %q, which the parser never stores under that name", path, marker) + } + if _, ok := written[marker]; !ok { + t.Errorf("%s reads %q, which the AWS filter does not add", path, marker) + } + } + } + return nil + }) + if err != nil { + t.Fatal(err) + } + + expected := map[string]bool{} + for rule, marker := range awsHistoryMarkers { + expected[prefix+marker] = true + if _, ok := written[prefix+marker]; !ok { + t.Errorf("filter does not add %s%s for %s", prefix, marker, rule) + } + if !read[prefix+marker][rule] { + t.Errorf("%s does not read %s%s", rule, prefix, marker) + } + } + for marker, rules := range read { + for rule := range rules { + if prefix+awsHistoryMarkers[rule] != marker { + t.Errorf("%s reads %s, expected only %s%s", rule, marker, prefix, awsHistoryMarkers[rule]) + } + } + } + for marker := range written { + if !expected[marker] { + t.Errorf("filter adds %s, which no AWS history rule counts", marker) + } + } + if len(written) != 22 || len(awsHistoryMarkers) != 22 { + t.Errorf("marker coverage: filter adds %d, table lists %d", len(written), len(awsHistoryMarkers)) + } +} diff --git a/plugins/alerts/testdata/aws_raw.json b/plugins/alerts/testdata/aws_raw.json index ccaf78538..99b30d4d0 100644 --- a/plugins/alerts/testdata/aws_raw.json +++ b/plugins/alerts/testdata/aws_raw.json @@ -2167,13 +2167,13 @@ { "name": "input-markers-discarded", "dataSource": "collector-test", - "raw": "{\"eventVersion\":\"1.09\",\"eventTime\":\"2026-09-17T12:00:00Z\",\"eventSource\":\"ec2.amazonaws.com\",\"eventName\":\"DescribeTags\",\"eventType\":\"AwsApiCall\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"198.51.100.10\",\"recipientAccountId\":\"123456789012\",\"userIdentity\":{\"type\":\"IAMUser\",\"accountId\":\"123456789012\",\"arn\":\"arn:aws:iam::123456789012:user/reviewer\",\"principalId\":\"TESTPRINCIPAL\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\"}}},\"requestParameters\":{},\"responseElements\":null,\"awsRecordType\":\"guardduty\",\"correlationCandidate\":{\"mass_resource_deletion\":\"match\"},\"awsActorKey\":\"fake\"}", + "raw": "{\"eventVersion\":\"1.09\",\"eventTime\":\"2026-09-17T12:00:00Z\",\"eventSource\":\"ec2.amazonaws.com\",\"eventName\":\"DescribeTags\",\"eventType\":\"AwsApiCall\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"198.51.100.10\",\"recipientAccountId\":\"123456789012\",\"userIdentity\":{\"type\":\"IAMUser\",\"accountId\":\"123456789012\",\"arn\":\"arn:aws:iam::123456789012:user/reviewer\",\"principalId\":\"TESTPRINCIPAL\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\"}}},\"requestParameters\":{},\"responseElements\":null,\"awsRecordType\":\"guardduty\",\"correlationCandidate\":{\"massResourceDeletion\":\"match\"},\"awsActorKey\":\"fake\"}", "expected": { "log.awsRecordType": "cloudtrail", "log.awsActorKey": "arn:aws:iam::123456789012:user/reviewer" }, "absent": [ - "log.correlationCandidate.mass_resource_deletion" + "log.correlationCandidate.massResourceDeletion" ], "matches": [] }, @@ -2336,7 +2336,7 @@ { "name": "unknown-json", "dataSource": "collector-test", - "raw": "{\"message\":\"not a CloudTrail record\",\"eventName\":\"DeleteTrail\",\"awsRecordType\":\"cloudtrail\",\"correlationCandidate\":{\"mass_resource_deletion\":\"match\"}}", + "raw": "{\"message\":\"not a CloudTrail record\",\"eventName\":\"DeleteTrail\",\"awsRecordType\":\"cloudtrail\",\"correlationCandidate\":{\"massResourceDeletion\":\"match\"}}", "expected": {}, "absent": [ "action", diff --git a/plugins/alerts/testdata/filter-contracts/aws.json b/plugins/alerts/testdata/filter-contracts/aws.json index cff5a4fb9..729154493 100644 --- a/plugins/alerts/testdata/filter-contracts/aws.json +++ b/plugins/alerts/testdata/filter-contracts/aws.json @@ -93,7 +93,7 @@ "errorCode": "AccessDenied", "awsRecordType": "forged", "correlationCandidate": { - "mass_resource_deletion": "match" + "massResourceDeletion": "match" } } }, diff --git a/rules/cloud/aws/aws/aws_ecs_credential_theft.yml b/rules/cloud/aws/aws/aws_ecs_credential_theft.yml index 26a49b794..1a1910da9 100644 --- a/rules/cloud/aws/aws/aws_ecs_credential_theft.yml +++ b/rules/cloud/aws/aws/aws_ecs_credential_theft.yml @@ -42,7 +42,7 @@ afterEvents: - field: origin.ip operator: filter_term value: '{{.origin.ip}}' - - field: log.correlationCandidate.aws_ecs_credential_theft + - field: log.correlationCandidate.awsEcsCredentialTheft operator: filter_term value: match within: 30m diff --git a/rules/cloud/aws/aws/aws_golden_saml_attack.yml b/rules/cloud/aws/aws/aws_golden_saml_attack.yml index 0ff0093b6..e690c4817 100644 --- a/rules/cloud/aws/aws/aws_golden_saml_attack.yml +++ b/rules/cloud/aws/aws/aws_golden_saml_attack.yml @@ -37,7 +37,7 @@ afterEvents: - field: log.awsAccountKey operator: filter_term value: '{{.log.awsAccountKey}}' - - field: log.correlationCandidate.saml_provider_change + - field: log.correlationCandidate.samlProviderChange operator: filter_term value: match within: 24h diff --git a/rules/cloud/aws/aws/aws_securityhub_finding_evasion.yml b/rules/cloud/aws/aws/aws_securityhub_finding_evasion.yml index 93235b5b2..2db3cfee0 100644 --- a/rules/cloud/aws/aws/aws_securityhub_finding_evasion.yml +++ b/rules/cloud/aws/aws/aws_securityhub_finding_evasion.yml @@ -57,7 +57,7 @@ afterEvents: - field: log.awsActorKey operator: filter_term value: '{{.log.awsActorKey}}' - - field: log.correlationCandidate.aws_securityhub_finding_evasion + - field: log.correlationCandidate.awsSecurityhubFindingEvasion operator: filter_term value: match within: 30m diff --git a/rules/cloud/aws/aws/aws_ssm_sendcommand_abuse.yml b/rules/cloud/aws/aws/aws_ssm_sendcommand_abuse.yml index 9d215057e..f21ef9cc2 100644 --- a/rules/cloud/aws/aws/aws_ssm_sendcommand_abuse.yml +++ b/rules/cloud/aws/aws/aws_ssm_sendcommand_abuse.yml @@ -55,7 +55,7 @@ afterEvents: - field: log.awsActorKey operator: filter_term value: '{{.log.awsActorKey}}' - - field: log.correlationCandidate.aws_ssm_sendcommand_abuse + - field: log.correlationCandidate.awsSsmSendcommandAbuse operator: filter_term value: match within: 30m diff --git a/rules/cloud/aws/aws/aws_sso_suspicious_activities.yml b/rules/cloud/aws/aws/aws_sso_suspicious_activities.yml index 0421391d3..bb084accf 100644 --- a/rules/cloud/aws/aws/aws_sso_suspicious_activities.yml +++ b/rules/cloud/aws/aws/aws_sso_suspicious_activities.yml @@ -41,7 +41,7 @@ afterEvents: - field: origin.ip operator: filter_term value: '{{.origin.ip}}' - - field: log.correlationCandidate.aws_sso_suspicious_activities + - field: log.correlationCandidate.awsSsoSuspiciousActivities operator: filter_term value: match within: 30m diff --git a/rules/cloud/aws/aws/cloudformation_stack_deletion.yml b/rules/cloud/aws/aws/cloudformation_stack_deletion.yml index aef371803..f05abc9ad 100644 --- a/rules/cloud/aws/aws/cloudformation_stack_deletion.yml +++ b/rules/cloud/aws/aws/cloudformation_stack_deletion.yml @@ -56,7 +56,7 @@ afterEvents: - field: log.awsActorKey operator: filter_term value: '{{.log.awsActorKey}}' - - field: log.correlationCandidate.cloudformation_stack_deletion + - field: log.correlationCandidate.cloudformationStackDeletion operator: filter_term value: match within: 30m diff --git a/rules/cloud/aws/aws/console_login_impossible_travel.yml b/rules/cloud/aws/aws/console_login_impossible_travel.yml index d6f52837d..26e760203 100644 --- a/rules/cloud/aws/aws/console_login_impossible_travel.yml +++ b/rules/cloud/aws/aws/console_login_impossible_travel.yml @@ -48,7 +48,7 @@ afterEvents: - field: log.awsActorKey operator: filter_term value: '{{.log.awsActorKey}}' - - field: log.correlationCandidate.console_login_impossible_travel + - field: log.correlationCandidate.consoleLoginImpossibleTravel operator: filter_term value: match - field: origin.geolocation.countryCode diff --git a/rules/cloud/aws/aws/cross_account_access_anomalies.yml b/rules/cloud/aws/aws/cross_account_access_anomalies.yml index cfe5008a4..2e000b989 100644 --- a/rules/cloud/aws/aws/cross_account_access_anomalies.yml +++ b/rules/cloud/aws/aws/cross_account_access_anomalies.yml @@ -53,7 +53,7 @@ afterEvents: - field: origin.ip operator: filter_term value: '{{.origin.ip}}' - - field: log.correlationCandidate.cross_account_access_anomalies + - field: log.correlationCandidate.crossAccountAccessAnomalies operator: filter_term value: match within: 15m diff --git a/rules/cloud/aws/aws/iam_backdoor_creation_attempts.yml b/rules/cloud/aws/aws/iam_backdoor_creation_attempts.yml index 4890e5da5..8ddb1e3ee 100644 --- a/rules/cloud/aws/aws/iam_backdoor_creation_attempts.yml +++ b/rules/cloud/aws/aws/iam_backdoor_creation_attempts.yml @@ -50,7 +50,7 @@ afterEvents: - field: origin.ip operator: filter_term value: '{{.origin.ip}}' - - field: log.correlationCandidate.iam_backdoor_creation_attempts + - field: log.correlationCandidate.iamBackdoorCreationAttempts operator: filter_term value: match within: 30m diff --git a/rules/cloud/aws/aws/iam_privilege_escalation_paths.yml b/rules/cloud/aws/aws/iam_privilege_escalation_paths.yml index 49560ccc1..31ceb8e56 100644 --- a/rules/cloud/aws/aws/iam_privilege_escalation_paths.yml +++ b/rules/cloud/aws/aws/iam_privilege_escalation_paths.yml @@ -58,7 +58,7 @@ afterEvents: - field: log.awsActorKey operator: filter_term value: '{{.log.awsActorKey}}' - - field: log.correlationCandidate.iam_privilege_escalation_paths + - field: log.correlationCandidate.iamPrivilegeEscalationPaths operator: filter_term value: match within: 30m diff --git a/rules/cloud/aws/aws/lambda_privilege_escalation.yml b/rules/cloud/aws/aws/lambda_privilege_escalation.yml index cafb93741..d1ef730db 100644 --- a/rules/cloud/aws/aws/lambda_privilege_escalation.yml +++ b/rules/cloud/aws/aws/lambda_privilege_escalation.yml @@ -47,7 +47,7 @@ afterEvents: - field: log.awsActorKey operator: filter_term value: '{{.log.awsActorKey}}' - - field: log.correlationCandidate.lambda_privilege_escalation + - field: log.correlationCandidate.lambdaPrivilegeEscalation operator: filter_term value: match within: 1h diff --git a/rules/cloud/aws/aws/mass_resource_deletion.yml b/rules/cloud/aws/aws/mass_resource_deletion.yml index 8d9cdbd4a..11005133e 100644 --- a/rules/cloud/aws/aws/mass_resource_deletion.yml +++ b/rules/cloud/aws/aws/mass_resource_deletion.yml @@ -57,7 +57,7 @@ afterEvents: - field: log.awsActorKey operator: filter_term value: '{{.log.awsActorKey}}' - - field: log.correlationCandidate.mass_resource_deletion + - field: log.correlationCandidate.massResourceDeletion operator: filter_term value: match within: 10m diff --git a/rules/cloud/aws/aws/route53_dns_hijacking.yml b/rules/cloud/aws/aws/route53_dns_hijacking.yml index 7557e92a4..e97d1f476 100644 --- a/rules/cloud/aws/aws/route53_dns_hijacking.yml +++ b/rules/cloud/aws/aws/route53_dns_hijacking.yml @@ -57,7 +57,7 @@ afterEvents: - field: log.awsActorKey operator: filter_term value: '{{.log.awsActorKey}}' - - field: log.correlationCandidate.route53_dns_hijacking + - field: log.correlationCandidate.route53DnsHijacking operator: filter_term value: match within: 30m diff --git a/rules/cloud/aws/aws/s3_bulk_data_exfiltration.yml b/rules/cloud/aws/aws/s3_bulk_data_exfiltration.yml index 1f07c808a..97e56e64e 100644 --- a/rules/cloud/aws/aws/s3_bulk_data_exfiltration.yml +++ b/rules/cloud/aws/aws/s3_bulk_data_exfiltration.yml @@ -57,7 +57,7 @@ afterEvents: - field: log.awsActorKey operator: filter_term value: '{{.log.awsActorKey}}' - - field: log.correlationCandidate.s3_bulk_data_exfiltration + - field: log.correlationCandidate.s3BulkDataExfiltration operator: filter_term value: match within: 15m diff --git a/rules/cloud/aws/aws/secrets_manager_suspicious_access.yml b/rules/cloud/aws/aws/secrets_manager_suspicious_access.yml index 786dd33be..b67f09d94 100644 --- a/rules/cloud/aws/aws/secrets_manager_suspicious_access.yml +++ b/rules/cloud/aws/aws/secrets_manager_suspicious_access.yml @@ -55,7 +55,7 @@ afterEvents: - field: log.awsActorKey operator: filter_term value: '{{.log.awsActorKey}}' - - field: log.correlationCandidate.secrets_manager_suspicious_access + - field: log.correlationCandidate.secretsManagerSuspiciousAccess operator: filter_term value: match - field: log.eventName @@ -81,7 +81,7 @@ afterEvents: - field: log.awsActorKey operator: filter_term value: '{{.log.awsActorKey}}' - - field: log.correlationCandidate.secrets_manager_suspicious_access + - field: log.correlationCandidate.secretsManagerSuspiciousAccess operator: filter_term value: match - field: log.eventName diff --git a/rules/cloud/aws/aws/security_group_modifications.yml b/rules/cloud/aws/aws/security_group_modifications.yml index 61a5f8cad..eeee63307 100644 --- a/rules/cloud/aws/aws/security_group_modifications.yml +++ b/rules/cloud/aws/aws/security_group_modifications.yml @@ -55,7 +55,7 @@ afterEvents: - field: origin.ip operator: filter_term value: '{{.origin.ip}}' - - field: log.correlationCandidate.security_group_modifications + - field: log.correlationCandidate.securityGroupModifications operator: filter_term value: match within: 30m diff --git a/rules/cloud/aws/aws/ssm_session_abuse.yml b/rules/cloud/aws/aws/ssm_session_abuse.yml index edcdb813c..0986dfd43 100644 --- a/rules/cloud/aws/aws/ssm_session_abuse.yml +++ b/rules/cloud/aws/aws/ssm_session_abuse.yml @@ -57,7 +57,7 @@ afterEvents: - field: log.awsActorKey operator: filter_term value: '{{.log.awsActorKey}}' - - field: log.correlationCandidate.ssm_session_abuse + - field: log.correlationCandidate.ssmSessionAbuse operator: filter_term value: match within: 30m diff --git a/rules/cloud/aws/aws/sts_token_abuse.yml b/rules/cloud/aws/aws/sts_token_abuse.yml index caa548d42..9f6b72e6f 100644 --- a/rules/cloud/aws/aws/sts_token_abuse.yml +++ b/rules/cloud/aws/aws/sts_token_abuse.yml @@ -56,7 +56,7 @@ afterEvents: - field: origin.ip operator: filter_term value: '{{.origin.ip}}' - - field: log.correlationCandidate.sts_token_abuse + - field: log.correlationCandidate.stsTokenAbuse operator: filter_term value: match within: 15m diff --git a/rules/cloud/aws/aws/unusual_api_call_patterns.yml b/rules/cloud/aws/aws/unusual_api_call_patterns.yml index faa7c5582..b75194026 100644 --- a/rules/cloud/aws/aws/unusual_api_call_patterns.yml +++ b/rules/cloud/aws/aws/unusual_api_call_patterns.yml @@ -56,7 +56,7 @@ afterEvents: - field: origin.ip operator: filter_term value: '{{.origin.ip}}' - - field: log.correlationCandidate.unusual_api_call_patterns + - field: log.correlationCandidate.unusualApiCallPatterns operator: filter_term value: match within: 10m diff --git a/rules/cloud/aws/aws/vpc_flow_log_anomalies.yml b/rules/cloud/aws/aws/vpc_flow_log_anomalies.yml index 0d6f6bab9..de04cfad1 100644 --- a/rules/cloud/aws/aws/vpc_flow_log_anomalies.yml +++ b/rules/cloud/aws/aws/vpc_flow_log_anomalies.yml @@ -40,7 +40,7 @@ afterEvents: - field: origin.ip operator: filter_term value: '{{.origin.ip}}' - - field: log.correlationCandidate.vpc_flow_log_anomalies + - field: log.correlationCandidate.vpcFlowLogAnomalies operator: filter_term value: match within: 24h diff --git a/rules/cloud/aws/credential_access_aws_iam_assume_role_brute_force.yml b/rules/cloud/aws/credential_access_aws_iam_assume_role_brute_force.yml index b0953b071..a2d4e1089 100644 --- a/rules/cloud/aws/credential_access_aws_iam_assume_role_brute_force.yml +++ b/rules/cloud/aws/credential_access_aws_iam_assume_role_brute_force.yml @@ -48,7 +48,7 @@ afterEvents: - field: log.awsActorKey operator: filter_term value: '{{.log.awsActorKey}}' - - field: log.correlationCandidate.credential_access_aws_iam_assume_role_brute_force + - field: log.correlationCandidate.credentialAccessAwsIamAssumeRoleBruteForce operator: filter_term value: match within: 15m diff --git a/rules/cloud/aws/credential_access_root_console_failure_brute_force.yml b/rules/cloud/aws/credential_access_root_console_failure_brute_force.yml index ee4f951cf..eeea68583 100644 --- a/rules/cloud/aws/credential_access_root_console_failure_brute_force.yml +++ b/rules/cloud/aws/credential_access_root_console_failure_brute_force.yml @@ -45,7 +45,7 @@ afterEvents: - field: origin.ip operator: filter_term value: '{{.origin.ip}}' - - field: log.correlationCandidate.credential_access_root_console_failure_brute_force + - field: log.correlationCandidate.credentialAccessRootConsoleFailureBruteForce operator: filter_term value: match within: 15m