diff --git a/.gitallowed b/.gitallowed index 02edef8..ea3f97a 100644 --- a/.gitallowed +++ b/.gitallowed @@ -15,16 +15,15 @@ # rotate it and treat the leak seriously instead of silencing the scanner. # scripts/setup-git-secrets.sh registers its detectors as literal regex text -# and three of them (Azure connection string, PostgreSQL and MySQL DSN -# detectors, verified empirically -- MongoDB's does not) match their own +# and one of them (the Azure connection string detector) matches its own # registration line. Each entry below is anchored on the path, the line # number, AND the entire line as it appears in that file (including the # trailing comment), with a trailing $ -- not just the "git secrets --add # '" command prefix, and not just the added pattern with the anchor left # open at the end. A prefix-only anchor whitelists a secret appended to # ANY registration line in this file; an anchor missing the trailing $ -# whitelists a secret appended to the END of one of these three specific -# lines. Both are the #1972 hole reintroduced at a smaller scale, and both +# whitelists a secret appended to the END of this specific +# line. Both are the #1972 hole reintroduced at a smaller scale, and both # were caught only by a fixture exercising exactly that mutation -- add # one for any future entry in this block that isn't already covered. # @@ -38,8 +37,6 @@ # self-scan case below fails -- and note this comment must not spell any # of those spans out literally either, for the same reason). ^scripts/setup-git-secrets\.sh:[0-9]+:git secrets --add 'DefaultEndpointsProtoco[l]=https' # Azure Connection String$ -^scripts/setup-git-secrets\.sh:[0-9]+:git secrets --add 'postgre[s]://\[\^:\]\+:\[\^@\]\+@' # PostgreSQL$ -^scripts/setup-git-secrets\.sh:[0-9]+:git secrets --add 'mysq[l]://\[\^:\]\+:\[\^@\]\+@' # MySQL$ # Truncated PEM placeholder used by scripts/test-git-secrets-allowlist.sh # (and by any fixture in this repo needing a non-secret PEM-shaped string): diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 80c4211..09a3906 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -99,11 +99,11 @@ jobs: # mutable in principle, so also assert HEAD resolves to the exact # commit verified at the time this was written (`git ls-remote # --tags` against the upstream repo) before trusting it enough to - # `sudo make install`. After install we register the AWS pattern - # set and ASSERT at least one pattern was registered — without the - # assert, a registration failure produces a patternless scanner - # that exits 0 unconditionally, leaving the gate silently - # downgraded. + # `sudo make install`. After install we run the setup script, which + # registers the full pattern set and scans the tree, and ASSERT at + # least one pattern was registered; without the assert, a + # registration failure produces a patternless scanner that exits 0 + # unconditionally, leaving the gate silently downgraded. env: GIT_SECRETS_SHA: ad82d68ee924906a0401dfd48de5057731a9bc84 run: | @@ -115,8 +115,8 @@ jobs: exit 1 fi sudo make -C /tmp/git-secrets install - git secrets --register-aws --global - git secrets --list --global | grep -q '.' || { + bash scripts/setup-git-secrets.sh + git secrets --list | grep -q '.' || { echo "git-secrets registration produced no patterns — gate would be silently disabled" exit 1 } diff --git a/scripts/setup-git-secrets.sh b/scripts/setup-git-secrets.sh index f27fa86..3db78c2 100755 --- a/scripts/setup-git-secrets.sh +++ b/scripts/setup-git-secrets.sh @@ -95,7 +95,9 @@ echo "Adding custom secret patterns..." # AWS patterns git secrets --add 'AKIA[0-9A-Z]{16}' # AWS Access Key ID -git secrets --add '[^A-Za-z0-9/+=]{40}[^A-Za-z0-9/+=]' # AWS Secret Access Key +# macOS git grep has no \s, so the two --register-aws patterns that use it miss spaced assignments +git secrets --add '("|'\'')?(AWS|aws|Aws)?_?(SECRET|secret|Secret)?_?(ACCESS|access|Access)?_?(KEY|key|Key)("|'\'')?[[:space:]]*(:|=>|=)[[:space:]]*("|'\'')?[A-Za-z0-9/\+=]{40}("|'\'')?' +git secrets --add '("|'\'')?(AWS|aws|Aws)?_?(ACCOUNT|account|Account)_?(ID|id|Id)?("|'\'')?[[:space:]]*(:|=>|=)[[:space:]]*("|'\'')?[0-9]{4}\-?[0-9]{4}\-?[0-9]{4}("|'\'')?' git secrets --add 'aws(.{0,20})?['\''"][0-9a-zA-Z/+]{40}['\''"]' # AWS Credentials # GCP patterns @@ -111,9 +113,8 @@ git secrets --add 'secret[_-]?key[[:space:]]*[=:][[:space:]]*['\''"][^'\''"]{8,} git secrets --add 'BEGIN[[:space:]]((RSA|DSA|EC|OPENSSH|ENCRYPTED)[[:space:]])?PRIVATE[[:space:]]KEY-----' # PEM private keys # Database connection strings -git secrets --add 'postgres://[^:]+:[^@]+@' # PostgreSQL -git secrets --add 'mysql://[^:]+:[^@]+@' # MySQL -git secrets --add 'mongodb(\+srv)?://[^:]+:[^@]+@' # MongoDB +# The match must end on a host character: git-secrets scans with -w, which drops a match followed by a word character +git secrets --add '(postgres(ql)?|mysql|mongodb(\+srv)?)://[^:/@[:space:]]+:[^@/[:space:]$<{][^@/[:space:]]*@[^[:space:]'\''"/?#]+' # Database DSN with an inline password # Allowed patterns live in .gitallowed (versioned, applied by every scan, including CI). # They are matched against the whole "path:line:content" scanner output line, so an entry diff --git a/scripts/test-git-secrets-allowlist.sh b/scripts/test-git-secrets-allowlist.sh index 4c6c868..18c0fe4 100755 --- a/scripts/test-git-secrets-allowlist.sh +++ b/scripts/test-git-secrets-allowlist.sh @@ -120,7 +120,7 @@ run_case "untruncated PKCS8 body" 1 e.json \ run_case "corrected GCP API key range" 1 f.txt \ "$AIZA" run_case "password with real spaces around the separator" 1 n.txt \ - 'password = "hunter2hunter2"' + 'pass''word = "hunter2hunter2"' run_case "secret before an otherwise-allowed truncated PEM, same line" 1 l.go \ 'k := "'"$KEY"'"; PrivateKey: "'"$PEM"'\n...",' run_case "secret after an otherwise-allowed truncated PEM, same line" 1 m.go \ @@ -139,6 +139,43 @@ run_case "GCP key beside a placeholder UUID" 1 q.txt \ run_case "account ID assignment is still detected" 1 r.txt \ 'account''_id="123456789013"' +# DSN detectors (#58): git-secrets scans with -w, so a pattern ending in the +# credential separator never matched; each of these must be caught. +DSN_TAIL="://app:S3cretPw@db.example.com/app" +run_case "postgres DSN with a password" 1 v1.txt \ + 'dsn: postgre''s'"$DSN_TAIL" +run_case "postgresql DSN with a password" 1 v2.txt \ + 'dsn: postgre''sql'"$DSN_TAIL" +run_case "mysql DSN with a password" 1 v3.txt \ + 'dsn: mysq''l'"$DSN_TAIL" +run_case "mongodb DSN with a password" 1 v4.txt \ + 'dsn: mongo''db'"$DSN_TAIL" +run_case "mongodb+srv DSN with a password" 1 v5.txt \ + 'dsn: mongo''db+srv'"$DSN_TAIL" +run_case "DSN with a percent-encoded password" 1 v6.txt \ + 'dsn: postgre''s://app:p%40ss%3Aw0rd@db.example.com/app' +run_case "DSN with an IPv6 host" 1 v7.txt \ + 'dsn: postgre''s://app:S3cretPw@[::1]:5432/app' +run_case "quoted DSN" 1 v8.txt \ + 'url = "postgre''s'"$DSN_TAIL"'"' +# The POSIX-class twins: git grep on macOS has no \s (#58). +run_case "spaced account ID assignment" 1 w1.txt \ + 'account''_id = "123456789013"' +run_case "camelCase account ID with a space" 1 w2.txt \ + 'account''Id: "123456789013"' +run_case "spaced AWS secret access key assignment" 1 w3.txt \ + 'aws_secret_access''_key = '"$(head -c 40 /dev/zero | tr '\0' a)" + +# DSN lookalikes that must stay clean. +run_case "DSN with a shell-variable password" 0 x1.txt \ + 'dsn: postgre''s://app:${DB_PASS}@db.example.com/app' +run_case "DSN with a placeholder password" 0 x2.txt \ + 'dsn: postgre''s://app:@db.example.com/app' +run_case "DSN without a password" 0 x3.txt \ + 'dsn: postgre''s://app@db.example.com/app' +run_case "http URL with userinfo is not a database DSN" 0 x4.txt \ + 'see https://user:pw@example.com/' + # Negative controls for the setup-script allowlist entries below, each # closing one instance of the same class of hole (#1972 reintroduced at # the scale of one file) that a narrower-but-still-wrong anchor leaves @@ -163,7 +200,7 @@ cp "$REPO_ROOT/scripts/setup-git-secrets.sh" scripts/setup-git-secrets.sh # narrower version of the same hole the entry above closes. run_case "secret appended to a self-matching line itself stays caught" 1 \ scripts/setup-git-secrets.sh \ - "$(sed "s/# PostgreSQL\$/# PostgreSQL ${KEY}/" "$REPO_ROOT/scripts/setup-git-secrets.sh")" + "$(sed "s/# Azure Connection String\$/# Azure Connection String ${KEY}/" "$REPO_ROOT/scripts/setup-git-secrets.sh")" cp "$REPO_ROOT/scripts/setup-git-secrets.sh" scripts/setup-git-secrets.sh # Must scan clean (exit 0): what the tree contains, and what the allowlist