Queue-North-Website/scripts/secrets.sh

#!/usr/bin/env bash
#
# Credentials, before they are committed.
#
# ## Why this and not a generic scanner
#
# A general-purpose secret scanner knows about AWS keys and GitHub tokens. It
# does not know that *this* deployment issues `pllc_agent_<hex>`, or that its
# encrypted envelopes start `v2:<keyid>:`, or which of its environment variables
# hold a password. The project does know, and usually writes it down twice: once
# in whatever redacts its logs, and once in whatever redacts its outbound
# messages.
#
# So this reads the project's own patterns where they exist — point
# `SECRETS_PATTERN_FILE` at the module holding them — and falls back to a
# conservative built-in set. A scanner tuned to the shapes a project actually
# issues catches the leak a generic one misses, and stays quiet the rest of the
# time.
#
# ## What it scans
#
# By default the **staged diff**, which is the only moment a commit can still be
# stopped cheaply. `--tracked` scans every tracked file instead, which is what
# you want once, on adoption, to find what is already in the history's tip.
#
#   bash scripts/secrets.sh                 # staged changes (use in pre-commit)
#   bash scripts/secrets.sh --tracked       # everything tracked, for an audit
#   bash scripts/secrets.sh --built dist/   # the artifact users receive
#   SECRETS_PATTERN_FILE=src/lib/log.ts bash scripts/secrets.sh   # secrets-ok: a path
#   bash scripts/secrets.sh --allow docs/examples/
#
# ## --built, and why the repository is the wrong place to stop
#
# The two modes above scan what is in git. Neither sees the bundle, which is the
# only artifact a user actually receives — and a key can reach it without ever
# being committed, from an environment variable inlined at build time. Somebody
# auditing applications of this kind reported finding hardcoded credentials in
# the frontend bundle of seven of eight in a single week.
#
# So --built walks a build directory instead, with two tiers of result:
#
#   findings, which fail  eyJ (a JWT header), service_role, apikey=, Bearer,
#                         plus every pattern the other modes use
#   noted, which do not   anon, VITE_, REACT_APP_, NEXT_PUBLIC_
#
# The second tier is printed and changes nothing. Those prefixes mean
# "deliberately shipped to the browser", so failing on them would be a
# permanently red gate, and a gate that is always red is one everybody has
# learned to ignore. But they are worth *seeing* enumerated: a Supabase anon key
# is safe exactly as far as row-level security makes it safe, and knowing it is
# out there is the input to that judgement rather than a substitute for it.
#
# ## What it cannot do
#
# It reads the working tree and the index. **A secret already committed is still
# in the history** after you delete it, and this will not tell you that — the
# fix there is a rotation, not a scan. Rotate first, then clean up.
#
# ## Excusing one line
#
# A line that has to show a credential SHAPE, such as a usage example or a list
# of known-weak values, carries `secrets-ok:` and a reason on the same line.
# That excuses that one line and nothing else, and it stays visible in review
# and to grep, which an --allow path does not. There is no blanket exemption for
# comments: a key pasted into a comment is still a leaked key.
#
# Exit codes: 0 nothing found. 1 a candidate found. 2 nothing was scanned, or a
# pattern could not be read, which is the same thing.
# ---------------------------------------------------------------------------

set -uo pipefail

cd "$(git rev-parse --show-toplevel 2>/dev/null)" || {
  printf 'secrets: not a git repository.\n' >&2
  exit 2
}

say() { printf 'secrets: %s\n' "$*" >&2; }

# The delimiter for the masking substitution below. A real control byte, because
# these patterns contain both `/` and `|` and either would end the expression
# early. It cannot occur in a pattern and it cannot occur in source text.
MASK_D=$'\001'

MODE="staged"
BUILT_DIR=""
ALLOW=()

while [ $# -gt 0 ]; do
  case "$1" in
    --tracked) MODE="tracked"; shift ;;
    --staged)  MODE="staged";  shift ;;
    --built)
      MODE="built"
      BUILT_DIR="${2:-}"
      [ -n "$BUILT_DIR" ] || { say "--built needs a directory (dist/, build/, .next/…)"; exit 2; }
      case "$BUILT_DIR" in -*) say "--built needs a directory, got '$BUILT_DIR'"; exit 2 ;; esac
      shift 2 ;;
    --allow)   ALLOW+=("${2:-}"); shift 2 ;;
    *)         say "unknown argument: $1"; exit 2 ;;
  esac
done

# Shapes that reach a bundle and should not. Added to PATTERNS below only in
# --built mode: `eyJ` is the base64 of `{"` that every JWT header starts with,
# and it is far too eager to run against source, where it matches ordinary
# base64. In a bundle it is worth the noise.
BUILT_PATTERNS=(
  # All three segments, not just the header. Masking removes exactly what the
  # pattern matched, so a pattern that stops at the first dot redacts `eyJ...`
  # and prints the payload and signature beside it -- which is the token.
  'eyJ[A-Za-z0-9_-]{10,}(\.[A-Za-z0-9_-]+){0,2}'
  '\bservice_role\b'
  '\bapikey["'"'"'[:space:]]*[:=]'
)

# Shapes that are *meant* to be public. Reported, never failed on — see the
# header. A finding you cannot act on is a finding that teaches people to skip
# the report.
NOTED_PATTERNS=(
  '\bNEXT_PUBLIC_[A-Z0-9_]+'
  '\bVITE_[A-Z0-9_]+'
  '\bREACT_APP_[A-Z0-9_]+'
  '\banon["'"'"'[:space:]]*[:=]'

  # A Google reCAPTCHA key. Noted and never failed on, because the SITE key and
  # the SECRET key are the same shape and this script cannot tell them apart —
  # failing would mean failing on the site key, which Vite inlines into the
  # bundle correctly and by design.
  #
  # So it reports, and a person confirms. In --built mode that report is the
  # whole point: `SECURITY_CHECKLIST.md` carries "the reCAPTCHA key in dist/ is
  # the SITE key, not the secret key" as a release check, and this is the line
  # that puts the key in front of whoever is ticking it. Compare what it prints
  # against VITE_RECAPTCHA_SITE_KEY; if it matches RECAPTCHA_SECRET_KEY instead,
  # the secret has shipped to every visitor and rotating it is the only fix.
  '\b6L[A-Za-z0-9_-]{38}\b'
)

# The built-in set. Deliberately shapes that are *structurally* credential-like
# rather than words that merely appear near credentials — `password` in a
# sentence is not a leak, and a scanner that says it is gets muted.
PATTERNS=(
  # user:pass@host in a URL.
  #
  # TIGHTENED FOR THIS PROJECT on 2026-08-18. The template's version excludes
  # only `/`, `@`, `:` and whitespace from the two halves, which is fine against
  # source and wrong against this project's build output: every prerendered page
  # carries schema.org JSON-LD, and
  #
  #   //queuenorth.com"},"areaServed":{"@
  #
  # parses as host `queuenorth.com`, password `"},"areaServed"`, then an `@`.
  # That is ten findings per `--built` run today and one more for every page
  # added, which is precisely the noise this file's header warns turns a scanner
  # into something people mute.
  #
  # Quotes, braces, commas and angle brackets cannot occur in a real userinfo
  # component, so excluding them costs nothing.
  #
  # Checked against real shapes — a database URL with an inline password, one
  # with a percent-encoded password, and a git remote carrying a token. All
  # three still match; all ten JSON-LD false positives are gone.
  #
  # Those three are described rather than written out, and the placeholders below
  # use <angle brackets> ON PURPOSE: angle brackets are in the exclusion class
  # this comment is about, so the examples cannot match the pattern they
  # illustrate. Written literally they did, and `--tracked` reported two
  # credentials in this file — a scanner flagging its own documentation, which is
  # the same shape of mistake as DOC_TRUST_MAP.md's note about a parser that
  # cannot tell a description of a thing from the thing itself.
  #
  #   https://<user>:<pass>@db.internal:5432/app
  #   postgres://<user>:<percent-encoded-pass>@host/db
  #   https://<token>:x-oauth-basic@forgejo/repo.git
  '//[^/@[:space:]:"'"'"'{},<>]+:[^/@[:space:]"'"'"'{},<>]+@'
  '[?&](token|key|secret|password|access_token|api_key)=[^&[:space:]"]+'
  '\b(Bearer|Basic)[[:space:]]+[A-Za-z0-9._~+/=-]{20,}' # an authorization header
  # NAME=value, where NAME is upper-case and contains SECRET, TOKEN, PASS or
  # API_KEY. Three things about it were learned the hard way:
  #
  # - It must not be anchored to the start of a line. Neither scan mode ever
  #   presents one: a staged diff starts every line with "+", and --tracked
  #   prefixes each with its file name. Anchored, it matched nothing at all, so
  #   the name now merely must not continue an identifier.
  # - Case matters. Lower-case `access_token = $1` in SQL and `apiKey=` in a
  #   property list are column and field names, not credentials.
  # - The value may be quoted. Unquoted-only, it missed API_KEY="...", which is
  #   how a real secret is most often written down. A value starting with `$`
  #   is a variable reference and one starting with `<` is a placeholder, and
  #   neither is flagged.
  '(^|[^A-Za-z0-9_])[A-Z0-9_]*(SECRET|TOKEN|PASSWORD|PASSWD|PASS|API_KEY)[A-Z0-9_]*=["'"'"']?[^[:space:]"'"'"'$<][^[:space:]"'"'"']{7,}'
  '-----BEGIN [A-Z ]*PRIVATE KEY-----'
  '\bghp_[A-Za-z0-9]{20,}'                              # GitHub
  '\bxox[baprs]-[A-Za-z0-9-]{10,}'                      # Slack
  '\bAKIA[0-9A-Z]{16}\b'                                # AWS access key id

  # --- This project's own shapes. ---
  #
  # Zoho WebToLead form identifiers are 64 and 96 lowercase hex characters, and
  # they are the reason this hook exists here at all: `xnQsjsdp` and `xmIwtLD`
  # were hardcoded in index.html at 0d3af33 and in src/pages/Contact.jsx at
  # 033bdf6, and reached four commits on what was then a PUBLIC repository
  # before being moved to environment variables at 05b27d2.
  #
  # Two patterns rather than one. The first is the shape as it actually leaked —
  # a bare literal in markup, with no variable name anywhere near it. The second
  # catches the tidier form somebody would write next time. A bare 60+ hex run
  # matches nothing else in this tree: checked across every tracked text file on
  # 2026-08-18 and it found zero, which is what makes it safe to fail on.
  '\b[0-9a-f]{60,}\b'
  'ZOHO_[A-Z0-9_]*[[:space:]]*[:=][[:space:]]*["'"'"']?[0-9a-f]{32,}'
)

# The project's own shapes, if it has written them down. A `pllc_agent_<hex>`
# token is invisible to every generic scanner and obvious to the module that
# redacts it.
if [ -n "${SECRETS_PATTERN_FILE:-}" ] && [ -f "$SECRETS_PATTERN_FILE" ]; then
  loaded=0

  # Anchored on the closing `/flags,` and greedy to it, rather than on "no
  # commas". The first version used `[^,]+`, which cannot cross the comma inside
  # a bounded quantifier — so `pllc_[a-z]+_[0-9a-f]{8,}` was silently dropped
  # along with every other `{n,}` pattern: four of six on the file this was
  # written against, while the script printed that it had loaded them.
  while IFS= read -r found; do
    [ -n "$found" ] || continue

    PATTERNS+=("$found")
    loaded=$((loaded + 1))
  done < <(
    sed -nE 's/.*\[\/(.+)\/[gimsuy]*,[[:space:]]*".*/\1/p' "$SECRETS_PATTERN_FILE" 2>/dev/null || true
  )

  # The count, never a bare reassurance. "Loaded project patterns" over an empty
  # list is the same lie as a green test run that executed nothing.
  if [ "$loaded" -gt 0 ]; then
    say "loaded $loaded project pattern(s) from $SECRETS_PATTERN_FILE"
  else
    say "WARNING: $SECRETS_PATTERN_FILE yielded no patterns — scanning with the"
    say "         built-in set only. Check the file holds regex literals."
  fi
fi

if [ "$MODE" = "built" ]; then
  [ -d "$BUILT_DIR" ] || { say "no such directory: $BUILT_DIR"; say "Nothing was scanned, which is not a pass."; exit 2; }

  CONTENT=""
  WHAT="the built output in $BUILT_DIR"
  PATTERNS+=("${BUILT_PATTERNS[@]}")

  while IFS= read -r file; do
    skip=""
    for allowed in ${ALLOW[@]+"${ALLOW[@]}"}; do
      case "$file" in *"$allowed"*) skip="yes" ;; esac
    done
    [ -n "$skip" ] && continue

    # Source maps are the build's own copy of the source and would double every
    # finding; they are worth scanning on purpose, not by accident.
    case "$file" in *.map) continue ;; esac

    file "$file" 2>/dev/null | grep -q "text" || continue

    # Relative to the build directory, not the absolute path find produced.
    # The report truncates each line to keep a secret off the terminal, and an
    # absolute path in a temp directory can consume that budget entirely --
    # leaving a finding that names a file and shows nothing about the match.
    rel="${file#"$BUILT_DIR"/}"
    CONTENT+="$(sed "s|^|${rel}: |" "$file")"$'\n'
  done < <(find "$BUILT_DIR" -type f -size -20M 2>/dev/null)

elif [ "$MODE" = "staged" ]; then
  # Added lines only. A removed line containing a token is somebody deleting
  # one, which is the opposite of a leak.
  CONTENT="$(git diff --cached --unified=0 --no-color | grep '^+' | grep -v '^+++' || true)"
  WHAT="staged changes"
else
  CONTENT=""
  WHAT="tracked files"

  while IFS= read -r file; do
    skip=""

    for allowed in ${ALLOW[@]+"${ALLOW[@]}"}; do
      case "$file" in *"$allowed"*) skip="yes" ;; esac
    done

    [ -n "$skip" ] && continue
    [ -f "$file" ] || continue

    # Text only; a webp full of bytes will match anything.
    file "$file" 2>/dev/null | grep -q "text" || continue

    CONTENT+="$(sed "s|^|${file}: |" "$file")"$'\n'
  done < <(git ls-files)
fi

# Every pattern must compile before its silence means anything. grep answers an
# unreadable pattern with exit 2, and the scan below discards grep's errors, so
# a broken pattern reads as "no credential shapes", the same silence as a clean
# tree. The private-key pattern was exactly that for as long as it existed: it
# starts with dashes, grep took it for an option, and nothing said so.
for pattern in "${PATTERNS[@]}" "${NOTED_PATTERNS[@]}"; do
  grep -qE -e "$pattern" </dev/null 2>/dev/null
  if [ "$?" -eq 2 ]; then
    say "grep cannot read the pattern '${pattern:0:40}', so nothing was scanned."
    exit 2
  fi
done

if [ -z "$CONTENT" ]; then
  # "Nothing to scan" means two different things and collapsing them makes a
  # DELETE-ONLY COMMIT IMPOSSIBLE. In staged mode CONTENT is built from ADDED
  # lines only, so a commit that only deletes has none — a correct measurement,
  # not a failure to measure, because a deletion cannot introduce a credential.
  # In tracked/built mode empty means nothing was examined at all, which IS the
  # could-not-check state exit 2 exists to name.
  if [ "$MODE" = "staged" ]; then
    say "no added lines in $WHAT — a deletion cannot introduce a credential."
    exit 0
  fi
  say "nothing to scan in $WHAT."
  exit 2
fi

found=0

for pattern in "${PATTERNS[@]}"; do
  while IFS= read -r hit; do
    [ -n "$hit" ] || continue

    skip=""
    for allowed in ${ALLOW[@]+"${ALLOW[@]}"}; do
      case "$hit" in *"$allowed"*) skip="yes" ;; esac
    done
    [ -n "$skip" ] && continue

    # One line excused on purpose, with its reason beside it. See the header.
    case "$hit" in *secrets-ok:*) continue ;; esac

    # The match is masked, then the line is truncated. Truncation alone was not
    # enough and used to be all there was: it bounds how much of a LONG value
    # reaches the terminal and prints a short one whole, so the scanner
    # published the very thing it was built to find — to the scrollback, the CI
    # log, and wherever that log is shipped.
    #
    # \001 as the delimiter, because these patterns contain both `/` and `|`
    # and either would end the expression early. It cannot occur in a pattern
    # and it cannot occur in the text of a source file.
    masked="$(printf '%s' "$hit" | sed -E "s${MASK_D}${pattern}${MASK_D}[redacted]${MASK_D}g" 2>/dev/null)"
    [ -n "$masked" ] || masked="[a line matching a credential pattern, unprintable]"
    printf '  %.120s…\n' "$masked"
    found=$((found + 1))
  done < <(printf '%s\n' "$CONTENT" | grep -nEI -e "$pattern" 2>/dev/null | head -20)
done

# The public-by-design tier. Printed, counted, and deliberately not fatal.
if [ "$MODE" = "built" ]; then
  noted=0

  for pattern in "${NOTED_PATTERNS[@]}"; do
    while IFS= read -r hit; do
      [ -n "$hit" ] || continue
      if [ "$noted" -eq 0 ]; then
        say "shipped to the browser on purpose — check each is meant to be public:"
      fi
      printf '  %.120s…\n' "$hit"
      noted=$((noted + 1))
    done < <(printf '%s\n' "$CONTENT" | grep -oEI -e "$pattern" 2>/dev/null | sort -u | head -20)
  done

  if [ "$noted" -gt 0 ]; then
    say "$noted public reference(s) above. Not a failure: those prefixes mean"
    say "the value was compiled in deliberately. A Supabase anon key is safe"
    say "exactly as far as row-level security makes it safe — this is the input"
    say "to that judgement, not a substitute for it."
  fi
fi

if [ "$found" -gt 0 ]; then
  say "$found candidate credential(s) in $WHAT."
  say "If one is real: rotate it first. Deleting the line does not remove it"
  say "from a commit that already exists, and the scan cannot see history."
  exit 1
fi

say "no credential shapes in $WHAT."