From c6aa2dcf3b26922195fd10834da37f23e70ac1d2 Mon Sep 17 00:00:00 2001 From: Ismail Rhoulam Date: Sun, 12 Jul 2026 14:30:48 +0100 Subject: [PATCH 1/4] =?UTF-8?q?wip:=20auto-checkpoint=20=E2=80=94=20120=20?= =?UTF-8?q?lines=20/=209=20files=20(2026-07-12T14:30:48+0100)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 2 + ctxguard/detectors.py | 75 +++++++++++++++++++++++++++ tests/conftest.py | 30 +++++++++-- tests/fixtures/clean/placeholders.env | 5 ++ tests/fixtures/secrets/cloud.txt | 3 ++ tests/fixtures/secrets/dburls.txt | 3 ++ tests/fixtures/secrets/gcp_sa.json | 6 +++ tests/fixtures/secrets/saas.txt | 5 ++ tests/test_detectors.py | 8 +++ 9 files changed, 134 insertions(+), 3 deletions(-) create mode 100644 tests/fixtures/secrets/cloud.txt create mode 100644 tests/fixtures/secrets/dburls.txt create mode 100644 tests/fixtures/secrets/gcp_sa.json create mode 100644 tests/fixtures/secrets/saas.txt diff --git a/.gitignore b/.gitignore index f4e1db3..b2b0f57 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,5 @@ htmlcov/ # ctxguard local config (the example ships instead) .ctxguard.toml +# local task notes +goal.md diff --git a/ctxguard/detectors.py b/ctxguard/detectors.py index d4ae71c..2e2c339 100644 --- a/ctxguard/detectors.py +++ b/ctxguard/detectors.py @@ -142,6 +142,11 @@ def shannon_entropy(value: str) -> float: "unset", "disabled", "string", + "pass", + "pwd", + "1234", + "12345", + "123456", } _PLACEHOLDER_SUBSTRINGS = ( @@ -216,6 +221,14 @@ def _plausible_env_value(value: str, m: "re.Match") -> bool: return shannon_entropy(v) >= 3.0 +def _plausible_db_password(value: str, m: "re.Match") -> bool: + # any real inline password in a connection URL is worth flagging; only + # obvious docs placeholders (user:password@, user:pass@, ${VAR}) get a pass + if is_placeholder(value): + return False + return shannon_entropy(value) >= 2.0 + + def _high_entropy_value(value: str, m: "re.Match") -> bool: v = value.strip() if len(v) < 16 or is_placeholder(v): @@ -283,6 +296,68 @@ class _Spec: 1, _plausible_token, ), + _Spec( + "gcp_api_key", + re.compile(r"\b(AIza[0-9A-Za-z_\-]{35})(?![0-9A-Za-z_\-])"), + "high", + 1, + _plausible_token, + ), + _Spec( + "gcp_service_account", + re.compile(r"[\"']type[\"']\s*:\s*[\"']service_account[\"']"), + "high", + 0, + None, + ), + _Spec( + "azure_storage_account_key", + re.compile(r"(?i)\bAccountKey=([A-Za-z0-9+/]{60,}={0,2})"), + "high", + 1, + _plausible_token, + ), + _Spec( + "twilio_api_key", + re.compile(r"\b(SK[0-9a-fA-F]{32})(?![0-9a-fA-F])"), + "high", + 1, + _plausible_token, + ), + _Spec( + "sendgrid_api_key", + re.compile(r"\b(SG\.[A-Za-z0-9_\-]{22}\.[A-Za-z0-9_\-]{43})(?![A-Za-z0-9_\-])"), + "high", + 1, + _plausible_token, + ), + _Spec( + "anthropic_api_key", + re.compile(r"\b(sk-ant-[A-Za-z0-9_\-]{32,})\b"), + "high", + 1, + _plausible_token, + ), + _Spec( + "openai_api_key", + re.compile( + r"\b(sk-(?:proj|svcacct|admin)-[A-Za-z0-9_\-]{32,}" + r"|sk-[A-Za-z0-9]{48}(?![A-Za-z0-9]))" + ), + "high", + 1, + _plausible_token, + ), + _Spec( + "database_url_password", + re.compile( + r"(?i)\b(?:postgres(?:ql)?|mysql|mariadb|mongodb(?:\+srv)?|rediss?" + r"|amqps?|mssql)://[^\s:@/]+:([^\s:@/]{4,})@" + ), + "high", + 1, + _plausible_db_password, + ), _Spec( "private_key_block", re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY(?: BLOCK)?-----"), diff --git a/tests/conftest.py b/tests/conftest.py index 050eed3..b56c967 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -36,6 +36,20 @@ API_TOKEN_VAL = "8fk2mZpQ7wR4" + "xV1bN6cY3sD5" STRIPE_KEY_VAL = "whsec_F8kL2mP9" + "qR4sT7vX1yZ3bC6d" +GCP_API_VAL = "AIzaSyQ9w8E7r6T5y4U3i2" + "O1p0Q9w8E7r6T5y0k" +AZURE_ACCOUNT_VAL = ( + "aB1cD2eF3gH4iJ5kL6mN7oP8qR9sT0uVwX1yZ2aB3cD4eF5g" + + "H6iJ7kL8mN9oP0qR1sT2uV3wX4yZ5aB6cD7eF8g==" +) +TWILIO_VAL = "SK9f8e7d6c5b4a3928" + "1706f5e4d3c2b1a0" +SENDGRID_VAL = ( + "SG.aB1cD2eF3gH4iJ5kL6mN7o" + ".pQ8rS9tU0vW1xY2zA3bC4dE5fG6hI7jK8lM9nO0pQ1r" +) +OPENAI_VAL = "sk-aB1cD2eF3gH4iJ5kL6mN7oP8" + "qR9sT0uV1wX2yZ3aB4cD5eF6" +ANTHROPIC_VAL = "sk-ant-api03-qW3eR5tY7uI9oP1a" + "S2dF4gH6jK8lZ0xC5vB7nM9q" +PG_URL_PW = "v8Kq2Lw9" + "Xt4z" +MONGO_URL_PW = "Zp4tK9qX" + "2mR7w" + RAW_SECRETS = [ AWS_ID, AWS_SECRET, @@ -49,11 +63,21 @@ DB_PASSWORD_VAL, API_TOKEN_VAL, STRIPE_KEY_VAL, + GCP_API_VAL, + AZURE_ACCOUNT_VAL, + TWILIO_VAL, + SENDGRID_VAL, + OPENAI_VAL, + ANTHROPIC_VAL, + PG_URL_PW, + MONGO_URL_PW, ] # Expected totals when scanning tests/fixtures/secrets: # aws.txt 2, github.txt 2, slack.txt 1, stripe.txt 2, keys.txt 1, jwt.txt 1, # dotenv/.env 4 (1 filename + 3 env), settings.py 1, -# dummy.pem 1, credentials.json 1, id_rsa 1 (filename-only) -EXPECTED_SECRET_FINDINGS = 17 -EXPECTED_FLAGGED_FILES = 11 +# dummy.pem 1, credentials.json 1, id_rsa 1 (filename-only), +# gcp_sa.json 2 (type marker + private_key_id), cloud.txt 2 (gcp + azure), +# saas.txt 4 (twilio, sendgrid, openai, anthropic), dburls.txt 2 +EXPECTED_SECRET_FINDINGS = 27 +EXPECTED_FLAGGED_FILES = 15 diff --git a/tests/fixtures/clean/placeholders.env b/tests/fixtures/clean/placeholders.env index b713750..2fb7f17 100644 --- a/tests/fixtures/clean/placeholders.env +++ b/tests/fixtures/clean/placeholders.env @@ -9,3 +9,8 @@ AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY STRIPE_KEY=sk_live_XXXXXXXXXXXXXXXXXXXXXXXX GITHUB_TOKEN=ghp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +GOOGLE_API_KEY=AIzaXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +TWILIO_API_KEY=SK00000000000000000000000000000000 +SENDGRID_API_KEY=SG.XXXXXXXXXXXXXXXXXXXXXX.XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +OPENAI_API_KEY=sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +DATABASE_URL=postgres://user:password@localhost:5432/dev diff --git a/tests/fixtures/secrets/cloud.txt b/tests/fixtures/secrets/cloud.txt new file mode 100644 index 0000000..f63959b --- /dev/null +++ b/tests/fixtures/secrets/cloud.txt @@ -0,0 +1,3 @@ +cloud credentials (fixture, fake but format-valid) +google maps key for staging: AIzaSyQ9w8E7r6T5y4U3i2O1p0Q9w8E7r6T5y0k +AZURE_STORAGE_CONNECTION_STRING=DefaultEndpointsProtocol=https;AccountName=fixtureacct;AccountKey=aB1cD2eF3gH4iJ5kL6mN7oP8qR9sT0uVwX1yZ2aB3cD4eF5gH6iJ7kL8mN9oP0qR1sT2uV3wX4yZ5aB6cD7eF8g==;EndpointSuffix=core.windows.net diff --git a/tests/fixtures/secrets/dburls.txt b/tests/fixtures/secrets/dburls.txt new file mode 100644 index 0000000..38082ec --- /dev/null +++ b/tests/fixtures/secrets/dburls.txt @@ -0,0 +1,3 @@ +connection strings (fixture, fake credentials) +DATABASE_URL=postgres://svc_app:v8Kq2Lw9Xt4z@db.internal:5432/appdb +MONGO_URI=mongodb+srv://appuser:Zp4tK9qX2mR7w@cluster0.fixture.mongodb.net/prod diff --git a/tests/fixtures/secrets/gcp_sa.json b/tests/fixtures/secrets/gcp_sa.json new file mode 100644 index 0000000..7df7e6c --- /dev/null +++ b/tests/fixtures/secrets/gcp_sa.json @@ -0,0 +1,6 @@ +{ + "type": "service_account", + "project_id": "fixture-project", + "private_key_id": "0f1e2d3c4b5a69788796a5b4c3d2e1f001234567", + "client_email": "svc@fixture-project.iam.gserviceaccount.com" +} diff --git a/tests/fixtures/secrets/saas.txt b/tests/fixtures/secrets/saas.txt new file mode 100644 index 0000000..2fb1454 --- /dev/null +++ b/tests/fixtures/secrets/saas.txt @@ -0,0 +1,5 @@ +saas api keys (fixture, fake but format-valid) +twilio api key sid SK9f8e7d6c5b4a39281706f5e4d3c2b1a0 +sendgrid mailer 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 +openai for the bot sk-aB1cD2eF3gH4iJ5kL6mN7oP8qR9sT0uV1wX2yZ3aB4cD5eF6 +anthropic for eval sk-ant-api03-qW3eR5tY7uI9oP1aS2dF4gH6jK8lZ0xC5vB7nM9q diff --git a/tests/test_detectors.py b/tests/test_detectors.py index 9e60ef8..4ff26dc 100644 --- a/tests/test_detectors.py +++ b/tests/test_detectors.py @@ -3,20 +3,28 @@ import pytest from conftest import ( + ANTHROPIC_VAL, AWS_ID, AWS_SECRET, + AZURE_ACCOUNT_VAL, CLEAN_DIR, DB_PASSWORD_VAL, ENTROPY_VAL, EXPECTED_SECRET_FINDINGS, + GCP_API_VAL, GH_PAT, GHP, JWT, + MONGO_URL_PW, + OPENAI_VAL, + PG_URL_PW, RAW_SECRETS, SECRETS_DIR, + SENDGRID_VAL, SLACK, STRIPE_LIVE, STRIPE_TEST, + TWILIO_VAL, ) from ctxguard import detectors as det From 316513e9033390dafa2beedd86b8f2d08753309b Mon Sep 17 00:00:00 2001 From: Ismail Rhoulam Date: Sun, 12 Jul 2026 14:32:27 +0100 Subject: [PATCH 2/4] feat(detectors): GCP, Azure, Twilio, SendGrid, OpenAI-style, and DB URL patterns New high-confidence detectors: GCP API keys (AIza...) and service-account JSON content markers, Azure storage AccountKey values, Twilio API key SIDs, SendGrid keys, OpenAI-style sk-/sk-proj- keys, Anthropic sk-ant- keys, and database URLs with inline passwords (postgres, mysql, mariadb, mongodb, redis, amqp, mssql). Placeholder rejection extended (pass, pwd, 123456). Fixtures and positive/negative tests for each; fixture totals now 27 findings across 15 files. --- README.md | 8 ++++ tests/test_detectors.py | 104 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+) diff --git a/README.md b/README.md index b6d491f..4b3e515 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,14 @@ Run the synthetic demonstration with `python3 scripts/demo.py`. See - GitHub classic and fine-grained personal access tokens - Slack tokens - Stripe live and test secret keys +- GCP API keys (`AIza...`) and service-account JSON (content marker plus the filename rule) +- Azure storage, Service Bus, Event Hub, and IoT Hub connection string keys + (`AccountKey=...`, `SharedAccessKey=...`) +- Twilio API key SIDs and SendGrid API keys +- OpenAI-style keys (`sk-...`, `sk-proj-...`) and Anthropic keys (`sk-ant-...`) +- Database URLs with inline passwords: Postgres, MySQL/MariaDB (including + SQLAlchemy dialect+driver and Rails `mysql2://` schemes), MongoDB, Redis, + AMQP, and MSSQL - private key blocks and JWT-shaped strings - credential-like assignments that pass length, digit, and entropy thresholds - high-entropy values assigned to names containing `key`, `secret`, `token`, diff --git a/tests/test_detectors.py b/tests/test_detectors.py index 4ff26dc..ea26428 100644 --- a/tests/test_detectors.py +++ b/tests/test_detectors.py @@ -124,6 +124,110 @@ def test_header_only_negative(self): assert det.scan_text("eyJhbGciOiJIUzI1NiJ9") == [] +class TestGcp: + def test_api_key_positive(self): + findings = det.scan_text(f"maps key {GCP_API_VAL}\n") + assert names(findings) == ["gcp_api_key"] + + def test_api_key_placeholder_negative(self): + assert det.scan_text("AIza" + "X" * 35) == [] + + def test_service_account_marker_positive(self): + marker = '"type": ' + '"service_account"' + assert names(det.scan_text(marker)) == ["gcp_service_account"] + + def test_service_account_other_type_negative(self): + assert det.scan_text('"type": "authorized_user_config"') == [] + + +class TestAzure: + def test_connection_string_positive(self): + text = f"DefaultEndpointsProtocol=https;AccountKey={AZURE_ACCOUNT_VAL};x=y" + findings = det.scan_text(text) + assert names(findings) == ["azure_storage_account_key"] + assert AZURE_ACCOUNT_VAL not in str(findings[0]) + + def test_low_entropy_negative(self): + assert det.scan_text("AccountKey=" + "A" * 70) == [] + + +class TestTwilio: + def test_positive(self): + assert names(det.scan_text(f"sid {TWILIO_VAL}\n")) == ["twilio_api_key"] + + def test_zeroed_negative(self): + assert det.scan_text("SK" + "0" * 32) == [] + + def test_too_short_negative(self): + assert det.scan_text("SKa1b2c3") == [] + + +class TestSendGrid: + def test_positive(self): + assert names(det.scan_text(SENDGRID_VAL)) == ["sendgrid_api_key"] + + def test_placeholder_negative(self): + fake = "SG." + "X" * 22 + "." + "X" * 43 + assert det.scan_text(fake) == [] + + def test_wrong_shape_negative(self): + assert det.scan_text("SG.short.token") == [] + + +class TestOpenAiStyle: + def test_classic_positive(self): + assert names(det.scan_text(OPENAI_VAL)) == ["openai_api_key"] + + def test_project_key_positive(self): + proj = "sk-proj-" + OPENAI_VAL[3:] + assert names(det.scan_text(proj)) == ["openai_api_key"] + + def test_anthropic_positive(self): + assert names(det.scan_text(ANTHROPIC_VAL)) == ["anthropic_api_key"] + + def test_placeholder_negative(self): + assert det.scan_text("sk-" + "x" * 48) == [] + + def test_too_short_negative(self): + assert det.scan_text("sk-abc123") == [] + + +class TestDatabaseUrl: + @pytest.mark.parametrize( + "url", + [ + "postgres://svc:{pw}@db.internal:5432/app", + "postgresql://svc:{pw}@db/app", + "mysql://app:{pw}@10.0.0.5/x", + "mongodb+srv://appuser:{pw}@cluster0.example.net/prod", + "redis://default:{pw}@cache:6379/0", + "amqp://worker:{pw}@mq:5672/vhost", + ], + ) + def test_inline_password_positive(self, url): + findings = det.scan_text(url.format(pw=PG_URL_PW)) + assert names(findings) == ["database_url_password"] + assert PG_URL_PW not in str(findings[0]) + + def test_mongo_positive(self): + url = f"mongodb://svc:{MONGO_URL_PW}@db/x" + assert names(det.scan_text(url)) == ["database_url_password"] + + @pytest.mark.parametrize( + "url", + [ + "postgres://user:password@localhost:5432/dev", + "mysql://root:pass@127.0.0.1/x", + "postgresql://u:${DB_PASSWORD}@h/db", + "postgres://user@localhost/db", + "postgres://localhost:5432/db", + "https://example.com/path", + ], + ) + def test_placeholder_or_no_password_negative(self, url): + assert det.scan_text(url) == [] + + class TestEnvAssignment: def test_positive(self): findings = det.scan_text(f"DB_PASSWORD={DB_PASSWORD_VAL}\n") From 3529cb2c444ede1254d1bd115776d4a443ee08a4 Mon Sep 17 00:00:00 2001 From: Ismail Rhoulam Date: Sun, 12 Jul 2026 14:34:32 +0100 Subject: [PATCH 3/4] fix(detectors): treat {var} and $VAR wrapped values as placeholders Brace-wrapped template values (postgres://u:{password}@h) and bare shell variable references (:$PGPASSWORD@) are interpolation placeholders, not credentials; the DB URL detector was flagging them. --- ctxguard/detectors.py | 2 +- tests/test_detectors.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ctxguard/detectors.py b/ctxguard/detectors.py index 2e2c339..3d47243 100644 --- a/ctxguard/detectors.py +++ b/ctxguard/detectors.py @@ -168,7 +168,7 @@ def shannon_entropy(value: str) -> float: "****", ) -_PLACEHOLDER_WRAPPED = re.compile(r"^(<.*>|\$\{.*\}|\{\{.*\}\}|%.*%|__.*__|\[.*\])$") +_PLACEHOLDER_WRAPPED = re.compile(r"^(<.*>|\$\{.*\}|\{.*\}|%.*%|__.*__|\[.*\]|\$\w+)$") def is_placeholder(value: str) -> bool: diff --git a/tests/test_detectors.py b/tests/test_detectors.py index ea26428..cb3ba07 100644 --- a/tests/test_detectors.py +++ b/tests/test_detectors.py @@ -219,6 +219,8 @@ def test_mongo_positive(self): "postgres://user:password@localhost:5432/dev", "mysql://root:pass@127.0.0.1/x", "postgresql://u:${DB_PASSWORD}@h/db", + "postgres://u:{password}@h/db", + "postgres://u:$PGPASSWORD@h/db", "postgres://user@localhost/db", "postgres://localhost:5432/db", "https://example.com/path", From 17035e24df747518d296d1e3f01c809ee5a92c4b Mon Sep 17 00:00:00 2001 From: Ismail Rhoulam Date: Sun, 12 Jul 2026 17:39:17 +0100 Subject: [PATCH 4/4] fix(detectors): close coverage gaps found by adversarial review Confirmed by review workflow, verified empirically before and after: - database_url_password missed SQLAlchemy dialect+driver schemes (postgresql+psycopg2://, mysql+pymysql://, mssql+pyodbc://) and Rails' mysql2://; scheme alternation now accepts an optional +driver suffix. - database_url_password missed empty-username URLs (redis://:pw@host), the canonical Redis/Mongo auth form; username segment now optional. - azure_storage_account_key missed SharedAccessKey= (Service Bus, Event Hubs, IoT Hub connection strings use this instead of AccountKey=). Also fixed two false positives surfaced during manual verification: - The well-known, publicly documented Azurite/Storage-Emulator default key is now excluded via is_placeholder() (shared, so every detector respects it, not just the one whose regex happens to capture it - AccountKey also matches the generic env_assignment name filter). - gcp_service_account no longer fires on the bare type marker alone (common in docs/example snippets); now requires private_key evidence nearby, which every real service-account credential file has. README honest-limitations section documents the remaining deliberate tradeoffs: any non-placeholder DB password is flagged regardless of strength, and GCP API keys are flagged even though some (Firebase web config) are meant to be client-visible. --- README.md | 13 +++++++++ ctxguard/detectors.py | 45 ++++++++++++++++++++++++----- tests/test_detectors.py | 63 ++++++++++++++++++++++++++++++++++++++++- 3 files changed, 113 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 4b3e515..5050196 100644 --- a/README.md +++ b/README.md @@ -153,6 +153,19 @@ ctxguard --version preserves availability but means a broken hook does not provide protection. - Native Windows compatibility is not claimed; current testing covers the Python package and hook contract on Unix-like CI runners. +- Any non-placeholder password inline in a database URL is flagged, including + weak or "demo" ones (`hunter2`, `letmein123`). This is intentional: a weak + credential is still a working credential, and ctxguard can't judge which + ones are safe to expose. Allowlist known-safe fixtures in `.ctxguard.toml` + instead. +- GCP `AIza...`-style API keys are flagged even though some (notably Firebase + web config keys) are designed to be usable client-side. Restrict such keys + by API and referrer in the Google Cloud Console rather than relying on + ctxguard to tell intent apart; allowlist the specific file if it's noisy. +- A handful of well-known, publicly documented development constants (e.g. + the Azurite/Azure Storage Emulator default key) are excluded by name since + they're identical on every machine and not secrets. This list is small and + only grows for equally unambiguous cases. Use layered controls: least-privilege credentials, a secrets manager, `.gitignore`, repository secret scanning, short-lived tokens, and revocation. diff --git a/ctxguard/detectors.py b/ctxguard/detectors.py index 3d47243..a1a78bd 100644 --- a/ctxguard/detectors.py +++ b/ctxguard/detectors.py @@ -170,11 +170,25 @@ def shannon_entropy(value: str) -> float: _PLACEHOLDER_WRAPPED = re.compile(r"^(<.*>|\$\{.*\}|\{.*\}|%.*%|__.*__|\[.*\]|\$\w+)$") +# Well-known, publicly documented development keys that are constant across +# every install and are not secrets. Checked from is_placeholder() so every +# detector (not just the one whose regex happens to capture them) rejects +# them consistently, regardless of which pattern matches the surrounding text. +_KNOWN_BENIGN_VALUES = { + # Azurite / Azure Storage Emulator default account key (Microsoft docs) + ( + "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/" + "K1SZFPTOtr/KBHBeksoGMGw==" + ).lower(), +} + def is_placeholder(value: str) -> bool: v = value.strip().strip("\"'").lower() if not v or v in _PLACEHOLDER_EXACT: return True + if v in _KNOWN_BENIGN_VALUES: + return True if _PLACEHOLDER_WRAPPED.match(v): return True if any(s in v for s in _PLACEHOLDER_SUBSTRINGS): @@ -222,13 +236,30 @@ def _plausible_env_value(value: str, m: "re.Match") -> bool: def _plausible_db_password(value: str, m: "re.Match") -> bool: - # any real inline password in a connection URL is worth flagging; only - # obvious docs placeholders (user:password@, user:pass@, ${VAR}) get a pass + # any real inline password in a connection URL is worth flagging, even a + # weak one: the point is that no credential (strong or not) belongs + # inline in a URL that might enter an AI's context. only obvious docs + # placeholders (user:password@, user:pass@, ${VAR}) get a pass. if is_placeholder(value): return False return shannon_entropy(value) >= 2.0 +def _plausible_azure_key(value: str, m: "re.Match") -> bool: + if is_placeholder(value): + return False + return _plausible_token(value, m) + + +def _plausible_gcp_service_account(value: str, m: "re.Match") -> bool: + # the bare service-account type marker alone also appears in docs and + # example snippets; require a private key nearby, which every real GCP + # service-account credential file has right next to it. + text = m.string + window = text[max(0, m.start() - 2000) : m.start() + 4000] + return "private_key" in window + + def _high_entropy_value(value: str, m: "re.Match") -> bool: v = value.strip() if len(v) < 16 or is_placeholder(v): @@ -308,14 +339,14 @@ class _Spec: re.compile(r"[\"']type[\"']\s*:\s*[\"']service_account[\"']"), "high", 0, - None, + _plausible_gcp_service_account, ), _Spec( "azure_storage_account_key", - re.compile(r"(?i)\bAccountKey=([A-Za-z0-9+/]{60,}={0,2})"), + re.compile(r"(?i)\b(?:Account|SharedAccess)Key=([A-Za-z0-9+/]{40,}={0,2})"), "high", 1, - _plausible_token, + _plausible_azure_key, ), _Spec( "twilio_api_key", @@ -351,8 +382,8 @@ class _Spec: _Spec( "database_url_password", re.compile( - r"(?i)\b(?:postgres(?:ql)?|mysql|mariadb|mongodb(?:\+srv)?|rediss?" - r"|amqps?|mssql)://[^\s:@/]+:([^\s:@/]{4,})@" + r"(?i)\b(?:postgres(?:ql)?|mysql2?|mariadb|mongodb(?:\+srv)?|rediss?" + r"|amqps?|mssql)(?:\+[a-z0-9_]+)?://[^\s:@/]*:([^\s:@/]{4,})@" ), "high", 1, diff --git a/tests/test_detectors.py b/tests/test_detectors.py index cb3ba07..bbd162f 100644 --- a/tests/test_detectors.py +++ b/tests/test_detectors.py @@ -133,12 +133,30 @@ def test_api_key_placeholder_negative(self): assert det.scan_text("AIza" + "X" * 35) == [] def test_service_account_marker_positive(self): - marker = '"type": ' + '"service_account"' + # the bare marker alone is no longer sufficient (see + # test_marker_without_private_key_negative below); a real service + # account file always carries private_key right alongside it. + # Built via concatenation so this test file's own self-scan stays + # clean (same convention as the RAW_SECRETS constants in conftest.py). + marker = '"type": "service' + '_account", "private' + '_key_id": "abc123def456"' assert names(det.scan_text(marker)) == ["gcp_service_account"] def test_service_account_other_type_negative(self): assert det.scan_text('"type": "authorized_user_config"') == [] + def test_marker_without_private_key_negative(self): + # bare type marker in docs/example snippets, no key material nearby + assert ( + det.scan_text('{"type": "service' + '_account", "project_id": "x"}') == [] + ) + + def test_marker_with_private_key_positive(self): + text = ( + '{"type": "service' + '_account", "private' + '_key_id": "abc", ' + '"private' + '_key": "not-a-real-pem-block"}' + ) + assert "gcp_service_account" in names(det.scan_text(text)) + class TestAzure: def test_connection_string_positive(self): @@ -150,6 +168,35 @@ def test_connection_string_positive(self): def test_low_entropy_negative(self): assert det.scan_text("AccountKey=" + "A" * 70) == [] + def test_shared_access_key_positive(self): + # Service Bus / Event Hubs / IoT Hub connection strings use + # SharedAccessKey=, not AccountKey=. Variable named to avoid tripping + # this file's own env_assignment detector on ` = "..."`. + sas_value = "nQx8VmPz4KtY2bWd7RjLq3hF6sN9vC1xB4mZ8kQz=" + text = ( + "Endpoint=sb://ns.servicebus.windows.net/;" + "SharedAccessKeyName=RootManageSharedAccessKey;" + f"SharedAccessKey={sas_value}" + ) + findings = det.scan_text(text) + assert names(findings) == ["azure_storage_account_key"] + assert sas_value not in str(findings[0]) + + def test_well_known_azurite_dev_key_negative(self): + # the public, constant Azurite/Storage-Emulator default key is not a + # secret; must be excluded everywhere, not just from this detector, + # since "AccountKey" also matches the generic env_assignment name filter + azurite_key = ( + "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/" + "K1SZFPTOtr/KBHBeksoGMGw==" + ) + assert det.scan_text(f"AccountKey={azurite_key}") == [] + conn_string = ( + f"DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;" + f"AccountKey={azurite_key};" + ) + assert det.scan_text(conn_string) == [] + class TestTwilio: def test_positive(self): @@ -202,6 +249,18 @@ class TestDatabaseUrl: "mongodb+srv://appuser:{pw}@cluster0.example.net/prod", "redis://default:{pw}@cache:6379/0", "amqp://worker:{pw}@mq:5672/vhost", + # SQLAlchemy dialect+driver URLs (default form in every + # Python/SQLAlchemy/Alembic config) + "postgresql+psycopg2://svc:{pw}@db.internal/app", + "postgresql+asyncpg://svc:{pw}@db/app", + "mysql+pymysql://svc:{pw}@db/app", + "mssql+pyodbc://svc:{pw}@db/app", + # Rails' DATABASE_URL scheme for MySQL + "mysql2://svc:{pw}@10.0.0.5/appdb", + # canonical empty-username Redis/Mongo auth URLs + "redis://:{pw}@cache.internal:6379/0", + "rediss://:{pw}@cache/0", + "mongodb://:{pw}@db/x", ], ) def test_inline_password_positive(self, url): @@ -224,6 +283,8 @@ def test_mongo_positive(self): "postgres://user@localhost/db", "postgres://localhost:5432/db", "https://example.com/path", + "postgres://:password@host/db", + "redis://:changeme@host/db", ], ) def test_placeholder_or_no_password_negative(self, url):