(\d+)
`)
+)
+
+type documentedCategory struct {
+ heading string
+ kind VerifierKind
+}
+
+func TestVerificationCoverageDocs_MatchCapabilityManifest(t *testing.T) {
+ counts := VerificationCapabilityCounts()
+ tests := []struct {
+ name string
+ path string
+ categories []documentedCategory
+ summary []string
+ }{
+ {
+ name: "English",
+ path: filepath.Join("..", "..", "docs", "user-manuals", "en", "verification", "verification-coverage.md"),
+ categories: []documentedCategory{
+ {heading: "## Live-verified", kind: VerifierLive},
+ {heading: "## Requires trusted or companion context", kind: VerifierRequiresContext},
+ {heading: "## Format-validated only", kind: VerifierFormatOnly},
+ {heading: "## Not verifiable", kind: VerifierNone},
+ },
+ summary: []string{
+ fmt.Sprintf("| Live-verified | %d |", counts.Live),
+ fmt.Sprintf("| Requires trusted/companion context | %d |", counts.RequiresContext),
+ fmt.Sprintf("| Format-validated only | %d |", counts.FormatOnly),
+ fmt.Sprintf("| Not verifiable | %d |", counts.None),
+ fmt.Sprintf("| **Total detectors** | **%d** |", Detectors),
+ },
+ },
+ {
+ name: "Turkish",
+ path: filepath.Join("..", "..", "docs", "user-manuals", "tr", "verification", "verification-coverage.md"),
+ categories: []documentedCategory{
+ {heading: "## Canlı doğrulanan", kind: VerifierLive},
+ {heading: "## Güvenilir veya eşlik eden bağlam gerektiren", kind: VerifierRequiresContext},
+ {heading: "## Yalnızca format doğrulaması", kind: VerifierFormatOnly},
+ {heading: "## Doğrulanamaz", kind: VerifierNone},
+ },
+ summary: []string{
+ fmt.Sprintf("| Canlı doğrulanan | %d |", counts.Live),
+ fmt.Sprintf("| Güvenilir/eşlik eden bağlam gerektiren | %d |", counts.RequiresContext),
+ fmt.Sprintf("| Yalnızca format doğrulaması | %d |", counts.FormatOnly),
+ fmt.Sprintf("| Doğrulanamaz | %d |", counts.None),
+ fmt.Sprintf("| **Toplam dedektör** | **%d** |", Detectors),
+ },
+ },
+ }
+
+ want := make(map[string]VerifierKind, Detectors)
+ for _, capability := range VerificationCapabilities() {
+ want[capability.DetectorID] = capability.VerifierKind
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ contents, err := os.ReadFile(tc.path)
+ require.NoError(t, err)
+ got := parseDocumentedCapabilities(t, string(contents), tc.categories)
+ assert.Equal(t, want, got, "documentation categories drifted from capability manifest")
+ for _, summary := range tc.summary {
+ assert.Contains(t, string(contents), summary)
+ }
+ })
+ }
+}
+
+func TestPublishedCapabilityClaims_MatchManifest(t *testing.T) {
+ counts := VerificationCapabilityCounts()
+ require.Equal(t, CapabilityCounts{Live: 39, FormatOnly: 6, RequiresContext: 9, None: 11}, counts)
+ registryCoverage := 100 * float64(Verifiers) / float64(Detectors)
+
+ tests := []struct {
+ path string
+ required []string
+ forbid []string
+ }{
+ {
+ path: filepath.Join("..", "..", "README.md"),
+ required: []string{
+ fmt.Sprintf("**%d built-in detectors**", Detectors),
+ fmt.Sprintf("**%d direct live checks** + **%d context-required checks** + **%d offline format validators**", counts.Live, counts.RequiresContext, counts.FormatOnly),
+ fmt.Sprintf("**%d of %d detectors (%.1f%%)**", Verifiers, Detectors, registryCoverage),
+ },
+ forbid: []string{"54 live verifiers", "54 verifiers, 84.4%", "~48 detectors"},
+ },
+ {
+ path: filepath.Join("..", "..", "site", "index.html"),
+ required: []string{
+ fmt.Sprintf("%d detectors · %d direct-live · %d context-required · %d sources", Detectors, counts.Live, counts.RequiresContext, Sources),
+ fmt.Sprintf(`%d`, counts.Live),
+ fmt.Sprintf(`%d`, counts.RequiresContext),
+ fmt.Sprintf(`%d`, counts.FormatOnly),
+ fmt.Sprintf(`%d`, counts.None),
+ },
+ forbid: []string{"54 live verifiers", "~49", "Redacted index — 64 detectors"},
+ },
+ {
+ path: filepath.Join("..", "..", "site", "js", "translations.js"),
+ required: []string{
+ fmt.Sprintf(`"hero.kv": "%d detectors · %d direct-live · %d context-required · %d sources"`, Detectors, counts.Live, counts.RequiresContext, Sources),
+ fmt.Sprintf(`"hero.kv": "%d dedektör · %d doğrudan canlı · %d bağlam gerektiren · %d kaynak"`, Detectors, counts.Live, counts.RequiresContext, Sources),
+ },
+ forbid: []string{"54 live verifiers", "54 canlı doğrulayıcı", "64 detectors", "63 dedektör"},
+ },
+ {
+ path: filepath.Join("..", "..", "docs", "guides", "secret-verification.md"),
+ required: []string{
+ fmt.Sprintf("**%d** detectors support a direct live check", counts.Live),
+ fmt.Sprintf("**%d** require trusted issuer, region, or companion context", counts.RequiresContext),
+ fmt.Sprintf("### Not Verifiable (%d detectors)", counts.None),
+ },
+ forbid: []string{"Live API Verification (48 detectors)", "84.4% of its 64 detectors"},
+ },
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.path, func(t *testing.T) {
+ contents, err := os.ReadFile(tc.path)
+ require.NoError(t, err)
+ text := string(contents)
+ for _, required := range tc.required {
+ assert.Contains(t, text, required)
+ }
+ for _, forbidden := range tc.forbid {
+ assert.NotContains(t, text, forbidden)
+ }
+ })
+ }
+}
+
+func TestSiteVisibleCounts_MatchCapabilityManifest(t *testing.T) {
+ contents, err := os.ReadFile(filepath.Join("..", "..", "site", "index.html"))
+ require.NoError(t, err)
+ counts := VerificationCapabilityCounts()
+
+ wantCapabilities := map[string]int{
+ "live": counts.Live, "context": counts.RequiresContext,
+ "fmt": counts.FormatOnly, "no": counts.None,
+ }
+ gotCapabilities := make(map[string]int, len(wantCapabilities))
+ for _, match := range siteCapabilityRow.FindAllStringSubmatch(string(contents), -1) {
+ var count int
+ _, err := fmt.Sscanf(match[2], "%d", &count)
+ require.NoError(t, err)
+ gotCapabilities[match[1]] = count
+ }
+ assert.Equal(t, wantCapabilities, gotCapabilities)
+
+ wantStats := map[string]int{
+ "detectors": Detectors, "verifiers": counts.Live,
+ "sources": Sources, "formats": OutputFormats,
+ }
+ gotStats := make(map[string]int, len(wantStats))
+ for _, match := range siteStatCell.FindAllStringSubmatch(string(contents), -1) {
+ var count int
+ _, err := fmt.Sscanf(match[1], "%d", &count)
+ require.NoError(t, err)
+ gotStats[match[2]] = count
+ }
+ assert.Equal(t, wantStats, gotStats)
+}
+
+func parseDocumentedCapabilities(
+ t *testing.T,
+ contents string,
+ categories []documentedCategory,
+) map[string]VerifierKind {
+ t.Helper()
+ headingKinds := make(map[string]VerifierKind, len(categories))
+ for _, category := range categories {
+ headingKinds[category.heading] = category.kind
+ }
+
+ got := make(map[string]VerifierKind, Detectors)
+ current := VerifierKind("")
+ scanner := bufio.NewScanner(strings.NewReader(contents))
+ for scanner.Scan() {
+ line := scanner.Text()
+ if strings.HasPrefix(line, "## ") {
+ current = ""
+ for heading, kind := range headingKinds {
+ if strings.HasPrefix(line, heading) {
+ current = kind
+ break
+ }
+ }
+ continue
+ }
+ if current == "" {
+ continue
+ }
+ match := markdownDetectorRow.FindStringSubmatch(line)
+ if match == nil {
+ continue
+ }
+ if previous, duplicate := got[match[1]]; duplicate {
+ t.Errorf("detector %q documented twice (%s and %s)", match[1], previous, current)
+ }
+ got[match[1]] = current
+ }
+ require.NoError(t, scanner.Err())
+ return got
+}
diff --git a/internal/meta/capabilities_test.go b/internal/meta/capabilities_test.go
new file mode 100644
index 0000000..13ba209
--- /dev/null
+++ b/internal/meta/capabilities_test.go
@@ -0,0 +1,257 @@
+package meta
+
+import (
+ "net/url"
+ "regexp"
+ "sort"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+var detectorIDPattern = regexp.MustCompile(`^[a-z0-9]+(?:-[a-z0-9]+)*$`)
+
+var validEndpointClasses = map[EndpointClass]struct{}{
+ EndpointNone: {}, EndpointOfflineFormat: {}, EndpointFixedProviderAPI: {},
+ EndpointFixedProviderSDK: {}, EndpointRegionalProviderAPI: {},
+ EndpointBoundedRegionalProviderAPI: {}, EndpointIssuerDerivedProviderAPI: {},
+ EndpointDetectorContextProviderAPI: {},
+ EndpointOperatorContextProviderAPI: {}, EndpointCompanionContextProviderAPI: {},
+ EndpointCompanionContextFixedAPI: {}, EndpointEmbeddedProviderURL: {},
+}
+
+var validInactiveContracts = map[InactiveStatusContract]struct{}{
+ InactiveNone: {}, InactiveDefinitiveAuthRejection: {}, InactiveHTTP401Only: {},
+ InactiveAllRegionsHTTP401: {}, InactiveRegionAppropriateRejection: {},
+ InactiveProviderBodyRejection: {}, InactiveProviderSpecificRejection: {},
+ InactivePairedAuthRejection: {}, InactiveTrustedInstanceHTTP401: {},
+ InactiveTrustedIssuerHTTP401: {}, InactiveTrustedOriginHTTP401: {},
+ InactiveTrustedOriginInvalid401: {},
+ InactiveTrustedOriginRejection: {},
+ InactiveTrustedSiteRejection: {}, InactiveTrustedStoreHTTP401: {},
+ InactiveTrustedStoreRejection: {},
+ InactiveTypedAuthenticationError: {},
+}
+
+func TestVerificationCapabilities_AreCompleteAndWellFormed(t *testing.T) {
+ capabilities := VerificationCapabilities()
+ require.Len(t, capabilities, Detectors)
+
+ ids := make([]string, 0, len(capabilities))
+ seen := make(map[string]struct{}, len(capabilities))
+ for _, capability := range capabilities {
+ assert.Regexp(t, detectorIDPattern, capability.DetectorID)
+ if _, duplicate := seen[capability.DetectorID]; duplicate {
+ t.Errorf("duplicate capability for detector %q", capability.DetectorID)
+ }
+ seen[capability.DetectorID] = struct{}{}
+ ids = append(ids, capability.DetectorID)
+ assert.Contains(t, validEndpointClasses, capability.EndpointClass, capability.DetectorID)
+ assert.Contains(t, validInactiveContracts, capability.InactiveStatusContract, capability.DetectorID)
+
+ switch capability.VerifierKind {
+ case VerifierLive:
+ assert.NotEqual(t, EndpointNone, capability.EndpointClass, capability.DetectorID)
+ assert.NotEqual(t, InactiveNone, capability.InactiveStatusContract, capability.DetectorID)
+ case VerifierFormatOnly:
+ assert.Empty(t, capability.RequiredContextFields, capability.DetectorID)
+ assert.Equal(t, EndpointOfflineFormat, capability.EndpointClass, capability.DetectorID)
+ assert.Equal(t, InactiveNone, capability.InactiveStatusContract, capability.DetectorID)
+ case VerifierRequiresContext:
+ assert.NotEmpty(t, capability.RequiredContextFields, capability.DetectorID)
+ assert.NotEqual(t, EndpointNone, capability.EndpointClass, capability.DetectorID)
+ assert.NotEqual(t, InactiveNone, capability.InactiveStatusContract, capability.DetectorID)
+ case VerifierNone:
+ assert.Empty(t, capability.RequiredContextFields, capability.DetectorID)
+ assert.Equal(t, EndpointNone, capability.EndpointClass, capability.DetectorID)
+ assert.Equal(t, InactiveNone, capability.InactiveStatusContract, capability.DetectorID)
+ default:
+ t.Errorf("detector %q has unknown verifier kind %q", capability.DetectorID, capability.VerifierKind)
+ }
+
+ if capability.LastContractReviewedAt != "" {
+ _, err := time.Parse(time.DateOnly, capability.LastContractReviewedAt)
+ assert.NoError(t, err, capability.DetectorID)
+ assert.NotEmpty(t, capability.ContractReferenceURLs, capability.DetectorID)
+ } else {
+ assert.Empty(t, capability.ContractReferenceURLs, capability.DetectorID)
+ }
+ for _, reference := range capability.ContractReferenceURLs {
+ parsed, err := url.Parse(reference)
+ if assert.NoError(t, err, capability.DetectorID) {
+ assert.Equal(t, "https", parsed.Scheme, capability.DetectorID)
+ assert.NotEmpty(t, parsed.Hostname(), capability.DetectorID)
+ }
+ }
+ }
+
+ sorted := append([]string(nil), ids...)
+ sort.Strings(sorted)
+ assert.Equal(t, sorted, ids, "capability manifest must remain sorted by detector ID")
+}
+
+func TestVerificationCapabilities_ReviewedContractsAreFresh(t *testing.T) {
+ const maximumAuditAge = 180 * 24 * time.Hour
+ now := time.Now().UTC()
+
+ for _, capability := range VerificationCapabilities() {
+ if capability.LastContractReviewedAt == "" {
+ continue
+ }
+ reviewedAt, err := time.Parse(time.DateOnly, capability.LastContractReviewedAt)
+ require.NoError(t, err, capability.DetectorID)
+ assert.False(t, reviewedAt.After(now), "%s contract review is future-dated", capability.DetectorID)
+ assert.LessOrEqual(t, now.Sub(reviewedAt), maximumAuditAge,
+ "%s provider contract audit is stale; review the primary references and update code/tests before advancing the date",
+ capability.DetectorID)
+ }
+}
+
+func TestVerificationCapabilityCounts_MatchPublishedCategories(t *testing.T) {
+ counts := VerificationCapabilityCounts()
+ assert.Equal(t, CapabilityCounts{Live: 39, FormatOnly: 6, RequiresContext: 9, None: 11}, counts)
+ assert.Equal(t, Detectors, counts.Live+counts.FormatOnly+counts.RequiresContext+counts.None)
+ assert.Equal(t, Verifiers, counts.Live+counts.FormatOnly+counts.RequiresContext,
+ "every non-none capability has one registered verifier")
+}
+
+func TestVerificationCapabilities_ReturnsDefensiveCopy(t *testing.T) {
+ first := VerificationCapabilities()
+ require.NotEmpty(t, first)
+ originalID := first[0].DetectorID
+ contextIndex, regionIndex, subtypeIndex, referenceIndex := -1, -1, -1, -1
+ for i := range first {
+ if contextIndex < 0 && len(first[i].RequiredContextFields) > 0 {
+ contextIndex = i
+ }
+ if regionIndex < 0 && len(first[i].ProviderRegions) > 0 {
+ regionIndex = i
+ }
+ if subtypeIndex < 0 && len(first[i].VerifiableSubtypes) > 0 && len(first[i].UnverifiableSubtypes) > 0 {
+ subtypeIndex = i
+ }
+ if referenceIndex < 0 && len(first[i].ContractReferenceURLs) > 0 {
+ referenceIndex = i
+ }
+ }
+ require.NotEqual(t, -1, contextIndex)
+ require.NotEqual(t, -1, regionIndex)
+ require.NotEqual(t, -1, subtypeIndex)
+ require.NotEqual(t, -1, referenceIndex)
+ originalContext := first[contextIndex].RequiredContextFields[0]
+ originalRegion := first[regionIndex].ProviderRegions[0]
+ originalVerifiableSubtype := first[subtypeIndex].VerifiableSubtypes[0]
+ originalUnverifiableSubtype := first[subtypeIndex].UnverifiableSubtypes[0]
+ originalReference := first[referenceIndex].ContractReferenceURLs[0]
+ first[0].DetectorID = "mutated"
+ first[contextIndex].RequiredContextFields[0] = "mutated"
+ first[regionIndex].ProviderRegions[0] = "mutated"
+ first[subtypeIndex].VerifiableSubtypes[0] = "mutated"
+ first[subtypeIndex].UnverifiableSubtypes[0] = "mutated"
+ first[referenceIndex].ContractReferenceURLs[0] = "https://mutated.invalid"
+
+ second := VerificationCapabilities()
+ assert.Equal(t, originalID, second[0].DetectorID)
+ assert.Equal(t, originalContext, second[contextIndex].RequiredContextFields[0])
+ assert.Equal(t, originalRegion, second[regionIndex].ProviderRegions[0])
+ assert.Equal(t, originalVerifiableSubtype, second[subtypeIndex].VerifiableSubtypes[0])
+ assert.Equal(t, originalUnverifiableSubtype, second[subtypeIndex].UnverifiableSubtypes[0])
+ assert.Equal(t, originalReference, second[referenceIndex].ContractReferenceURLs[0])
+}
+
+func TestVerificationCapabilities_GitHubOAuthSubtypeContract(t *testing.T) {
+ for _, capability := range VerificationCapabilities() {
+ if capability.DetectorID != "github-oauth-token" {
+ continue
+ }
+ assert.Equal(t, []string{"gho", "ghu", "ghs"}, capability.VerifiableSubtypes)
+ assert.Equal(t, []string{"ghr"}, capability.UnverifiableSubtypes)
+ return
+ }
+ t.Fatal("github-oauth-token capability missing")
+}
+
+func TestVerificationCapabilities_GitLabSubtypeAndInactiveContract(t *testing.T) {
+ for _, capability := range VerificationCapabilities() {
+ if capability.DetectorID != "gitlab-pat" {
+ continue
+ }
+ assert.Equal(t, []string{"glpat"}, capability.VerifiableSubtypes)
+ assert.Equal(t, []string{
+ "gldt", "glrt", "glrtr", "glcbt", "glptt", "glimt", "glagent",
+ "glwt", "glsoat", "glffct", "gloas", "glft",
+ }, capability.UnverifiableSubtypes)
+ assert.Equal(t, InactiveTrustedOriginInvalid401, capability.InactiveStatusContract)
+ return
+ }
+ t.Fatal("gitlab-pat capability missing")
+}
+
+func TestVerificationCapabilities_SnykInactiveContractIsTrustedOrigin401Only(t *testing.T) {
+ for _, capability := range VerificationCapabilities() {
+ if capability.DetectorID != "snyk-api-key" {
+ continue
+ }
+ assert.Equal(t, InactiveTrustedOriginHTTP401, capability.InactiveStatusContract)
+ return
+ }
+ t.Fatal("snyk-api-key capability missing")
+}
+
+func TestVerificationCapabilities_TwilioAndShopifyAreOperatorContextOnly(t *testing.T) {
+ want := map[string]struct {
+ inactive InactiveStatusContract
+ regions []string
+ }{
+ "shopify-access-token": {inactive: InactiveTrustedStoreHTTP401},
+ "twilio-api-key": {inactive: InactiveTrustedOriginHTTP401, regions: []string{"US1", "IE1", "AU1"}},
+ }
+ for _, capability := range VerificationCapabilities() {
+ expected, ok := want[capability.DetectorID]
+ if !ok {
+ continue
+ }
+ assert.Equal(t, VerifierRequiresContext, capability.VerifierKind, capability.DetectorID)
+ assert.Equal(t, EndpointOperatorContextProviderAPI, capability.EndpointClass, capability.DetectorID)
+ assert.Equal(t, expected.inactive, capability.InactiveStatusContract, capability.DetectorID)
+ assert.Equal(t, expected.regions, capability.ProviderRegions, capability.DetectorID)
+ assert.Equal(t, "2026-08-11", capability.LastContractReviewedAt, capability.DetectorID)
+ delete(want, capability.DetectorID)
+ }
+ assert.Empty(t, want)
+}
+
+func TestVerificationCapabilities_CriticalContextContracts(t *testing.T) {
+ want := map[string]struct {
+ kind VerifierKind
+ fields []string
+ regions []string
+ }{
+ "aws-access-key-id": {VerifierLive, []string{"raw_v2"}, nil},
+ "databricks-token": {VerifierLive, []string{"host"}, nil},
+ "okta-api-token": {VerifierLive, []string{"domain"}, nil},
+ "datadog-api-key": {VerifierRequiresContext, []string{"trusted_api_origin"}, []string{"US1", "US3", "US5", "EU", "AP1", "AP2", "UK1", "US1-FED", "US2-FED"}},
+ "github-token": {VerifierRequiresContext, []string{"trusted_api_origin"}, []string{"GitHub.com", "GHES"}},
+ "github-oauth-token": {VerifierRequiresContext, []string{"trusted_api_origin"}, []string{"GitHub.com", "GHES"}},
+ "snyk-api-key": {VerifierRequiresContext, []string{"trusted_api_origin"}, []string{"US1", "US2", "EU", "AU", "GOV", "PRIVATE"}},
+ "grafana-api-key": {VerifierRequiresContext, []string{"trusted_instance_origin"}, nil},
+ "shopify-access-token": {VerifierRequiresContext, []string{"trusted_store_origin"}, nil},
+ "twilio-api-key": {VerifierRequiresContext, []string{"api_key_sid", "trusted_api_origin"}, []string{"US1", "IE1", "AU1"}},
+ }
+
+ got := make(map[string]VerificationCapability, len(want))
+ for _, capability := range VerificationCapabilities() {
+ if _, tracked := want[capability.DetectorID]; tracked {
+ got[capability.DetectorID] = capability
+ }
+ }
+ require.Len(t, got, len(want))
+ for detectorID, expected := range want {
+ capability := got[detectorID]
+ assert.Equal(t, expected.kind, capability.VerifierKind, detectorID)
+ assert.Equal(t, expected.fields, capability.RequiredContextFields, detectorID)
+ assert.Equal(t, expected.regions, capability.ProviderRegions, detectorID)
+ }
+}
diff --git a/internal/meta/counts.go b/internal/meta/counts.go
index 85812ae..693883d 100644
--- a/internal/meta/counts.go
+++ b/internal/meta/counts.go
@@ -1,7 +1,10 @@
-// Package meta holds the canonical, human-maintained project counts that are
-// published in the README banner, the social-preview image, and the docs.
+// Package meta holds canonical project counts and verification-capability
+// metadata published in the README banner, social-preview image, and docs.
//
-// These constants are the single source of truth for the published numbers:
+// These constants are the single source of truth for release metadata and
+// registry/output counts.
+// VerificationCapabilities is the separate source of truth for direct-live,
+// context-required, format-only, and no-verifier capability counts:
//
// - Detectors and Verifiers are guarded at test time against the live
// registries (detector.All() / verifier.All()), so adding or removing one
@@ -17,12 +20,14 @@
// re-render their PNGs (the re-render command is in each asset's header).
package meta
+import "strings"
+
//go:generate go run ./statsgen
const (
// Detectors is the number of compile-time registered secret detectors;
// it must equal len(detector.All()).
- Detectors = 64
+ Detectors = 65
// Verifiers is the number of registered verifiers; it must equal
// len(verifier.All()).
@@ -35,4 +40,23 @@ const (
// OutputFormats is the number of output formats: json, sarif, csv, table,
// and github.
OutputFormats = 5
+
+ // OutputFormatList is the canonical user-facing order and spelling used by
+ // CLI help and documentation contracts.
+ OutputFormatList = "json, sarif, csv, table, github"
)
+
+// OutputFormatNames returns the canonical output formats as a fresh slice.
+func OutputFormatNames() []string {
+ return strings.Split(OutputFormatList, ", ")
+}
+
+// IsOutputFormat reports whether name is a canonical output format.
+func IsOutputFormat(name string) bool {
+ for _, format := range OutputFormatNames() {
+ if name == format {
+ return true
+ }
+ }
+ return false
+}
diff --git a/internal/meta/counts_test.go b/internal/meta/counts_test.go
index 4a96699..f06769f 100644
--- a/internal/meta/counts_test.go
+++ b/internal/meta/counts_test.go
@@ -3,8 +3,10 @@ package meta
import (
"os"
"path/filepath"
+ "regexp"
"strings"
"testing"
+ "time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -65,4 +67,43 @@ func TestOutputFormats_MatchOutputPackageDirectories(t *testing.T) {
assert.Equal(t, OutputFormats, got,
"meta.OutputFormats drifted from the number of internal/output subpackages; "+
"update internal/meta then run `go generate ./...`")
+ assert.Len(t, OutputFormatNames(), OutputFormats,
+ "meta.OutputFormatList and meta.OutputFormats must change together")
+
+ seen := make(map[string]struct{}, OutputFormats)
+ for _, format := range OutputFormatNames() {
+ assert.Regexp(t, regexp.MustCompile(`^[a-z][a-z0-9-]*$`), format)
+ if _, duplicate := seen[format]; duplicate {
+ t.Errorf("duplicate output format %q", format)
+ }
+ seen[format] = struct{}{}
+ assert.True(t, IsOutputFormat(format))
+ }
+ assert.False(t, IsOutputFormat(""))
+ assert.False(t, IsOutputFormat("unknown"))
+}
+
+func TestReleaseMetadata_MatchesPublishedDocumentation(t *testing.T) {
+ assert.Regexp(t, regexp.MustCompile(`^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$`), ReleaseVersion)
+ assert.Regexp(t, regexp.MustCompile(`^[0-9]{4}-[0-9]{2}-[0-9]{2}$`), ReleaseDate)
+ _, err := time.Parse(time.DateOnly, ReleaseDate)
+ require.NoError(t, err, "ReleaseDate must be a real ISO 8601 calendar date")
+
+ changelog, err := os.ReadFile(filepath.Join("..", "..", "CHANGELOG.md"))
+ require.NoError(t, err)
+ assert.Contains(t, string(changelog), "## ["+ReleaseVersion+"] - "+ReleaseDate,
+ "release metadata must identify a published changelog entry")
+
+ roadmap, err := os.ReadFile(filepath.Join("..", "..", "docs", "05-ROADMAP.md"))
+ require.NoError(t, err)
+ assert.Contains(t, string(roadmap),
+ "| Phase 9 — Detection Accuracy & FP Reduction | Completed | `"+ReleaseVersion+"` | "+ReleaseDate+" |",
+ "roadmap release record must match canonical release metadata")
+ assert.Contains(t, string(roadmap),
+ "| Phase 10 — Detector Library Expansion | Planned | `v1.9.0` | — |",
+ "the next minor version must remain assigned to planned Phase 10")
+ assert.NotContains(t, string(roadmap),
+ "| Phase 9 — Detection Accuracy & FP Reduction | Planned | `"+ReleaseVersion+"` | — |")
+ assert.Contains(t, string(roadmap), "| Broaden OpenAI key coverage | Delivered in `v1.7.0` |")
+ assert.Contains(t, string(roadmap), "| GitHub fine-grained PAT support | Delivered in `v1.7.0` |")
}
diff --git a/internal/meta/documentation_contract_test.go b/internal/meta/documentation_contract_test.go
new file mode 100644
index 0000000..8220507
--- /dev/null
+++ b/internal/meta/documentation_contract_test.go
@@ -0,0 +1,479 @@
+package meta
+
+import (
+ "fmt"
+ "io/fs"
+ "os"
+ "path/filepath"
+ "regexp"
+ "sort"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+ "gopkg.in/yaml.v3"
+)
+
+const releaseVersionTokenPattern = `v[0-9]+\.[0-9]+\.[0-9]+(?:[-_][0-9A-Za-z]+(?:[._-][0-9A-Za-z]+)*)?`
+
+var (
+ releaseVersionToken = regexp.MustCompile(`\b(` + releaseVersionTokenPattern + `)(?:[^0-9A-Za-z._-]|$)`)
+ releaseVersionTokenExact = regexp.MustCompile(`^` + releaseVersionTokenPattern + `$`)
+ directLeakwatchPinPatterns = []*regexp.Regexp{
+ regexp.MustCompile(`(?i)github\.com/hodetech/leakwatch@(` + releaseVersionTokenPattern + `)(?:[^0-9A-Za-z._-]|$)`),
+ regexp.MustCompile(`(?i)(?:^|[^0-9A-Za-z/])hodetech/leakwatch@(` + releaseVersionTokenPattern + `)(?:[^0-9A-Za-z._-]|$)`),
+ regexp.MustCompile(`(?i)ghcr\.io/hodetech/leakwatch:(` + releaseVersionTokenPattern + `)(?:[^0-9A-Za-z._-]|$)`),
+ regexp.MustCompile(`(?i)LEAKWATCH_VERSION[[:space:]]*[:=][[:space:]]*['\"]?(` + releaseVersionTokenPattern + `)(?:[^0-9A-Za-z._-]|$)`),
+ regexp.MustCompile(`(?i)leakwatch[[:space:]]+(` + releaseVersionTokenPattern + `)(?:[^0-9A-Za-z._-]|$)`),
+ regexp.MustCompile(`(?i)leakwatch[[:space:]]+(?:version|sürümü).*?(` + releaseVersionTokenPattern + `)(?:[^0-9A-Za-z._-]|$)`),
+ }
+ historicalVersionMarker = regexp.MustCompile(`(?i)`)
+)
+
+type documentationBlock struct {
+ startLine int
+ text string
+ leakwatchImageTags bool
+}
+
+type versionPin struct {
+ line int
+ version string
+}
+
+func TestSupplementalGuides_DeclareCanonicalManualBoundary(t *testing.T) {
+ guides, err := filepath.Glob(filepath.Join("..", "..", "docs", "guides", "*.md"))
+ require.NoError(t, err)
+ require.NotEmpty(t, guides)
+
+ for _, path := range guides {
+ if filepath.Base(path) == "README.md" {
+ continue
+ }
+ contents, readErr := os.ReadFile(path)
+ require.NoError(t, readErr)
+ text := string(contents)
+ assert.Contains(t, text, "> **Documentation role:** Supplemental", path)
+ assert.Contains(t, text, "(../user-manuals/en/", path)
+ }
+}
+
+func TestDocumentation_LeakwatchReleasePinsAreCurrentOrExplicitlyHistorical(t *testing.T) {
+ root := filepath.Join("..", "..")
+ paths, err := currentDocumentationMarkdownPaths(root)
+ require.NoError(t, err)
+ require.NotEmpty(t, paths)
+
+ for _, path := range paths {
+ contents, readErr := os.ReadFile(path)
+ require.NoError(t, readErr)
+ for _, block := range markdownContractBlocks(string(contents)) {
+ for _, pin := range leakwatchReleasePins(path, block) {
+ if pin.version == ReleaseVersion || blockMarksHistoricalVersion(block.text, pin.version) {
+ continue
+ }
+ t.Errorf("%s:%d uses stale Leakwatch pin %s; use %s or mark the same logical example with ",
+ filepath.ToSlash(path), pin.line, pin.version, ReleaseVersion, pin.version)
+ }
+ }
+ }
+}
+
+func TestLeakwatchReleasePins_ContextAndHistoricalMarkerMatrix(t *testing.T) {
+ tests := []struct {
+ name string
+ path string
+ contents string
+ wantPins []string
+ wantLines []int
+ wantExempt bool
+ }{
+ {name: "environment equals", contents: "LEAKWATCH_VERSION = v1.6.0", wantPins: []string{"v1.6.0"}, wantLines: []int{1}},
+ {name: "environment YAML", contents: "LEAKWATCH_VERSION: 'v1.6.0'", wantPins: []string{"v1.6.0"}, wantLines: []int{1}},
+ {name: "Leakwatch action input", contents: "```yaml\n- uses: HodeTech/Leakwatch@v1\n with:\n version: v1.6.0\n- uses: actions/setup-node@v4\n with:\n version: v22.3.0\n```", wantPins: []string{"v1.6.0"}, wantLines: []int{4}},
+ {name: "exact Leakwatch action pin", contents: "```yaml\n- uses: HodeTech/Leakwatch@v1.6.0\n```", wantPins: []string{"v1.6.0"}, wantLines: []int{2}},
+ {name: "Leakwatch pre-commit rev", contents: "```yaml\n- repo: https://github.com/HodeTech/Leakwatch\n rev: v1.6.0\n```", wantPins: []string{"v1.6.0"}, wantLines: []int{3}},
+ {name: "unrelated rev in same YAML", contents: "```yaml\n- repo: https://github.com/HodeTech/Leakwatch\n rev: v1.7.0\n- repo: https://example.com/tool\n rev: v9.8.7\n```", wantPins: []string{"v1.7.0"}, wantLines: []int{3}},
+ {name: "unrelated version assignment", contents: "```yaml\nname: unrelated\nversion: v2.3.0\n```"},
+ {name: "suffix cannot hide behind current stable prefix", contents: "LEAKWATCH_VERSION=v1.7.0-typo", wantPins: []string{"v1.7.0-typo"}, wantLines: []int{1}},
+ {name: "contextual image tag table", contents: "```text\nghcr.io/hodetech/leakwatch\n```\n\nAvailable tags:\n\n| Tag | Description |\n|---|---|\n| `:v1.6.0` | stale |", wantPins: []string{"v1.6.0"}, wantLines: []int{9}},
+ {name: "bare tag for unrelated image is ignored", contents: "Available tags:\n\n| Tag | Description |\n|---|---|\n| `:v9.8.7` | unrelated |"},
+ {name: "exact historical marker", contents: "\n\n```bash\nleakwatch v1.6.0\n```", wantPins: []string{"v1.6.0"}, wantLines: []int{4}, wantExempt: true},
+ {name: "marker cannot exempt different pin", contents: "\n\nleakwatch v1.6.0", wantPins: []string{"v1.6.0"}, wantLines: []int{3}},
+ {name: "marker cannot exempt later block", contents: "\n\nleakwatch v1.6.0\n\nUnrelated paragraph.\n\nleakwatch v1.6.0", wantPins: []string{"v1.6.0", "v1.6.0"}, wantLines: []int{3, 7}},
+ }
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ var got []string
+ var gotLines []int
+ var exemptions []bool
+ for _, block := range markdownContractBlocks(tc.contents) {
+ for _, pin := range leakwatchReleasePins(tc.path, block) {
+ got = append(got, pin.version)
+ gotLines = append(gotLines, pin.line)
+ exemptions = append(exemptions, blockMarksHistoricalVersion(block.text, pin.version))
+ }
+ }
+ assert.Equal(t, tc.wantPins, got)
+ assert.Equal(t, tc.wantLines, gotLines)
+ if tc.wantExempt {
+ assert.Equal(t, []bool{true}, exemptions)
+ } else if len(exemptions) > 0 {
+ assert.False(t, exemptions[len(exemptions)-1])
+ }
+ })
+ }
+}
+
+func TestEntropyPolicyDocs_MatchEngineContract(t *testing.T) {
+ root := filepath.Join("..", "..")
+ oldADR, err := os.ReadFile(filepath.Join(root, "docs", "decisions", "ADR-0005-pattern-matching.md"))
+ require.NoError(t, err)
+ assert.Contains(t, string(oldADR), "**Status:** Amended by [ADR-0010]")
+
+ currentADR, err := os.ReadFile(filepath.Join(root, "docs", "decisions", "ADR-0010-entropy-gating-policy.md"))
+ require.NoError(t, err)
+ for _, contract := range []string{"EntropyBased", "EntropyGated", "generic-api-key", "structural provider findings", "per-rule `entropy` threshold"} {
+ assert.Contains(t, string(currentADR), contract)
+ }
+
+ engineSource, err := os.ReadFile(filepath.Join(root, "internal", "engine", "engine.go"))
+ require.NoError(t, err)
+ assert.Contains(t, string(engineSource), "isEntropyGated(det, raw)")
+ genericSource, err := os.ReadFile(filepath.Join(root, "internal", "detector", "generic", "generic_api_key.go"))
+ require.NoError(t, err)
+ assert.Contains(t, string(genericSource), "func (d *APIKeyDetector) EntropyBased() bool { return true }")
+ assert.Contains(t, string(genericSource), "func (d *APIKeyDetector) EntropyGated(raw detector.RawFinding) bool")
+
+ forbidden := []*regexp.Regexp{
+ regexp.MustCompile(`(?is)entropy.{0,120}display-only`),
+ regexp.MustCompile(`(?is)engine never gates or suppresses.{0,120}entropy`),
+ regexp.MustCompile(`(?is)global entropy gate.{0,120}(?:planned|not yet implemented)`),
+ }
+ for _, dir := range []string{
+ filepath.Join(root, "docs", "architecture"),
+ filepath.Join(root, "docs", "guides"),
+ filepath.Join(root, "docs", "user-manuals"),
+ } {
+ walkErr := filepath.WalkDir(dir, func(path string, entry fs.DirEntry, walkErr error) error {
+ if walkErr != nil {
+ return walkErr
+ }
+ if entry.IsDir() || !strings.EqualFold(filepath.Ext(entry.Name()), ".md") {
+ return nil
+ }
+ contents, readErr := os.ReadFile(path)
+ if readErr != nil {
+ return readErr
+ }
+ for _, pattern := range forbidden {
+ assert.False(t, pattern.Match(contents), "%s contains superseded entropy policy matching %s", filepath.ToSlash(path), pattern)
+ }
+ return nil
+ })
+ require.NoError(t, walkErr)
+ }
+}
+
+func TestSlackScopeDocumentation_ContainsExecutableRequiredSet(t *testing.T) {
+ root := filepath.Join("..", "..")
+ paths := []string{
+ filepath.Join(root, "docs", "guides", "configuration.md"),
+ filepath.Join(root, "docs", "guides", "slack-scanning.md"),
+ filepath.Join(root, "docs", "user-manuals", "en", "scanning", "slack.md"),
+ filepath.Join(root, "docs", "user-manuals", "tr", "scanning", "slack.md"),
+ filepath.Join(root, "site", "js", "manuals", "en.js"),
+ filepath.Join(root, "site", "js", "manuals", "tr.js"),
+ }
+ scopes := []string{
+ "channels:read", "channels:history", "groups:read", "groups:history",
+ "im:read", "im:history", "mpim:read", "mpim:history", "files:read",
+ }
+ for _, path := range paths {
+ contents, err := os.ReadFile(path)
+ require.NoError(t, err)
+ for _, scope := range scopes {
+ assert.Contains(t, string(contents), scope, "%s is missing %s", filepath.ToSlash(path), scope)
+ }
+ }
+}
+
+func TestFindingWireSemanticsDocs_MatchEngine(t *testing.T) {
+ root := filepath.Join("..", "..")
+ paths := []string{
+ filepath.Join(root, "docs", "user-manuals", "en", "getting-started", "how-it-works.md"),
+ filepath.Join(root, "docs", "user-manuals", "tr", "getting-started", "how-it-works.md"),
+ }
+ for _, path := range paths {
+ contents, err := os.ReadFile(path)
+ require.NoError(t, err)
+ text := string(contents)
+ assert.Contains(t, text, "decimal(byteOffset)", path)
+ assert.Contains(t, text, "detection.entropy.enabled", path)
+ assert.NotContains(t, text, "sha256(detectorID + redacted + filePath + line)", path)
+ }
+ for _, path := range []string{
+ filepath.Join(root, "docs", "user-manuals", "en", "getting-started", "introduction.md"),
+ filepath.Join(root, "docs", "user-manuals", "tr", "getting-started", "introduction.md"),
+ } {
+ contents, err := os.ReadFile(path)
+ require.NoError(t, err)
+ counts := VerificationCapabilityCounts()
+ assert.Contains(t, string(contents), fmt.Sprintf("%d ", counts.None),
+ "not-verifiable detector count must come from the capability manifest")
+ }
+}
+
+func currentDocumentationMarkdownPaths(root string) ([]string, error) {
+ skipDirs := map[string]struct{}{
+ ".git": {}, "node_modules": {}, "vendor": {}, "dist": {}, "review": {},
+ }
+ var paths []string
+ err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, walkErr error) error {
+ if walkErr != nil {
+ return walkErr
+ }
+ if entry.IsDir() {
+ if path != root {
+ if _, skip := skipDirs[entry.Name()]; skip {
+ return filepath.SkipDir
+ }
+ }
+ return nil
+ }
+ if !strings.EqualFold(filepath.Ext(entry.Name()), ".md") {
+ return nil
+ }
+ rel, relErr := filepath.Rel(root, path)
+ if relErr != nil {
+ return relErr
+ }
+ rel = filepath.ToSlash(rel)
+ if strings.EqualFold(filepath.Base(rel), "CHANGELOG.md") || rel == "docs/05-ROADMAP.md" {
+ return nil
+ }
+ paths = append(paths, path)
+ return nil
+ })
+ sort.Strings(paths)
+ return paths, err
+}
+
+func markdownContractBlocks(contents string) []documentationBlock {
+ lines := strings.Split(strings.ReplaceAll(contents, "\r\n", "\n"), "\n")
+ var blocks []documentationBlock
+ for index := 0; index < len(lines); {
+ index = nextNonBlankLine(lines, index)
+ if index >= len(lines) {
+ break
+ }
+ start := index
+ prefix, contentStart := documentationBlockPrefix(lines, index)
+ if contentStart >= len(lines) {
+ blocks = append(blocks, documentationBlock{startLine: start + 1, text: prefix})
+ break
+ }
+ end := documentationBlockEnd(lines, contentStart)
+ blocks = append(blocks, documentationBlock{startLine: start + 1, text: prefix + strings.Join(lines[contentStart:end], "\n")})
+ index = end
+ }
+ markLeakwatchImageTagTables(blocks)
+ return blocks
+}
+
+func nextNonBlankLine(lines []string, index int) int {
+ for index < len(lines) && strings.TrimSpace(lines[index]) == "" {
+ index++
+ }
+ return index
+}
+
+func documentationBlockPrefix(lines []string, index int) (string, int) {
+ if !historicalVersionMarker.MatchString(lines[index]) {
+ return "", index
+ }
+ contentStart := nextNonBlankLine(lines, index+1)
+ return strings.Join(lines[index:contentStart], "\n") + "\n", contentStart
+}
+
+func documentationBlockEnd(lines []string, start int) int {
+ trimmed := strings.TrimSpace(lines[start])
+ if strings.HasPrefix(trimmed, "```") || strings.HasPrefix(trimmed, "~~~") {
+ return fencedDocumentationBlockEnd(lines, start, trimmed[:3])
+ }
+ end := start + 1
+ for end < len(lines) && strings.TrimSpace(lines[end]) != "" && !startsMarkdownFence(lines[end]) {
+ end++
+ }
+ return end
+}
+
+func fencedDocumentationBlockEnd(lines []string, start int, fence string) int {
+ for end := start + 1; end < len(lines); end++ {
+ if strings.HasPrefix(strings.TrimSpace(lines[end]), fence) {
+ return end + 1
+ }
+ }
+ return len(lines)
+}
+
+func startsMarkdownFence(line string) bool {
+ trimmed := strings.TrimSpace(line)
+ return strings.HasPrefix(trimmed, "```") || strings.HasPrefix(trimmed, "~~~")
+}
+
+func markLeakwatchImageTagTables(blocks []documentationBlock) {
+ pendingTagTable := false
+ for index := range blocks {
+ trimmed := strings.TrimSpace(blocks[index].text)
+ if strings.HasPrefix(trimmed, "#") {
+ pendingTagTable = false
+ }
+ if strings.Contains(strings.ToLower(blocks[index].text), "ghcr.io/hodetech/leakwatch") {
+ pendingTagTable = true
+ }
+ if pendingTagTable && strings.HasPrefix(trimmed, "|") && strings.Contains(blocks[index].text, "`:v") {
+ blocks[index].leakwatchImageTags = true
+ pendingTagTable = false
+ }
+ }
+}
+
+func leakwatchReleasePins(_ string, block documentationBlock) []versionPin {
+ byVersionAndLine := make(map[versionPin]struct{})
+ collect := func(pattern *regexp.Regexp) {
+ for _, match := range pattern.FindAllStringSubmatchIndex(block.text, -1) {
+ if len(match) < 4 || match[2] < 0 {
+ continue
+ }
+ version := block.text[match[2]:match[3]]
+ line := block.startLine + strings.Count(block.text[:match[2]], "\n")
+ byVersionAndLine[versionPin{line: line, version: version}] = struct{}{}
+ }
+ }
+ for _, pattern := range directLeakwatchPinPatterns {
+ collect(pattern)
+ }
+ for _, pin := range leakwatchYAMLPins(block) {
+ byVersionAndLine[pin] = struct{}{}
+ }
+
+ if block.leakwatchImageTags {
+ for _, location := range releaseVersionToken.FindAllStringSubmatchIndex(block.text, -1) {
+ if len(location) < 4 || location[2] < 0 {
+ continue
+ }
+ version := block.text[location[2]:location[3]]
+ line := block.startLine + strings.Count(block.text[:location[2]], "\n")
+ byVersionAndLine[versionPin{line: line, version: version}] = struct{}{}
+ }
+ }
+
+ pins := make([]versionPin, 0, len(byVersionAndLine))
+ for pin := range byVersionAndLine {
+ pins = append(pins, pin)
+ }
+ sort.Slice(pins, func(i, j int) bool {
+ if pins[i].line == pins[j].line {
+ return pins[i].version < pins[j].version
+ }
+ return pins[i].line < pins[j].line
+ })
+ return pins
+}
+
+func leakwatchYAMLPins(block documentationBlock) []versionPin {
+ source, lineOffset := yamlSourceFromBlock(block.text)
+ if strings.TrimSpace(source) == "" {
+ return nil
+ }
+ var document yaml.Node
+ if err := yaml.Unmarshal([]byte(source), &document); err != nil {
+ return nil
+ }
+ var pins []versionPin
+ var walk func(*yaml.Node)
+ walk = func(node *yaml.Node) {
+ if node.Kind == yaml.MappingNode {
+ values := yamlMappingValues(node)
+ if repo := values["repo"]; repo != nil && isLeakwatchRepository(repo.Value) {
+ if rev := values["rev"]; rev != nil {
+ if version := releaseVersionTokenExact.FindString(rev.Value); version != "" {
+ pins = append(pins, versionPin{
+ line: block.startLine + lineOffset + rev.Line - 1,
+ version: version,
+ })
+ }
+ }
+ }
+ if uses := values["uses"]; uses != nil && isLeakwatchAction(uses.Value) {
+ if with := values["with"]; with != nil && with.Kind == yaml.MappingNode {
+ if versionNode := yamlMappingValues(with)["version"]; versionNode != nil {
+ if version := releaseVersionTokenExact.FindString(versionNode.Value); version != "" {
+ pins = append(pins, versionPin{
+ line: block.startLine + lineOffset + versionNode.Line - 1,
+ version: version,
+ })
+ }
+ }
+ }
+ }
+ }
+ for _, child := range node.Content {
+ walk(child)
+ }
+ }
+ walk(&document)
+ return pins
+}
+
+func yamlSourceFromBlock(block string) (string, int) {
+ lines := strings.Split(block, "\n")
+ for index, line := range lines {
+ trimmed := strings.ToLower(strings.TrimSpace(line))
+ if !strings.HasPrefix(trimmed, "```yaml") && !strings.HasPrefix(trimmed, "```yml") &&
+ !strings.HasPrefix(trimmed, "~~~yaml") && !strings.HasPrefix(trimmed, "~~~yml") {
+ continue
+ }
+ fence := strings.TrimSpace(line)[:3]
+ for end := index + 1; end < len(lines); end++ {
+ if strings.HasPrefix(strings.TrimSpace(lines[end]), fence) {
+ return strings.Join(lines[index+1:end], "\n"), index + 1
+ }
+ }
+ return "", 0
+ }
+ return block, 0
+}
+
+func yamlMappingValues(mapping *yaml.Node) map[string]*yaml.Node {
+ values := make(map[string]*yaml.Node, len(mapping.Content)/2)
+ for index := 0; index+1 < len(mapping.Content); index += 2 {
+ key := mapping.Content[index]
+ if key.Kind == yaml.ScalarNode {
+ values[strings.ToLower(strings.TrimSpace(key.Value))] = mapping.Content[index+1]
+ }
+ }
+ return values
+}
+
+func isLeakwatchRepository(value string) bool {
+ normalized := strings.TrimSuffix(strings.ToLower(strings.TrimSpace(value)), ".git")
+ return normalized == "https://github.com/hodetech/leakwatch" || normalized == "github.com/hodetech/leakwatch"
+}
+
+func isLeakwatchAction(value string) bool {
+ normalized := strings.ToLower(strings.TrimSpace(value))
+ return strings.HasPrefix(normalized, "hodetech/leakwatch@") ||
+ strings.HasPrefix(normalized, "github.com/hodetech/leakwatch@")
+}
+
+func blockMarksHistoricalVersion(block, version string) bool {
+ for _, match := range historicalVersionMarker.FindAllStringSubmatch(block, -1) {
+ if strings.EqualFold(match[1], version) {
+ return true
+ }
+ }
+ return false
+}
diff --git a/internal/meta/provider_contract_links_test.go b/internal/meta/provider_contract_links_test.go
new file mode 100644
index 0000000..95dc49f
--- /dev/null
+++ b/internal/meta/provider_contract_links_test.go
@@ -0,0 +1,124 @@
+package meta
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+ "os"
+ "regexp"
+ "sort"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+const providerLinkCheckEnvironment = "LEAKWATCH_CHECK_PROVIDER_LINKS"
+
+var officialProviderDocumentationHosts = map[string]struct{}{
+ "api.slack.com": {},
+ "auth0.com": {},
+ "docs.datadoghq.com": {},
+ "docs.github.com": {},
+ "docs.gitlab.com": {},
+ "docs.newrelic.com": {},
+ "docs.slack.dev": {},
+ "docs.snyk.io": {},
+ "github.blog": {},
+ "grafana.com": {},
+ "shopify.dev": {},
+ "supabase.com": {},
+ "www.twilio.com": {},
+}
+
+func isOfficialProviderDocumentationURL(raw string) bool {
+ parsed, err := url.Parse(raw)
+ if err != nil || parsed.Scheme != "https" || parsed.User != nil || parsed.Hostname() == "" {
+ return false
+ }
+ _, ok := officialProviderDocumentationHosts[strings.ToLower(parsed.Hostname())]
+ return ok
+}
+
+func TestVerificationCapabilities_ReferencesUseOfficialProviderHosts(t *testing.T) {
+ for _, capability := range VerificationCapabilities() {
+ for _, reference := range capability.ContractReferenceURLs {
+ assert.True(t, isOfficialProviderDocumentationURL(reference), "%s: %s", capability.DetectorID, reference)
+ }
+ }
+ assert.False(t, isOfficialProviderDocumentationURL("https://docs.gitlab.com.attacker.example/api/users/"))
+ assert.False(t, isOfficialProviderDocumentationURL("http://docs.gitlab.com/api/users/"))
+}
+
+func TestVerificationCapabilities_CriticalReferencesMatchImplementedEndpoints(t *testing.T) {
+ byDetector := make(map[string][]string)
+ for _, capability := range VerificationCapabilities() {
+ byDetector[capability.DetectorID] = capability.ContractReferenceURLs
+ }
+ assert.Contains(t, byDetector["gitlab-pat"], "https://docs.gitlab.com/api/users/#retrieve-the-current-user")
+ assert.Contains(t, byDetector["gitlab-pat"], "https://docs.gitlab.com/api/personal_access_tokens/")
+ assert.Contains(t, byDetector["github-oauth-token"], "https://docs.github.com/en/rest/apps/installations#list-repositories-accessible-to-the-app-installation")
+}
+
+func TestVerificationCapabilities_ReviewedReferenceLinksResolve(t *testing.T) {
+ if os.Getenv(providerLinkCheckEnvironment) != "1" {
+ t.Skip("network contract audit is enabled only by the protected scheduled workflow")
+ }
+
+ client := &http.Client{
+ Timeout: 20 * time.Second,
+ CheckRedirect: func(request *http.Request, via []*http.Request) error {
+ if len(via) >= 5 {
+ return fmt.Errorf("provider documentation redirect limit exceeded")
+ }
+ if !isOfficialProviderDocumentationURL(request.URL.String()) {
+ return fmt.Errorf("provider documentation redirected outside official hosts")
+ }
+ return nil
+ },
+ }
+
+ seen := make(map[string]struct{})
+ var references []string
+ for _, capability := range VerificationCapabilities() {
+ for _, reference := range capability.ContractReferenceURLs {
+ if _, exists := seen[reference]; !exists {
+ seen[reference] = struct{}{}
+ references = append(references, reference)
+ }
+ }
+ }
+ sort.Strings(references)
+ for _, reference := range references {
+ t.Run(reference, func(t *testing.T) {
+ parsed, err := url.Parse(reference)
+ require.NoError(t, err)
+ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+ defer cancel()
+ request, err := http.NewRequestWithContext(ctx, http.MethodGet, reference, nil)
+ require.NoError(t, err)
+ request.Header.Set("User-Agent", "leakwatch-provider-contract-audit")
+ response, err := client.Do(request)
+ require.NoError(t, err)
+ defer response.Body.Close()
+ require.GreaterOrEqual(t, response.StatusCode, http.StatusOK)
+ require.Less(t, response.StatusCode, http.StatusMultipleChoices)
+
+ body, err := io.ReadAll(io.LimitReader(response.Body, 8*1024*1024+1))
+ require.NoError(t, err)
+ require.LessOrEqual(t, len(body), 8*1024*1024, "provider documentation response exceeds audit bound")
+ if parsed.Fragment == "" {
+ return
+ }
+ fragment, err := url.PathUnescape(parsed.Fragment)
+ require.NoError(t, err)
+ fragmentPattern := regexp.MustCompile(`(?i)\bid=(?:"` + regexp.QuoteMeta(fragment) + `"|'` + regexp.QuoteMeta(fragment) + `'|` + regexp.QuoteMeta(fragment) + `(?:[ >]))`)
+ require.True(t, fragmentPattern.Match(body),
+ "fragment %q is absent from provider documentation", fragment)
+ })
+ }
+}
diff --git a/internal/meta/release.go b/internal/meta/release.go
new file mode 100644
index 0000000..22c9bc1
--- /dev/null
+++ b/internal/meta/release.go
@@ -0,0 +1,11 @@
+package meta
+
+const (
+ // ReleaseVersion is the latest stable release published to users. Release
+ // tooling still injects the running binary's exact build version; this value
+ // owns static product surfaces such as the website footer and roadmap guards.
+ ReleaseVersion = "v1.8.0"
+
+ // ReleaseDate is the publication date of ReleaseVersion (ISO 8601).
+ ReleaseDate = "2026-08-11"
+)
diff --git a/internal/meta/releasecheck/main.go b/internal/meta/releasecheck/main.go
new file mode 100644
index 0000000..083e6c9
--- /dev/null
+++ b/internal/meta/releasecheck/main.go
@@ -0,0 +1,74 @@
+// Command releasecheck rejects malformed release tags and prevents a stable
+// tag from publishing artifacts whose metadata still names another release.
+package main
+
+import (
+ "flag"
+ "fmt"
+ "os"
+ "regexp"
+ "strings"
+
+ "github.com/HodeTech/leakwatch/internal/meta"
+)
+
+var (
+ stableTagPattern = regexp.MustCompile(`^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$`)
+ prereleaseIdentifierRule = regexp.MustCompile(`^[0-9A-Za-z-]+$`)
+)
+
+func main() {
+ tag := flag.String("tag", "", "Git release tag being published")
+ flag.Parse()
+ if flag.NArg() != 0 {
+ fail(fmt.Errorf("unexpected positional arguments"))
+ }
+ prerelease, err := validateReleaseTag(*tag)
+ if err != nil {
+ fail(err)
+ }
+ if prerelease {
+ fmt.Printf("prerelease tag %s is valid; stable metadata remains %s\n", *tag, meta.ReleaseVersion)
+ return
+ }
+ fmt.Printf("release metadata matches stable tag %s\n", *tag)
+}
+
+func validateReleaseTag(tag string) (bool, error) {
+ base, prerelease, hasPrerelease := strings.Cut(tag, "-")
+ if !stableTagPattern.MatchString(base) {
+ return false, fmt.Errorf("release tag %q does not start with a valid vMAJOR.MINOR.PATCH version", tag)
+ }
+ if !hasPrerelease {
+ if tag != meta.ReleaseVersion {
+ return false, fmt.Errorf("release tag %q does not match internal/meta.ReleaseVersion %q", tag, meta.ReleaseVersion)
+ }
+ return false, nil
+ }
+ if prerelease == "" {
+ return false, fmt.Errorf("release tag %q has an empty prerelease suffix", tag)
+ }
+ for _, identifier := range strings.Split(prerelease, ".") {
+ if !prereleaseIdentifierRule.MatchString(identifier) {
+ return false, fmt.Errorf("release tag %q has invalid prerelease identifier %q", tag, identifier)
+ }
+ if len(identifier) > 1 && identifier[0] == '0' && isDecimal(identifier) {
+ return false, fmt.Errorf("release tag %q has a numeric prerelease identifier with a leading zero", tag)
+ }
+ }
+ return true, nil
+}
+
+func isDecimal(value string) bool {
+ for _, r := range value {
+ if r < '0' || r > '9' {
+ return false
+ }
+ }
+ return true
+}
+
+func fail(err error) {
+ fmt.Fprintln(os.Stderr, "releasecheck:", err)
+ os.Exit(1)
+}
diff --git a/internal/meta/releasecheck/main_test.go b/internal/meta/releasecheck/main_test.go
new file mode 100644
index 0000000..ad5a5ab
--- /dev/null
+++ b/internal/meta/releasecheck/main_test.go
@@ -0,0 +1,53 @@
+package main
+
+import (
+ "testing"
+
+ "github.com/HodeTech/leakwatch/internal/meta"
+)
+
+func TestValidateStableReleaseTag(t *testing.T) {
+ prerelease, err := validateReleaseTag(meta.ReleaseVersion)
+ if err != nil {
+ t.Fatalf("validateReleaseTag(%q) error = %v", meta.ReleaseVersion, err)
+ }
+ if prerelease {
+ t.Fatalf("validateReleaseTag(%q) prerelease = true, want false", meta.ReleaseVersion)
+ }
+}
+
+func TestValidateReleaseTag_AcceptsValidPrereleases(t *testing.T) {
+ for _, tag := range []string{"v1.8.0-rc.1", "v2.0.0-beta.2", "v2.0.0-preview-x"} {
+ t.Run(tag, func(t *testing.T) {
+ prerelease, err := validateReleaseTag(tag)
+ if err != nil {
+ t.Fatalf("validateReleaseTag(%q) error = %v", tag, err)
+ }
+ if !prerelease {
+ t.Fatalf("validateReleaseTag(%q) prerelease = false, want true", tag)
+ }
+ })
+ }
+}
+
+func TestValidateReleaseTag_FailsClosed(t *testing.T) {
+ for _, tag := range []string{
+ "",
+ "vnext",
+ "1.7.0",
+ "v01.7.0",
+ "v1.7",
+ "v999.0.0",
+ "v1.8.0-",
+ "v1.8.0-rc..1",
+ "v1.8.0-rc+build",
+ "v1.8.0-01",
+ "v1.8.0_rc1",
+ } {
+ t.Run(tag, func(t *testing.T) {
+ if _, err := validateReleaseTag(tag); err == nil {
+ t.Fatalf("validateReleaseTag(%q) error = nil, want fail-closed error", tag)
+ }
+ })
+ }
+}
diff --git a/internal/meta/statsgen/main.go b/internal/meta/statsgen/main.go
index 9337e69..e2879e8 100644
--- a/internal/meta/statsgen/main.go
+++ b/internal/meta/statsgen/main.go
@@ -28,6 +28,7 @@ const (
// managedFiles are rewritten relative to the repository root.
var managedFiles = []string{
"docs/assets/banner.html",
+ "docs/assets/banner.svg",
"site/assets/og.svg",
}
@@ -40,9 +41,10 @@ type replacement struct {
}
func replacements() []replacement {
+ capabilities := meta.VerificationCapabilityCounts()
return []replacement{
{regexp.MustCompile(`\d+ detectors`), fmt.Sprintf("%d detectors", meta.Detectors)},
- {regexp.MustCompile(`\d+ live verifiers`), fmt.Sprintf("%d live verifiers", meta.Verifiers)},
+ {regexp.MustCompile(`\d+ (?:live verifiers|direct-live checks)`), fmt.Sprintf("%d direct-live checks", capabilities.Live)},
{regexp.MustCompile(`\d+ sources`), fmt.Sprintf("%d sources", meta.Sources)},
{regexp.MustCompile(`\d+ output formats`), fmt.Sprintf("%d output formats", meta.OutputFormats)},
}
@@ -82,7 +84,7 @@ func main() {
}
if len(stale) > 0 {
- fail(fmt.Errorf("stat blocks out of date in: %s\nrun `go generate ./...` and re-render the PNGs",
+ fail(fmt.Errorf("stat blocks out of date in: %s\nrun `go generate ./...`",
strings.Join(stale, ", ")))
}
}
diff --git a/internal/meta/statsgen/main_test.go b/internal/meta/statsgen/main_test.go
index 9e3129e..5b7d069 100644
--- a/internal/meta/statsgen/main_test.go
+++ b/internal/meta/statsgen/main_test.go
@@ -8,8 +8,9 @@ import (
// TestManagedAssetsUpToDate fails when a marketing asset's stat block no longer
// matches internal/meta — i.e. a count was bumped but `go generate ./...` was
-// not run (or the PNG was re-rendered from a stale source). It runs as part of
-// `go test ./...`, so CI catches the drift without a dedicated workflow step.
+// not run. README renders the directly managed SVG, so the visible banner is
+// covered without a browser-dependent rasterization step. It runs as part of
+// `go test ./...`, so CI catches source drift without a dedicated workflow step.
func TestManagedAssetsUpToDate(t *testing.T) {
root, err := repoRoot()
if err != nil {
@@ -25,7 +26,19 @@ func TestManagedAssetsUpToDate(t *testing.T) {
t.Fatalf("%s: %v", rel, err)
}
if updated != string(orig) {
- t.Errorf("%s stat block is stale; run `go generate ./...` and re-render its PNG", rel)
+ t.Errorf("%s stat block is stale; run `go generate ./...`", rel)
}
}
}
+
+func TestRewriteMigratesLegacyLiveVerifierLabel(t *testing.T) {
+ input := "54 live verifiers"
+ got, err := rewrite(input)
+ if err != nil {
+ t.Fatalf("rewrite legacy label: %v", err)
+ }
+ want := "39 direct-live checks"
+ if got != want {
+ t.Fatalf("rewrite legacy label = %q, want %q", got, want)
+ }
+}
diff --git a/internal/meta/vscode_release_contract_test.go b/internal/meta/vscode_release_contract_test.go
new file mode 100644
index 0000000..f734ac7
--- /dev/null
+++ b/internal/meta/vscode_release_contract_test.go
@@ -0,0 +1,39 @@
+package meta
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestVSCodeMarketplaceWorkflow_FailsClosedOnRepositoryPolicy(t *testing.T) {
+ path := filepath.Join("..", "..", ".github", "workflows", "vscode-release.yml")
+ contents, err := os.ReadFile(path)
+ require.NoError(t, err)
+ workflow := string(contents)
+
+ for _, contract := range []string{
+ "inputs.publish && github.ref == 'refs/heads/main'",
+ "environment: vscode-marketplace",
+ "actions: read",
+ "deployments: read",
+ "prevent_self_review == true",
+ "deployment_branch_policy.protected_branches == true",
+ "VSCODE_ENV_AUDIT_TOKEN",
+ "${{ secrets.VSCODE_ENV_AUDIT_TOKEN }}",
+ "@<(printf 'Authorization: Bearer %s\\n' \"${GH_ENVIRONMENT_AUDIT_TOKEN:?}\")",
+ "environments/vscode-marketplace/secrets/VSCE_PAT",
+ ".name == \"VSCE_PAT\"",
+ "${{ secrets.VSCE_PAT }}",
+ "${VSCE_PAT:?missing environment-scoped VSCE_PAT}",
+ "npx --no-install vsce publish --packagePath",
+ "if [[ ${#packages[@]} -ne 1 ]]",
+ } {
+ assert.Contains(t, workflow, contract)
+ }
+ assert.NotContains(t, workflow, "--pat")
+ assert.NotContains(t, workflow, "Authorization: Bearer ${", "tokens must not be exposed in process arguments")
+}
diff --git a/internal/output/formatter.go b/internal/output/formatter.go
index 48f6506..0450aef 100644
--- a/internal/output/formatter.go
+++ b/internal/output/formatter.go
@@ -3,6 +3,7 @@
package output
import (
+ "fmt"
"io"
"strings"
"unicode"
@@ -27,7 +28,9 @@ type Formatter interface {
// SanitizeForDisplay strips Unicode control characters — including the ESC
// (0x1B) byte that begins ANSI/OSC/DCS escape sequences, and the C0/C1
-// control ranges more generally — from s.
+// control ranges more generally — from s. Unicode bidi controls are rendered
+// as visible \\uXXXX escapes so they cannot reorder terminal text while their
+// presence and exact code point remain available to the operator.
//
// Apply this to any attacker-influenced field (file path, detector ID,
// redacted match value, ...) before it is written somewhere an operator
@@ -38,20 +41,38 @@ type Formatter interface {
// custom-rule match can inject cursor moves, color changes, title-bar
// writes, or other terminal escape sequences into the operator's session.
//
-// Bytes are removed rather than substituted: the safe behavior for a
-// terminal-facing display field is to drop the disallowed byte entirely
-// rather than risk re-introducing structure (e.g. a substitute delimiter)
-// that could itself be misinterpreted downstream.
+// Control bytes are removed rather than substituted: the safe behavior for a
+// terminal-facing display field is to drop them entirely rather than risk
+// re-introducing structure (e.g. a substitute delimiter) that could itself be
+// misinterpreted downstream. Bidi controls are escaped instead because their
+// code-point identity is useful evidence and the visible ASCII form is inert.
//
// Do NOT apply this to the unredacted secret value (Raw) shown under
// --show-raw: that field's entire purpose is exact fidelity to the actual
// secret, and silently dropping bytes from it would misrepresent the secret
// being reported.
func SanitizeForDisplay(s string) string {
- return strings.Map(func(r rune) rune {
+ var sanitized strings.Builder
+ for _, r := range s {
if unicode.IsControl(r) {
- return -1
+ continue
}
- return r
- }, s)
+ if isBidiControl(r) {
+ _, _ = fmt.Fprintf(&sanitized, "\\u%04X", r)
+ continue
+ }
+ sanitized.WriteRune(r)
+ }
+ return sanitized.String()
+}
+
+// isBidiControl covers the Unicode bidi controls that can alter the visual
+// order of surrounding text. Other format characters, including emoji ZWJ,
+// remain intact so ordinary internationalized paths are not degraded.
+func isBidiControl(r rune) bool {
+ return r == '\u061C' ||
+ r == '\u200E' ||
+ r == '\u200F' ||
+ (r >= '\u202A' && r <= '\u202E') ||
+ (r >= '\u2066' && r <= '\u2069')
}
diff --git a/internal/output/formatter_test.go b/internal/output/formatter_test.go
index 9e0d712..5f4e57c 100644
--- a/internal/output/formatter_test.go
+++ b/internal/output/formatter_test.go
@@ -53,6 +53,16 @@ func TestSanitizeForDisplay(t *testing.T) {
in: "config-é.yaml",
want: "config-é.yaml",
},
+ {
+ name: "bidi controls are visibly escaped without reordering text",
+ in: "safe.go\u202ecod.exe\u2066tail",
+ want: "safe.go\\u202Ecod.exe\\u2066tail",
+ },
+ {
+ name: "emoji ZWJ and combining marks are preserved",
+ in: "family-👩\u200d💻-cafe\u0301",
+ want: "family-👩\u200d💻-cafe\u0301",
+ },
}
for _, tt := range tests {
diff --git a/internal/output/json/json_formatter.go b/internal/output/json/json_formatter.go
index eb626f2..fc29133 100644
--- a/internal/output/json/json_formatter.go
+++ b/internal/output/json/json_formatter.go
@@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"io"
+ "time"
"github.com/HodeTech/leakwatch/pkg/finding"
)
@@ -26,10 +27,14 @@ type Formatter struct {
// gated the same way Raw is: it must never carry secret material (see
// finding.Finding's doc comment), but it is still opt-in behind --show-raw
// alongside Raw so it does not appear in default output either.
+type findingAlias finding.Finding
+
type findingJSON struct {
- finding.Finding
- Raw string `json:"raw,omitempty"`
- ExtraData map[string]string `json:"extra_data,omitempty"`
+ findingAlias
+ Raw string `json:"raw,omitempty"`
+ ExtraData map[string]string `json:"extra_data,omitempty"`
+ DetectedAt *time.Time `json:"detected_at,omitempty"`
+ Entropy *float64 `json:"entropy,omitempty"`
}
// Format writes findings as JSON to the given writer.
@@ -50,7 +55,14 @@ func (f *Formatter) Format(w io.Writer, findings []finding.Finding) error {
output := make([]findingJSON, len(findings))
for i, fd := range findings {
- output[i] = findingJSON{Finding: fd, Raw: fd.Raw, ExtraData: fd.ExtraData}
+ wire := findingJSON{findingAlias: findingAlias(fd), Raw: fd.Raw, ExtraData: fd.ExtraData}
+ if !fd.DetectedAt.IsZero() {
+ wire.DetectedAt = &fd.DetectedAt
+ }
+ if fd.EntropyCalculated || fd.Entropy != 0 {
+ wire.Entropy = &fd.Entropy
+ }
+ output[i] = wire
}
if err := encoder.Encode(output); err != nil {
return fmt.Errorf("failed to write JSON output: %w", err)
diff --git a/internal/output/json/json_formatter_test.go b/internal/output/json/json_formatter_test.go
index fd2b194..adaff3d 100644
--- a/internal/output/json/json_formatter_test.go
+++ b/internal/output/json/json_formatter_test.go
@@ -223,6 +223,21 @@ func TestFormatter_Format_ShowRawTrue_OmitsExtraDataWhenNil(t *testing.T) {
assert.False(t, hasExtraData, "extra_data should be omitted when nil, even with ShowRaw=true")
}
+func TestFormatter_Format_ShowRaw_PreservesNullableFindingFields(t *testing.T) {
+ formatted := finding.Finding{ID: "computed-zero", Raw: "synthetic-secret"}
+ formatted.SetEntropy(0)
+
+ var buf bytes.Buffer
+ require.NoError(t, (&Formatter{ShowRaw: true}).Format(&buf, []finding.Finding{formatted}))
+
+ var output []map[string]any
+ require.NoError(t, json.Unmarshal(buf.Bytes(), &output))
+ require.Len(t, output, 1)
+ assert.NotContains(t, output[0], "detected_at")
+ assert.Equal(t, float64(0), output[0]["entropy"])
+ assert.Equal(t, "synthetic-secret", output[0]["raw"])
+}
+
func TestFormatter_Format_ShowRawFalse_DoesNotMutateOriginal(t *testing.T) {
f := &Formatter{ShowRaw: false}
var buf bytes.Buffer
diff --git a/internal/output/table/table_formatter.go b/internal/output/table/table_formatter.go
index 5470db6..32195f2 100644
--- a/internal/output/table/table_formatter.go
+++ b/internal/output/table/table_formatter.go
@@ -6,7 +6,8 @@ import (
"io"
"strconv"
"strings"
- "text/tabwriter"
+
+ "github.com/mattn/go-runewidth"
"github.com/HodeTech/leakwatch/internal/output"
"github.com/HodeTech/leakwatch/pkg/finding"
@@ -34,7 +35,9 @@ type Formatter struct {
// Format writes findings as a formatted table to the given writer.
// Columns: SEVERITY | DETECTOR | FILE | LINE | REDACTED | STATUS | REMEDIATION
-// When ShowRaw is true, a trailing RAW column is appended.
+// When ShowRaw is true, a trailing RAW column is appended. Its value is a
+// reversible Go-quoted string so arbitrary secret bytes cannot inject terminal
+// controls and can still be recovered exactly with strconv.Unquote.
// A summary line is appended at the bottom.
//
// DetectorID, FilePath, and Redacted are attacker-influenced (a malicious
@@ -44,32 +47,20 @@ type Formatter struct {
// strips control/ANSI-escape bytes so a crafted finding cannot inject
// terminal escape sequences into the operator's session.
func (f *Formatter) Format(w io.Writer, findings []finding.Finding) error {
- tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0)
-
- // Write header.
- header := "SEVERITY\tDETECTOR\tFILE\tLINE\tREDACTED\tSTATUS\tREMEDIATION"
- separator := "--------\t--------\t----\t----\t--------\t------\t-----------"
+ headers := []string{"SEVERITY", "DETECTOR", "FILE", "LINE", "REDACTED", "STATUS", "REMEDIATION"}
if f.ShowRaw {
- header += "\tRAW"
- separator += "\t---"
- }
- if _, err := fmt.Fprintln(tw, header); err != nil {
- return fmt.Errorf("failed to write table header: %w", err)
+ headers = append(headers, "RAW")
}
- // Write separator.
- if _, err := fmt.Fprintln(tw, separator); err != nil {
- return fmt.Errorf("failed to write table separator: %w", err)
- }
-
- // Write rows.
+ rows := make([]tableRow, 0, len(findings))
for _, fd := range findings {
remediation := "-"
if fd.Remediation != nil && fd.Remediation.Title != "" {
remediation = fd.Remediation.Title
}
- sevText := strings.ToUpper(fd.Severity.String())
+ plainSeverity := strings.ToUpper(fd.Severity.String())
+ sevText := plainSeverity
if f.ColorEnabled {
sevText = f.colorizeSeverity(fd.Severity, sevText)
}
@@ -79,26 +70,23 @@ func (f *Formatter) Format(w io.Writer, findings []finding.Finding) error {
lineNo = strconv.Itoa(fd.SourceMetadata.Line)
}
- line := fmt.Sprintf(
- "%s\t%s\t%s\t%s\t%s\t%s\t%s",
- sevText,
- output.SanitizeForDisplay(fd.DetectorID),
- output.SanitizeForDisplay(fd.SourceMetadata.FilePath),
- lineNo,
- output.SanitizeForDisplay(fd.Redacted),
- fd.Verification.Status.String(),
- remediation,
- )
- if f.ShowRaw {
- line += "\t" + fd.Raw
+ cells := []tableCell{
+ {text: sevText, widthText: plainSeverity},
+ {text: output.SanitizeForDisplay(fd.DetectorID)},
+ {text: output.SanitizeForDisplay(fd.SourceMetadata.FilePath)},
+ {text: lineNo},
+ {text: output.SanitizeForDisplay(fd.Redacted)},
+ {text: fd.Verification.Status.String()},
+ {text: output.SanitizeForDisplay(remediation)},
}
- if _, err := fmt.Fprintln(tw, line); err != nil {
- return fmt.Errorf("failed to write table row: %w", err)
+ if f.ShowRaw {
+ cells = append(cells, tableCell{text: quoteRawForTable(fd.Raw)})
}
+ rows = append(rows, tableRow{cells: cells})
}
- if err := tw.Flush(); err != nil {
- return fmt.Errorf("failed to flush table output: %w", err)
+ if err := writeDisplayWidthTable(w, headers, rows); err != nil {
+ return err
}
// Write summary line.
@@ -113,6 +101,87 @@ func (f *Formatter) Format(w io.Writer, findings []finding.Finding) error {
return nil
}
+func quoteRawForTable(raw string) string {
+ return strconv.Quote(raw)
+}
+
+const columnPadding = 2
+
+type tableCell struct {
+ text string
+ widthText string
+}
+
+func (c tableCell) displayWidth() int {
+ if c.widthText != "" {
+ return runewidth.StringWidth(c.widthText)
+ }
+ return runewidth.StringWidth(c.text)
+}
+
+type tableRow struct {
+ cells []tableCell
+}
+
+// writeDisplayWidthTable aligns columns by terminal cells, not bytes or rune
+// count. This keeps CJK, emoji, and combining-character paths aligned while
+// allowing ANSI-colored severity text to use its uncolored widthText.
+func writeDisplayWidthTable(w io.Writer, headers []string, rows []tableRow) error {
+ widths := make([]int, len(headers))
+ for index, header := range headers {
+ widths[index] = runewidth.StringWidth(header)
+ }
+ for _, row := range rows {
+ for index, cell := range row.cells {
+ if index < len(widths) && cell.displayWidth() > widths[index] {
+ widths[index] = cell.displayWidth()
+ }
+ }
+ }
+
+ headerCells := make([]tableCell, len(headers))
+ separatorCells := make([]tableCell, len(headers))
+ for index, header := range headers {
+ headerCells[index] = tableCell{text: header}
+ separatorCells[index] = tableCell{text: strings.Repeat("-", widths[index])}
+ }
+ if err := writeTableRow(w, tableRow{cells: headerCells}, widths); err != nil {
+ return fmt.Errorf("failed to write table header: %w", err)
+ }
+ if err := writeTableRow(w, tableRow{cells: separatorCells}, widths); err != nil {
+ return fmt.Errorf("failed to write table separator: %w", err)
+ }
+ for _, row := range rows {
+ if err := writeTableRow(w, row, widths); err != nil {
+ return fmt.Errorf("failed to write table row: %w", err)
+ }
+ }
+ return nil
+}
+
+func writeTableRow(w io.Writer, row tableRow, widths []int) error {
+ for index, cell := range row.cells {
+ if index > 0 {
+ if _, err := io.WriteString(w, strings.Repeat(" ", columnPadding)); err != nil {
+ return err
+ }
+ }
+ if _, err := io.WriteString(w, cell.text); err != nil {
+ return err
+ }
+ if index < len(row.cells)-1 {
+ padding := widths[index] - cell.displayWidth()
+ if padding > 0 {
+ if _, err := io.WriteString(w, strings.Repeat(" ", padding)); err != nil {
+ return err
+ }
+ }
+ }
+ }
+ _, err := io.WriteString(w, "\n")
+ return err
+}
+
// colorizeSeverity wraps the severity text with the appropriate ANSI color code.
func (f *Formatter) colorizeSeverity(sev finding.Severity, text string) string {
var color string
diff --git a/internal/output/table/table_formatter_test.go b/internal/output/table/table_formatter_test.go
index 0a26a60..e48bbc3 100644
--- a/internal/output/table/table_formatter_test.go
+++ b/internal/output/table/table_formatter_test.go
@@ -3,11 +3,14 @@ package table
import (
"bytes"
"errors"
+ "strconv"
"strings"
"testing"
"time"
+ "unicode/utf8"
"github.com/HodeTech/leakwatch/pkg/finding"
+ "github.com/mattn/go-runewidth"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -138,6 +141,35 @@ func TestFormatter_Format_ShowRawTrue_AddsRawColumn(t *testing.T) {
"ShowRaw=true must include the raw secret value in table output")
}
+func TestFormatter_Format_ShowRawTrue_UsesReversibleTerminalSafeEncoding(t *testing.T) {
+ raw := "abc\ndef\tghi\x1b\x00\\suffix\u0301\u202e" + string([]byte{0xff})
+ encoded := quoteRawForTable(raw)
+
+ recovered, err := strconv.Unquote(encoded)
+ require.NoError(t, err)
+ assert.Equal(t, []byte(raw), []byte(recovered), "quoted RAW value must round-trip byte-for-byte")
+ assert.True(t, utf8.ValidString(encoded), "table display must remain valid UTF-8")
+ for _, unsafe := range []string{"\n", "\t", "\x1b", "\x00", "\u202e"} {
+ assert.NotContains(t, encoded, unsafe, "encoded RAW must not contain literal terminal controls")
+ }
+
+ f := &Formatter{ShowRaw: true}
+ var buf bytes.Buffer
+ require.NoError(t, f.Format(&buf, []finding.Finding{{
+ DetectorID: "generic-secret",
+ Raw: raw,
+ }}))
+ assert.Contains(t, buf.String(), encoded)
+}
+
+func TestTableCell_DisplayWidthUsesExplicitUncoloredText(t *testing.T) {
+ cell := tableCell{text: "\x1b[31mCRITICAL\x1b[0m", widthText: "CRITICAL"}
+ assert.Equal(t, 8, cell.displayWidth(), "ANSI decoration must not contribute to table alignment")
+
+ unicodeCell := tableCell{text: "ignored", widthText: "秘密"}
+ assert.Equal(t, 4, unicodeCell.displayWidth(), "widthText must retain terminal-cell semantics")
+}
+
func TestFormatter_Format_ShowRawFalse_DoesNotMutateOriginal(t *testing.T) {
f := &Formatter{ShowRaw: false}
var buf bytes.Buffer
@@ -156,7 +188,7 @@ func TestFormatter_Format_ShowRawFalse_DoesNotMutateOriginal(t *testing.T) {
"Format must not mutate the original slice")
}
-func TestFormatter_Format_ColumnsAligned_TabwriterProducesAlignedOutput(t *testing.T) {
+func TestFormatter_Format_ColumnsAligned_ProducesAlignedOutput(t *testing.T) {
f := &Formatter{}
var buf bytes.Buffer
@@ -173,6 +205,48 @@ func TestFormatter_Format_ColumnsAligned_TabwriterProducesAlignedOutput(t *testi
require.GreaterOrEqual(t, len(lines), 4)
}
+func TestFormatter_Format_UnicodeDisplayWidthAlignsColumns(t *testing.T) {
+ f := &Formatter{}
+ var buf bytes.Buffer
+ findings := []finding.Finding{
+ {
+ DetectorID: "detector-a",
+ Severity: finding.SeverityLow,
+ Redacted: "first-redacted",
+ SourceMetadata: finding.SourceMetadata{
+ FilePath: "設定/秘密.yaml",
+ },
+ },
+ {
+ DetectorID: "detector-b",
+ Severity: finding.SeverityHigh,
+ Redacted: "second-redacted",
+ SourceMetadata: finding.SourceMetadata{
+ FilePath: "cafe\u0301/emoji-🔐.yaml",
+ },
+ },
+ }
+
+ require.NoError(t, f.Format(&buf, findings))
+ lines := strings.Split(buf.String(), "\n")
+ require.GreaterOrEqual(t, len(lines), 4)
+
+ wantColumn := -1
+ for _, item := range []struct {
+ line int
+ needle string
+ }{{0, "REDACTED"}, {2, "first-redacted"}, {3, "second-redacted"}} {
+ line := lines[item.line]
+ byteIndex := strings.Index(line, item.needle)
+ require.NotEqual(t, -1, byteIndex)
+ column := runewidth.StringWidth(line[:byteIndex])
+ if wantColumn < 0 {
+ wantColumn = column
+ }
+ assert.Equal(t, wantColumn, column, "display column drifted for %q", item.needle)
+ }
+}
+
func TestFormatter_Format_WithRemediation_ShowsRemediationTitle(t *testing.T) {
f := &Formatter{}
var buf bytes.Buffer
diff --git a/internal/remediation/guidance.go b/internal/remediation/guidance.go
index c40940a..3b01cc4 100644
--- a/internal/remediation/guidance.go
+++ b/internal/remediation/guidance.go
@@ -214,6 +214,25 @@ func init() {
},
})
+ Register("structured-config-secret", finding.Remediation{
+ Title: "Move Configuration Secret to Secure Storage",
+ Steps: []string{
+ "Treat the exposed value as compromised and rotate or revoke it at its owning service.",
+ "Remove the plaintext value from the configuration file and its version history.",
+ "Store the replacement in a secrets manager or the platform's protected local-secret mechanism.",
+ "Reference the protected value through environment or secret-manager indirection.",
+ "Redeploy affected services and verify that the old credential no longer works.",
+ },
+ DocURL: "https://cheatsheetseries.owasp.org/cheatsheets/Secrets_Management_Cheat_Sheet.html",
+ Urgency: "immediate",
+ Checklist: []string{
+ "Review repository, artifact, backup, log, and chat history for additional copies.",
+ "Audit the owning service for unauthorized use of the exposed credential.",
+ checkNotifySecurityTeam,
+ checkScanCodebaseForKey,
+ },
+ })
+
Register("openai-api-key", finding.Remediation{
Title: "Rotate OpenAI API Key",
Steps: []string{
@@ -505,10 +524,10 @@ func init() {
})
Register("twilio-api-key", finding.Remediation{
- Title: "Rotate Twilio API Key",
+ Title: "Rotate Twilio API Key Pair",
Steps: []string{
"Go to Twilio Console > API Keys.",
- stepDeleteCompromisedKey,
+ "Identify the API Key SID paired with the exposed secret and revoke that key.",
stepCreateNewAPIKey,
},
DocURL: "https://www.twilio.com/docs/iam/api-keys",
@@ -715,8 +734,8 @@ func init() {
Register("shopify-access-token", finding.Remediation{
Title: "Rotate Shopify Access Token",
Steps: []string{
- "Go to Shopify Admin > Apps > Manage private apps.",
- "Rotate the compromised access token.",
+ "Go to Shopify Admin > Settings > Apps and sales channels > Develop apps.",
+ "Open the affected app and revoke or rotate the compromised Admin API access token.",
stepUpdateIntegrationsToken,
},
DocURL: "https://shopify.dev/docs/apps/auth",
@@ -728,14 +747,14 @@ func init() {
})
Register("supabase-service-key", finding.Remediation{
- Title: "Rotate Supabase Service Key",
+ Title: "Revoke Supabase Personal Access Token",
Steps: []string{
- "Go to app.supabase.com > Project Settings > API.",
- "Regenerate the service role key.",
- "Update all backend services with the new key.",
+ "Open the Supabase Dashboard account access-token page.",
+ "Identify and revoke the compromised personal access token.",
+ "Create a replacement only if the CLI or automation still requires one, then update those consumers.",
},
- DocURL: "https://supabase.com/docs/guides/api",
- ConsoleURL: "https://app.supabase.com",
+ DocURL: "https://supabase.com/docs/reference/api/introduction",
+ ConsoleURL: "https://supabase.com/dashboard/account/tokens",
Urgency: "immediate",
})
diff --git a/internal/remediation/registry_coverage_test.go b/internal/remediation/registry_coverage_test.go
index 4aadfa9..621d47d 100644
--- a/internal/remediation/registry_coverage_test.go
+++ b/internal/remediation/registry_coverage_test.go
@@ -48,7 +48,7 @@ import (
_ "github.com/HodeTech/leakwatch/internal/detector/figma" // register figma detector
_ "github.com/HodeTech/leakwatch/internal/detector/ftp" // register ftp credentials detector
_ "github.com/HodeTech/leakwatch/internal/detector/gcp" // register gcp service-account detector
- _ "github.com/HodeTech/leakwatch/internal/detector/generic" // register generic api-key detector
+ _ "github.com/HodeTech/leakwatch/internal/detector/generic" // register generic API-key and structured-config detectors
_ "github.com/HodeTech/leakwatch/internal/detector/github" // register github detectors (pat + oauth)
_ "github.com/HodeTech/leakwatch/internal/detector/gitlab" // register gitlab detector
_ "github.com/HodeTech/leakwatch/internal/detector/grafana" // register grafana detector
diff --git a/internal/scanner/scanner.go b/internal/scanner/scanner.go
index a0f0d87..a6ba55b 100644
--- a/internal/scanner/scanner.go
+++ b/internal/scanner/scanner.go
@@ -11,9 +11,11 @@ package scanner
import (
"context"
+ "errors"
"fmt"
"log/slog"
"path/filepath"
+ "sort"
"sync"
"time"
@@ -59,6 +61,16 @@ type Config struct {
VerifyTimeout time.Duration
VerifyConcurrency int
VerifyRateLimit float64
+ // GrafanaInstanceURL is accepted only from the explicit command-line flag,
+ // never from repository config or detector ExtraData. It is therefore a
+ // trusted operator choice rather than scan-controlled input.
+ GrafanaInstanceURL string
+ // TrustedVerifierOrigins maps a context-required detector ID to an HTTPS
+ // origin explicitly supplied by the operator on this invocation. The cmd
+ // layer never resolves this map from project configuration or environment
+ // variables, preventing scanned repositories from choosing credential
+ // destinations.
+ TrustedVerifierOrigins map[string]string
// CustomRules are user-defined YAML custom rules from the `custom-rules:`
// config block. BuildEngineConfig compiles them into detectors threaded
@@ -125,6 +137,11 @@ func BuildEngineConfig(cfg *Config) (engine.Config, error) {
slog.Warn("--only-verified has no effect when --no-verify is set")
}
+ verifiers, err := configuredVerifiers(cfg)
+ if err != nil {
+ return engine.Config{}, err
+ }
+
return engine.Config{
Concurrency: cfg.Concurrency,
Detectors: detectors,
@@ -132,12 +149,45 @@ func BuildEngineConfig(cfg *Config) (engine.Config, error) {
EntropyThreshold: cfg.EntropyThreshold,
ShowRaw: cfg.ShowRaw,
VerifierConfig: verifierCfg,
- Verifiers: verifier.All(),
+ Verifiers: verifiers,
OnlyVerified: cfg.OnlyVerified,
MinSeverity: cfg.MinSeverity,
}, nil
}
+func configuredVerifiers(cfg *Config) ([]verifier.Verifier, error) {
+ verifiers := verifier.All()
+ origins := make(map[string]string, len(cfg.TrustedVerifierOrigins)+1)
+ for detectorID, origin := range cfg.TrustedVerifierOrigins {
+ origins[detectorID] = origin
+ }
+ if cfg.GrafanaInstanceURL != "" {
+ if origin, exists := origins["grafana-api-key"]; exists && origin != cfg.GrafanaInstanceURL {
+ return nil, fmt.Errorf("configure trusted verification: conflicting Grafana origins were supplied")
+ }
+ origins["grafana-api-key"] = cfg.GrafanaInstanceURL
+ }
+ if len(origins) == 0 {
+ return verifiers, nil
+ }
+
+ ids := make([]string, 0, len(origins))
+ for detectorID := range origins {
+ ids = append(ids, detectorID)
+ }
+ sort.Strings(ids)
+
+ configured := verifiers
+ for _, detectorID := range ids {
+ var err error
+ configured, err = verifier.ConfigureTrustedInstance(configured, detectorID, origins[detectorID])
+ if err != nil {
+ return nil, fmt.Errorf("configure trusted verification for %q: %w", detectorID, err)
+ }
+ }
+ return configured, nil
+}
+
// Run executes a single-source scan: it builds the engine, runs the scan under
// the given (already signal-aware) context, and post-processes the result
// (.leakwatchignore filtering and remediation enrichment).
@@ -146,7 +196,9 @@ func BuildEngineConfig(cfg *Config) (engine.Config, error) {
// non-nil on interruption (ctx cancelled) or a terminal source failure; in the
// interruption case a partial, non-nil result is still returned so the caller can
// render partial findings and choose a distinct interrupted exit code. Only a
-// pre-scan failure (e.g. source validation) yields a nil result.
+// a non-cancellation pre-scan failure yields a nil result. Cancellation during
+// source validation returns an interrupted, non-nil result so the CLI can map
+// it to exit code 3 instead of a generic failure.
func Run(ctx context.Context, cfg *Config, src source.Source) (*engine.ScanResult, error) {
engCfg, err := BuildEngineConfig(cfg)
if err != nil {
@@ -203,6 +255,7 @@ func ScanRepos(ctx context.Context, cfg *Config, repoURLs []string, srcOpts []gi
result, scanErr := eng.Scan(ctx, src)
if closeErr := src.Close(); closeErr != nil {
slog.Warn("failed to clean up repo", "url", displayURL, "error", closeErr)
+ scanErr = errors.Join(scanErr, fmt.Errorf("failed to clean up repo %s: %w", displayURL, closeErr))
}
if result != nil {
slog.Info("repository scan completed", "url", displayURL, "findings", len(result.Findings), "files", result.ScannedChunks)
@@ -229,7 +282,7 @@ func ScanRepos(ctx context.Context, cfg *Config, repoURLs []string, srcOpts []gi
var aggErr error
if len(scanErrors) > 0 {
- aggErr = fmt.Errorf("%d repository scans failed", len(scanErrors))
+ aggErr = fmt.Errorf("%d repository scans failed: %w", len(scanErrors), errors.Join(scanErrors...))
}
return combined, aggErr
}
@@ -273,10 +326,10 @@ func scanReposParallel(
mu.Lock()
defer mu.Unlock()
+ if err != nil {
+ errs = append(errs, err)
+ }
if result == nil {
- if err != nil {
- errs = append(errs, err)
- }
return
}
findings = append(findings, result.Findings...)
diff --git a/internal/scanner/scanner_test.go b/internal/scanner/scanner_test.go
index 5027866..bb4eb04 100644
--- a/internal/scanner/scanner_test.go
+++ b/internal/scanner/scanner_test.go
@@ -2,6 +2,7 @@ package scanner
import (
"context"
+ "errors"
"fmt"
"sync"
"testing"
@@ -22,9 +23,9 @@ type fakeSource struct {
data []byte
}
-func (f *fakeSource) Type() string { return "fake" }
-func (f *fakeSource) Validate() error { return nil }
-func (f *fakeSource) Err() error { return nil }
+func (f *fakeSource) Type() string { return "fake" }
+func (f *fakeSource) Validate(context.Context) error { return nil }
+func (f *fakeSource) Err() error { return nil }
func (f *fakeSource) Chunks(ctx context.Context) <-chan source.Chunk {
ch := make(chan source.Chunk)
@@ -179,6 +180,7 @@ func TestScanReposParallel_SharesSingleEngine(t *testing.T) {
func TestScanReposParallel_AggregatesFindingsAndErrors(t *testing.T) {
eng := engine.New(engine.Config{Concurrency: 1})
+ partialErr := fmt.Errorf("partial")
scanOne := func(_ context.Context, _ *engine.Engine, url string) (*engine.ScanResult, error) {
switch url {
@@ -187,8 +189,8 @@ func TestScanReposParallel_AggregatesFindingsAndErrors(t *testing.T) {
return nil, fmt.Errorf("clone failed")
case "partial":
// Partial result WITH an error (e.g. interrupted mid-scan): findings
- // are still aggregated, the error is not separately recorded.
- return &engine.ScanResult{Findings: []finding.Finding{{ID: "p"}}, ScannedChunks: 1}, fmt.Errorf("partial")
+ // and its error must both be aggregated.
+ return &engine.ScanResult{Findings: []finding.Finding{{ID: "p"}}, ScannedChunks: 1}, partialErr
default:
return &engine.ScanResult{Findings: []finding.Finding{{ID: url}}, ScannedChunks: 2}, nil
}
@@ -197,7 +199,8 @@ func TestScanReposParallel_AggregatesFindingsAndErrors(t *testing.T) {
urls := []string{"ok1", "ok2", "fail", "partial"}
findings, chunks, errs := scanReposParallel(context.Background(), eng, urls, 2, scanOne)
- assert.Len(t, errs, 1, "only the nil-result failure is recorded as an error")
+ assert.Len(t, errs, 2, "both fatal and partial-result failures must be recorded")
+ assert.ErrorIs(t, errors.Join(errs...), partialErr)
assert.Len(t, findings, 3, "ok1, ok2 and partial's findings are aggregated")
assert.Equal(t, 2+2+1, chunks)
}
diff --git a/internal/source/container/container.go b/internal/source/container/container.go
index 1badf86..8f4dfb8 100644
--- a/internal/source/container/container.go
+++ b/internal/source/container/container.go
@@ -122,7 +122,10 @@ func (s *ContainerSource) captureErr(err error) {
}
// Validate checks that the image reference is parseable and accessible.
-func (s *ContainerSource) Validate() error {
+func (s *ContainerSource) Validate(ctx context.Context) error {
+ if err := ctx.Err(); err != nil {
+ return err
+ }
_, err := name.ParseReference(s.imageRef)
if err != nil {
return fmt.Errorf("invalid image reference %q: %w", s.imageRef, err)
diff --git a/internal/source/container/container_test.go b/internal/source/container/container_test.go
index dc8a8a1..5591413 100644
--- a/internal/source/container/container_test.go
+++ b/internal/source/container/container_test.go
@@ -19,17 +19,17 @@ func TestContainerSource_Type_ReturnsContainer(t *testing.T) {
func TestContainerSource_Validate_ValidRef_ReturnsNoError(t *testing.T) {
s := New("nginx:latest")
- assert.NoError(t, s.Validate())
+ assert.NoError(t, s.Validate(context.Background()))
}
func TestContainerSource_Validate_InvalidRef_ReturnsError(t *testing.T) {
s := New(":::invalid")
- assert.Error(t, s.Validate())
+ assert.Error(t, s.Validate(context.Background()))
}
func TestContainerSource_Validate_FullRef_ReturnsNoError(t *testing.T) {
s := New("ghcr.io/org/repo:v1.0.0")
- assert.NoError(t, s.Validate())
+ assert.NoError(t, s.Validate(context.Background()))
}
func TestShouldSkipContainerPath(t *testing.T) {
diff --git a/internal/source/filesystem/filesystem.go b/internal/source/filesystem/filesystem.go
index 4efb2b5..913f812 100644
--- a/internal/source/filesystem/filesystem.go
+++ b/internal/source/filesystem/filesystem.go
@@ -87,11 +87,14 @@ func (s *FilesystemSource) Type() string {
// Validate checks that every scan root exists and is accessible. A root may be
// either a directory or a regular file; only inaccessibility (e.g. a
// non-existent path or a permission error) is rejected.
-func (s *FilesystemSource) Validate() error {
+func (s *FilesystemSource) Validate(ctx context.Context) error {
if len(s.roots) == 0 {
return fmt.Errorf("no scan paths provided")
}
for _, root := range s.roots {
+ if err := ctx.Err(); err != nil {
+ return err
+ }
if _, err := os.Stat(root); err != nil {
return fmt.Errorf("source path inaccessible %s: %w", root, err)
}
diff --git a/internal/source/filesystem/filesystem_test.go b/internal/source/filesystem/filesystem_test.go
index 4a3e33f..92d4d16 100644
--- a/internal/source/filesystem/filesystem_test.go
+++ b/internal/source/filesystem/filesystem_test.go
@@ -56,18 +56,18 @@ func TestFilesystemSource_New_CleansPath(t *testing.T) {
}
}
-// Validate() calls os.Stat directly against s.root, so these tests genuinely
+// Validate(ctx) calls os.Stat directly against s.root, so these tests genuinely
// need a real filesystem rather than an injectable fs.FS.
func TestFilesystemSource_Validate_ValidDir(t *testing.T) {
dir := t.TempDir()
s := New(dir)
- assert.NoError(t, s.Validate())
+ assert.NoError(t, s.Validate(context.Background()))
}
func TestFilesystemSource_Validate_NonExistentDir(t *testing.T) {
s := New("/nonexistent/path")
- assert.Error(t, s.Validate())
+ assert.Error(t, s.Validate(context.Background()))
}
func TestFilesystemSource_Validate_SingleFile(t *testing.T) {
@@ -78,7 +78,7 @@ func TestFilesystemSource_Validate_SingleFile(t *testing.T) {
// A single file is now a valid scan root: `scan fs ` must work.
s := New(f.Name())
- assert.NoError(t, s.Validate())
+ assert.NoError(t, s.Validate(context.Background()))
}
func TestFilesystemSource_Validate_MultipleRoots(t *testing.T) {
@@ -87,10 +87,10 @@ func TestFilesystemSource_Validate_MultipleRoots(t *testing.T) {
require.NoError(t, os.WriteFile(f, []byte("x"), 0o644))
s := NewMulti([]string{dir, f})
- assert.NoError(t, s.Validate())
+ assert.NoError(t, s.Validate(context.Background()))
bad := NewMulti([]string{dir, "/nonexistent/path"})
- assert.Error(t, bad.Validate())
+ assert.Error(t, bad.Validate(context.Background()))
}
// shouldSkip is pure filtering logic over a path string and an fs.DirEntry;
diff --git a/internal/source/gcs/gcs.go b/internal/source/gcs/gcs.go
index 0e3df71..158cd17 100644
--- a/internal/source/gcs/gcs.go
+++ b/internal/source/gcs/gcs.go
@@ -10,6 +10,7 @@ import (
"fmt"
"io"
"log/slog"
+ "sync"
"time"
gcsstorage "cloud.google.com/go/storage"
@@ -24,11 +25,8 @@ import (
// defaultMaxFileSize is the maximum object size to scan (10 MB).
const defaultMaxFileSize int64 = 10 * 1024 * 1024
-// validateTimeout bounds the network calls made by Validate. The
-// source.Source interface's Validate() method takes no context.Context
-// parameter, so the caller's own cancellation cannot be threaded through
-// here; a bounded timeout at least prevents an unreachable/misconfigured
-// bucket from hanging Validate indefinitely.
+// validateTimeout bounds validation when the caller provides no earlier
+// deadline. An earlier caller deadline or cancellation always wins.
const validateTimeout = 30 * time.Second
// gcsClient defines the subset of the GCS API used by GCSSource.
@@ -102,6 +100,7 @@ type GCSSource struct {
maxFileSize int64
bufferSize int
excludePaths []string
+ clientMu sync.RWMutex
client gcsClient
// err records the first terminal failure that aborted listing (client
@@ -138,6 +137,21 @@ func (s *GCSSource) Err() error {
return s.err
}
+// Close releases the GCS client. A close failure retains the client so callers
+// can report the error and retry cleanup; successful repeated calls are no-ops.
+func (s *GCSSource) Close() error {
+ s.clientMu.Lock()
+ defer s.clientMu.Unlock()
+ if s.client == nil {
+ return nil
+ }
+ if err := s.client.Close(); err != nil {
+ return fmt.Errorf("gcs client close failed: %w", err)
+ }
+ s.client = nil
+ return nil
+}
+
// captureErr records the first terminal error that aborted chunk production. It
// is called only from the single Chunks goroutine, before close(ch), so a plain
// field write is safe (the channel close/drain publishes it to Err's reader).
@@ -156,19 +170,24 @@ func (s *GCSSource) captureErr(err error) {
// Validate checks that the GCS bucket is accessible.
// It initializes the GCS client if not already set and checks bucket attributes.
-func (s *GCSSource) Validate() error {
+func (s *GCSSource) Validate(ctx context.Context) error {
if s.bucket == "" {
return fmt.Errorf("gcs bucket name is required")
}
- ctx, cancel := context.WithTimeout(context.Background(), validateTimeout)
+ ctx, cancel := context.WithTimeout(ctx, validateTimeout)
defer cancel()
if err := s.ensureClient(ctx); err != nil {
return fmt.Errorf("gcs client initialization failed: %w", err)
}
- _, err := s.client.Bucket(s.bucket).Attrs(ctx)
+ client, release := s.acquireClient()
+ if client == nil {
+ return fmt.Errorf("gcs client was closed before validation")
+ }
+ defer release()
+ _, err := client.Bucket(s.bucket).Attrs(ctx)
if err != nil {
return fmt.Errorf("gcs bucket inaccessible %q: %w", s.bucket, err)
}
@@ -188,19 +207,22 @@ func (s *GCSSource) Chunks(ctx context.Context) <-chan source.Chunk {
s.captureErr(fmt.Errorf("gcs client initialization failed: %w", err))
return
}
- defer func() {
- if err := s.client.Close(); err != nil {
- slog.Warn("gcs client close failed", "error", err)
- }
- }()
+ client, release := s.acquireClient()
+ if client == nil {
+ s.captureErr(fmt.Errorf("gcs client was closed before object listing"))
+ return
+ }
+ defer release()
- s.listAndSendChunks(ctx, ch)
+ s.listAndSendChunks(ctx, ch, client)
}()
return ch
}
// ensureClient initializes the GCS client if not already set.
func (s *GCSSource) ensureClient(ctx context.Context) error {
+ s.clientMu.Lock()
+ defer s.clientMu.Unlock()
if s.client != nil {
return nil
}
@@ -219,14 +241,23 @@ func (s *GCSSource) ensureClient(ctx context.Context) error {
return nil
}
+func (s *GCSSource) acquireClient() (gcsClient, func()) {
+ s.clientMu.RLock()
+ if s.client == nil {
+ s.clientMu.RUnlock()
+ return nil, func() {}
+ }
+ return s.client, s.clientMu.RUnlock
+}
+
// listAndSendChunks iterates through bucket objects and emits chunks.
-func (s *GCSSource) listAndSendChunks(ctx context.Context, ch chan<- source.Chunk) {
+func (s *GCSSource) listAndSendChunks(ctx context.Context, ch chan<- source.Chunk, client gcsClient) {
query := &gcsstorage.Query{}
if s.prefix != "" {
query.Prefix = s.prefix
}
- it := s.client.Bucket(s.bucket).Objects(ctx, query)
+ it := client.Bucket(s.bucket).Objects(ctx, query)
for {
select {
case <-ctx.Done():
@@ -269,7 +300,7 @@ func (s *GCSSource) listAndSendChunks(ctx context.Context, ch chan<- source.Chun
continue
}
- data, err := s.downloadObject(ctx, key)
+ data, err := s.downloadObject(ctx, client, key)
if err != nil {
slog.Warn("gcs object download failed", "key", key, "error", err)
continue
@@ -302,8 +333,8 @@ func (s *GCSSource) listAndSendChunks(ctx context.Context, ch chan<- source.Chun
}
// downloadObject fetches the content of a single GCS object.
-func (s *GCSSource) downloadObject(ctx context.Context, key string) ([]byte, error) {
- reader, err := s.client.Bucket(s.bucket).Object(key).NewReader(ctx)
+func (s *GCSSource) downloadObject(ctx context.Context, client gcsClient, key string) ([]byte, error) {
+ reader, err := client.Bucket(s.bucket).Object(key).NewReader(ctx)
if err != nil {
return nil, fmt.Errorf("open object %q: %w", key, err)
}
diff --git a/internal/source/gcs/gcs_test.go b/internal/source/gcs/gcs_test.go
index bb1acd5..a71fa04 100644
--- a/internal/source/gcs/gcs_test.go
+++ b/internal/source/gcs/gcs_test.go
@@ -6,6 +6,7 @@ import (
"io"
"strings"
"testing"
+ "time"
gcsstorage "cloud.google.com/go/storage"
"github.com/stretchr/testify/assert"
@@ -17,8 +18,10 @@ import (
// mockGCSClient implements the gcsClient interface for testing.
type mockGCSClient struct {
- buckets map[string]*mockBucketHandle
- closed bool
+ buckets map[string]*mockBucketHandle
+ closed bool
+ closeErr error
+ closeCalls int
}
func (m *mockGCSClient) Bucket(name string) bucketHandle {
@@ -29,19 +32,32 @@ func (m *mockGCSClient) Bucket(name string) bucketHandle {
}
func (m *mockGCSClient) Close() error {
+ m.closeCalls++
+ if m.closeErr != nil {
+ return m.closeErr
+ }
m.closed = true
return nil
}
// mockBucketHandle implements the bucketHandle interface for testing.
type mockBucketHandle struct {
- name string
- notFound bool
- objects []*gcsstorage.ObjectAttrs
- data map[string]string
+ name string
+ notFound bool
+ objects []*gcsstorage.ObjectAttrs
+ data map[string]string
+ attrsBlock bool
+ attrsDeadline time.Time
+ attrsHasDeadline bool
+ readerStarted chan struct{}
}
-func (b *mockBucketHandle) Attrs(_ context.Context) (*gcsstorage.BucketAttrs, error) {
+func (b *mockBucketHandle) Attrs(ctx context.Context) (*gcsstorage.BucketAttrs, error) {
+ b.attrsDeadline, b.attrsHasDeadline = ctx.Deadline()
+ if b.attrsBlock {
+ <-ctx.Done()
+ return nil, ctx.Err()
+ }
if b.notFound {
return nil, fmt.Errorf("bucket not found")
}
@@ -64,15 +80,22 @@ func (b *mockBucketHandle) Object(name string) objectHandle {
if b.data != nil {
content = b.data[name]
}
- return &mockObjectHandle{content: content}
+ return &mockObjectHandle{content: content, readerStarted: b.readerStarted}
}
// mockObjectHandle implements the objectHandle interface for testing.
type mockObjectHandle struct {
- content string
+ content string
+ readerStarted chan struct{}
}
func (o *mockObjectHandle) NewReader(_ context.Context) (io.ReadCloser, error) {
+ if o.readerStarted != nil {
+ select {
+ case o.readerStarted <- struct{}{}:
+ default:
+ }
+ }
return io.NopCloser(strings.NewReader(o.content)), nil
}
@@ -136,7 +159,7 @@ func TestWithMaxFileSize_ValidValue_Applied(t *testing.T) {
func TestGCSSource_Validate_EmptyBucket_ReturnsError(t *testing.T) {
s := New("")
- err := s.Validate()
+ err := s.Validate(context.Background())
require.Error(t, err)
assert.Contains(t, err.Error(), "bucket name is required")
}
@@ -149,7 +172,34 @@ func TestGCSSource_Validate_AccessibleBucket_ReturnsNil(t *testing.T) {
}
s := New("my-bucket")
s.client = mock
- assert.NoError(t, s.Validate())
+ assert.NoError(t, s.Validate(context.Background()))
+}
+
+func TestGCSSource_Validate_AppliesDefaultDeadline(t *testing.T) {
+ bucket := &mockBucketHandle{name: "my-bucket"}
+ mock := &mockGCSClient{buckets: map[string]*mockBucketHandle{"my-bucket": bucket}}
+ s := New("my-bucket")
+ s.client = mock
+ started := time.Now()
+
+ require.NoError(t, s.Validate(context.Background()))
+ require.True(t, bucket.attrsHasDeadline)
+ assert.WithinDuration(t, started.Add(validateTimeout), bucket.attrsDeadline, time.Second)
+}
+
+func TestGCSSource_Validate_CallerDeadlineCancelsRequest(t *testing.T) {
+ bucket := &mockBucketHandle{name: "my-bucket", attrsBlock: true}
+ mock := &mockGCSClient{buckets: map[string]*mockBucketHandle{"my-bucket": bucket}}
+ s := New("my-bucket")
+ s.client = mock
+ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
+ defer cancel()
+ started := time.Now()
+
+ err := s.Validate(ctx)
+ require.ErrorIs(t, err, context.DeadlineExceeded)
+ assert.Less(t, time.Since(started), time.Second)
+ assert.WithinDuration(t, started.Add(20*time.Millisecond), bucket.attrsDeadline, 50*time.Millisecond)
}
func TestGCSSource_Validate_InaccessibleBucket_ReturnsError(t *testing.T) {
@@ -158,9 +208,11 @@ func TestGCSSource_Validate_InaccessibleBucket_ReturnsError(t *testing.T) {
}
s := New("missing-bucket")
s.client = mock
- err := s.Validate()
+ err := s.Validate(context.Background())
require.Error(t, err)
assert.Contains(t, err.Error(), "inaccessible")
+ require.NoError(t, s.Close())
+ assert.True(t, mock.closed, "a client created before validation failure must remain closeable")
}
func TestGCSSource_Chunks_SendsTextObjects(t *testing.T) {
@@ -392,7 +444,7 @@ func TestGCSSource_DownloadObject_AtLimit_NotSkipped(t *testing.T) {
s := New("my-bucket", WithMaxFileSize(1024))
s.client = mock
- data, err := s.downloadObject(context.Background(), "exact.txt")
+ data, err := s.downloadObject(context.Background(), mock, "exact.txt")
require.NoError(t, err)
assert.Len(t, data, 1024)
}
@@ -434,7 +486,7 @@ func TestGCSSource_New_WithExcludePaths_StoresPatterns(t *testing.T) {
assert.Equal(t, []string{"a/**", "b"}, s.excludePaths)
}
-func TestGCSSource_Chunks_ClosesClientOnCompletion(t *testing.T) {
+func TestGCSSource_CallerOwnsClientCloseAfterChunksComplete(t *testing.T) {
mock := &mockGCSClient{
buckets: map[string]*mockBucketHandle{
"my-bucket": {
@@ -451,10 +503,70 @@ func TestGCSSource_Chunks_ClosesClientOnCompletion(t *testing.T) {
s.client = mock
for range s.Chunks(context.Background()) {
- // Drain the channel so Chunks' goroutine can finish and close the client.
+ // Drain the channel so Chunks' goroutine releases its client lease.
+ }
+
+ assert.False(t, mock.closed, "chunk production must not compete with the CLI cleanup owner")
+ require.NoError(t, s.Close())
+ assert.True(t, mock.closed)
+ assert.Nil(t, s.client, "successful close must make later cleanup idempotent")
+}
+
+func TestGCSSource_CloseWaitsForCancelledProducerAndClosesOnce(t *testing.T) {
+ readerStarted := make(chan struct{}, 1)
+ mock := &mockGCSClient{buckets: map[string]*mockBucketHandle{
+ "my-bucket": {
+ name: "my-bucket",
+ objects: []*gcsstorage.ObjectAttrs{{Name: "secret.txt", Size: 6}},
+ data: map[string]string{"secret.txt": "secret"},
+ readerStarted: readerStarted,
+ },
+ }}
+ s := New("my-bucket")
+ s.bufferSize = 0
+ s.client = mock
+ ctx, cancel := context.WithCancel(context.Background())
+ chunks := s.Chunks(ctx)
+
+ select {
+ case <-readerStarted:
+ case <-time.After(time.Second):
+ t.Fatal("producer did not begin object download")
+ }
+ closed := make(chan error, 1)
+ go func() { closed <- s.Close() }()
+ select {
+ case err := <-closed:
+ t.Fatalf("Close returned while producer still held the client lease: %v", err)
+ case <-time.After(20 * time.Millisecond):
}
- assert.True(t, mock.closed, "gcs client must be closed once Chunks completes")
+ cancel()
+ for range chunks {
+ }
+ require.NoError(t, <-closed)
+ assert.Equal(t, 1, mock.closeCalls)
+ require.NoError(t, s.Close())
+ assert.Equal(t, 1, mock.closeCalls)
+}
+
+func TestGCSSource_Close_FailureRemainsRetryable(t *testing.T) {
+ closeErr := fmt.Errorf("synthetic close failure")
+ mock := &mockGCSClient{buckets: map[string]*mockBucketHandle{}, closeErr: closeErr}
+ s := New("my-bucket")
+ s.client = mock
+
+ err := s.Close()
+ require.ErrorIs(t, err, closeErr)
+ assert.Same(t, mock, s.client)
+ assert.Equal(t, 1, mock.closeCalls)
+
+ mock.closeErr = nil
+ require.NoError(t, s.Close())
+ assert.Nil(t, s.client)
+ assert.Equal(t, 2, mock.closeCalls)
+ require.NoError(t, s.Close())
+ assert.Equal(t, 2, mock.closeCalls, "an already-closed source must be a no-op")
}
func TestGCSSource_Chunks_WithPrefix_FiltersObjects(t *testing.T) {
diff --git a/internal/source/git/git.go b/internal/source/git/git.go
index 3c50628..1e15ca8 100644
--- a/internal/source/git/git.go
+++ b/internal/source/git/git.go
@@ -53,6 +53,7 @@ type GitSource struct {
excludePaths []string
tmpDir string // Temporary directory for cloned repos
resolvedBranch string // Cached branch resolution
+ removeAll func(string) error
// err records the first terminal failure that aborted history production
// (start-commit resolution, git log, or the history walk). It is written
@@ -87,23 +88,37 @@ func (s *GitSource) Type() string {
// when --since-commit is set it verifies the given commit is an ancestor of the
// walk's starting point; otherwise the diff-based walk would silently fall back
// to scanning the entire history or an unintended branch.
-func (s *GitSource) Validate() error {
- if s.isRemote() {
- if err := s.cloneRemote(); err != nil {
+
+func (s *GitSource) Validate(ctx context.Context) error {
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ remote, cloneTarget, err := classifyGitTarget(s.target)
+ if err != nil {
+ return fmt.Errorf("invalid git target %s: %w", s.displayTarget, err)
+ }
+ if remote {
+ if err := s.cloneRemote(ctx, cloneTarget); err != nil {
return err
}
} else if err := s.openLocal(); err != nil {
return err
}
+ if err := ctx.Err(); err != nil {
+ return err
+ }
if s.branch != "" {
+ if err := ctx.Err(); err != nil {
+ return err
+ }
if _, err := s.resolveStartHash(); err != nil {
return err
}
}
if s.sinceCommit != "" {
- if err := s.validateSinceCommit(); err != nil {
+ if err := s.validateSinceCommit(ctx); err != nil {
return err
}
}
@@ -115,27 +130,36 @@ func (s *GitSource) Validate() error {
// ancestor of the walk's starting commit (the configured branch tip, or HEAD).
// Returning an explicit error prevents the diff-based scan from silently
// degrading into a full-history scan.
-func (s *GitSource) validateSinceCommit() error {
- sinceCommitObj, err := s.resolveCommitHash(s.sinceCommit)
+func (s *GitSource) validateSinceCommit(ctx context.Context) error {
+ sinceCommitObj, err := s.resolveCommitHash(ctx, s.sinceCommit)
if err != nil {
return err
}
+ if err := ctx.Err(); err != nil {
+ return err
+ }
startHash, err := s.resolveStartHash()
if err != nil {
return fmt.Errorf("failed to resolve start commit for since-commit check: %w", err)
}
+ if err := ctx.Err(); err != nil {
+ return err
+ }
startCommit, err := s.repo.CommitObject(startHash)
if err != nil {
return fmt.Errorf("failed to resolve start commit for since-commit check: %w", err)
}
+ if err := ctx.Err(); err != nil {
+ return err
+ }
if sinceCommitObj.Hash == startCommit.Hash {
return nil
}
- isAncestor, err := sinceCommitObj.IsAncestor(startCommit)
+ isAncestor, err := isAncestorContext(ctx, sinceCommitObj, startCommit)
if err != nil {
return fmt.Errorf("failed to check ancestry of since-commit %q: %w", s.sinceCommit, err)
}
@@ -151,27 +175,43 @@ func (s *GitSource) validateSinceCommit() error {
// resolved by scanning the object store for a unique match, so the documented
// `--since-commit abc1234` short-hash usage works rather than silently producing
// a wrong zero-padded hash via plumbing.NewHash.
-func (s *GitSource) resolveCommitHash(ref string) (*object.Commit, error) {
- ref = strings.TrimSpace(ref)
-
- if len(ref) == fullHashLen {
- if !isHexString(ref) {
- return nil, fmt.Errorf("since-commit %q is not a valid commit hash", ref)
- }
- c, err := s.repo.CommitObject(plumbing.NewHash(ref))
- if err != nil {
- return nil, fmt.Errorf("since-commit %q not found: %w", ref, err)
- }
- return c, nil
+func (s *GitSource) resolveCommitHash(ctx context.Context, ref string) (*object.Commit, error) {
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
+ normalized, full, err := normalizeCommitHashReference(ref)
+ if err != nil {
+ return nil, err
}
+ if full {
+ return s.resolveFullCommitHash(ctx, normalized)
+ }
+ return s.resolveAbbreviatedCommitHash(ctx, normalized)
+}
+func normalizeCommitHashReference(ref string) (string, bool, error) {
+ ref = strings.TrimSpace(ref)
if len(ref) < minAbbrevHashLen {
- return nil, fmt.Errorf("since-commit %q is too short: provide at least %d hex characters or a full 40-character SHA", ref, minAbbrevHashLen)
+ return "", false, fmt.Errorf("since-commit %q is too short: provide at least %d hex characters or a full 40-character SHA", ref, minAbbrevHashLen)
}
if len(ref) > fullHashLen || !isHexString(ref) {
- return nil, fmt.Errorf("since-commit %q is not a valid commit hash", ref)
+ return "", false, fmt.Errorf("since-commit %q is not a valid commit hash", ref)
+ }
+ return ref, len(ref) == fullHashLen, nil
+}
+
+func (s *GitSource) resolveFullCommitHash(ctx context.Context, ref string) (*object.Commit, error) {
+ commit, err := s.repo.CommitObject(plumbing.NewHash(ref))
+ if err != nil {
+ return nil, fmt.Errorf("since-commit %q not found: %w", ref, err)
}
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
+ return commit, nil
+}
+func (s *GitSource) resolveAbbreviatedCommitHash(ctx context.Context, ref string) (*object.Commit, error) {
// Deliberately NOT go-git's Repository.ResolveRevision: it resolves a hash
// prefix by collecting every candidate and returning the first that resolves,
// with no ambiguity signal (there is no ErrAmbiguousRevision in go-git). For a
@@ -191,6 +231,9 @@ func (s *GitSource) resolveCommitHash(ref string) (*object.Commit, error) {
ambiguous bool
)
err = iter.ForEach(func(c *object.Commit) error {
+ if err := ctx.Err(); err != nil {
+ return err
+ }
if !strings.HasPrefix(c.Hash.String(), lower) {
return nil
}
@@ -213,6 +256,25 @@ func (s *GitSource) resolveCommitHash(ref string) (*object.Commit, error) {
return match, nil
}
+func isAncestorContext(ctx context.Context, ancestor, descendant *object.Commit) (bool, error) {
+ iter := object.NewCommitPreorderIter(descendant, nil, nil)
+ defer iter.Close()
+ for {
+ if err := ctx.Err(); err != nil {
+ return false, err
+ }
+ commit, err := iter.Next()
+ switch {
+ case errors.Is(err, io.EOF):
+ return false, nil
+ case err != nil:
+ return false, err
+ case commit.Hash == ancestor.Hash:
+ return true, nil
+ }
+ }
+}
+
// resolveStartHash returns the commit hash the history walk should start from.
// When --branch is configured it resolves that branch's local ref (so the flag
// steers which commits are scanned, not just the metadata label); otherwise it
@@ -236,7 +298,11 @@ func (s *GitSource) resolveStartHash() (plumbing.Hash, error) {
// the temporary directory.
func (s *GitSource) Close() error {
if s.tmpDir != "" {
- if err := os.RemoveAll(s.tmpDir); err != nil {
+ removeAll := s.removeAll
+ if removeAll == nil {
+ removeAll = os.RemoveAll
+ }
+ if err := removeAll(s.tmpDir); err != nil {
return fmt.Errorf("failed to remove temp directory %s: %w", s.tmpDir, err)
}
s.tmpDir = ""
@@ -245,31 +311,75 @@ func (s *GitSource) Close() error {
}
func (s *GitSource) isRemote() bool {
- return strings.HasPrefix(s.target, "http://") ||
- strings.HasPrefix(s.target, "https://") ||
- strings.HasPrefix(s.target, "git@") ||
- strings.HasPrefix(s.target, "ssh://")
+ remote, _, err := classifyGitTarget(s.target)
+ return err == nil && remote
+}
+
+// classifyGitTarget distinguishes the supported network transports from local
+// paths without relying on case-sensitive string prefixes. URI schemes are
+// case-insensitive under RFC 3986; unsupported schemes fail closed instead of
+// being reinterpreted as local filesystem paths. Windows drive-letter paths
+// remain local even when this code is cross-compiled on another platform.
+func classifyGitTarget(target string) (remote bool, normalized string, err error) {
+ if strings.HasPrefix(target, "git@") {
+ return true, target, nil
+ }
+ if len(target) >= 3 && target[1] == ':' &&
+ ((target[0] >= 'a' && target[0] <= 'z') || (target[0] >= 'A' && target[0] <= 'Z')) &&
+ (target[2] == '/' || target[2] == '\\') {
+ return false, target, nil
+ }
+
+ // A colon is legal in a local Unix filename. Treat a target as a URI only
+ // when it uses the explicit scheme:// form; SCP syntax was handled above.
+ if !strings.Contains(target, "://") {
+ return false, target, nil
+ }
+ parsed, parseErr := url.Parse(target)
+ if parseErr != nil {
+ return false, "", fmt.Errorf("malformed target")
+ }
+ if parsed.Scheme == "" {
+ return false, target, nil
+ }
+ switch strings.ToLower(parsed.Scheme) {
+ case "http", "https", "ssh":
+ parsed.Scheme = strings.ToLower(parsed.Scheme)
+ return true, parsed.String(), nil
+ default:
+ return false, "", fmt.Errorf("unsupported URL scheme %q", parsed.Scheme)
+ }
}
func (s *GitSource) openLocal() error {
+ info, err := os.Stat(s.target)
+ if err != nil {
+ return fmt.Errorf("failed to inspect git target %s: %w", s.displayTarget, err)
+ }
+ if !info.IsDir() {
+ return fmt.Errorf("git target %s is not a directory", s.displayTarget)
+ }
// DetectDotGit walks up parent directories to find the repository root,
// matching real git tooling when pointed at a subdirectory of a repo.
repo, err := git.PlainOpenWithOptions(s.target, &git.PlainOpenOptions{DetectDotGit: true})
if err != nil {
- return fmt.Errorf("failed to open git repository %s: %w", s.target, err)
+ return fmt.Errorf("failed to open git repository %s: %w", s.displayTarget, err)
}
s.repo = repo
return nil
}
-func (s *GitSource) cloneRemote() error {
+func (s *GitSource) cloneRemote(ctx context.Context, cloneTarget string) error {
tmpDir, err := os.MkdirTemp("", "leakwatch-clone-*")
if err != nil {
return fmt.Errorf("failed to create temp directory: %w", err)
}
+ // Own the directory immediately so every failure path can clean it through
+ // Close and a failed cleanup remains retryable by the caller.
+ s.tmpDir = tmpDir
cloneOpts := &git.CloneOptions{
- URL: s.target,
+ URL: cloneTarget,
Progress: nil,
}
@@ -284,19 +394,28 @@ func (s *GitSource) cloneRemote() error {
slog.Info("cloning remote repository", "url", s.displayTarget, "tmpDir", tmpDir)
- repo, err := git.PlainClone(tmpDir, false, cloneOpts)
+ repo, err := git.PlainCloneContext(ctx, tmpDir, false, cloneOpts)
if err != nil {
- _ = os.RemoveAll(tmpDir)
// go-git's HTTP transport and endpoint layers re-embed the raw
// user:password@host userinfo from the clone URL into their error
// strings, so the raw err must never be wrapped with %w. Sanitize it
// first so no credential can reach errors.Unwrap chains, stderr, or CI
// logs.
- return fmt.Errorf("failed to clone git repository %s: %w",
- s.displayTarget, sanitizeCloneError(err, s.target, s.displayTarget))
+ var cloneErr error
+ if ctxErr := ctx.Err(); ctxErr != nil {
+ // Preserve cancellation identity so the CLI maps an interrupted clone
+ // to exit 3. The context error itself contains no clone credential.
+ cloneErr = ctxErr
+ } else {
+ cloneErr = fmt.Errorf("failed to clone git repository %s: %w",
+ s.displayTarget, sanitizeCloneError(err, s.target, s.displayTarget))
+ }
+ if cleanupErr := s.Close(); cleanupErr != nil {
+ return errors.Join(cloneErr, cleanupErr)
+ }
+ return cloneErr
}
s.repo = repo
- s.tmpDir = tmpDir
return nil
}
@@ -416,7 +535,7 @@ func (s *GitSource) chunksFullHistory(ctx context.Context, ch chan<- source.Chun
// (exclusive) and the branch tip / HEAD (inclusive), diffing each commit against
// its parent.
func (s *GitSource) chunksSinceCommit(ctx context.Context, ch chan<- source.Chunk) {
- sinceCommitObj, err := s.resolveCommitHash(s.sinceCommit)
+ sinceCommitObj, err := s.resolveCommitHash(ctx, s.sinceCommit)
if err != nil {
slog.Error("since-commit resolution failed", "commit", s.sinceCommit, "error", err)
s.captureErr(fmt.Errorf("since-commit resolution failed: %w", err))
diff --git a/internal/source/git/git_test.go b/internal/source/git/git_test.go
index 01c190e..9430f3e 100644
--- a/internal/source/git/git_test.go
+++ b/internal/source/git/git_test.go
@@ -2,7 +2,10 @@ package git
import (
"context"
+ "errors"
"fmt"
+ "net/http"
+ "net/http/httptest"
"os"
"path/filepath"
"strings"
@@ -16,6 +19,19 @@ import (
"github.com/stretchr/testify/require"
)
+type cancelAfterErrChecks struct {
+ context.Context
+ remaining int
+}
+
+func (c *cancelAfterErrChecks) Err() error {
+ c.remaining--
+ if c.remaining <= 0 {
+ return context.Canceled
+ }
+ return nil
+}
+
// initTestRepo creates a temporary git repository for testing.
func initTestRepo(t *testing.T, files map[string]string) (string, *gogit.Repository) {
t.Helper()
@@ -81,18 +97,18 @@ func TestGitSource_Validate_ValidRepo(t *testing.T) {
dir, _ := initTestRepo(t, map[string]string{"README.md": "hello"})
s := New(dir)
- assert.NoError(t, s.Validate())
+ assert.NoError(t, s.Validate(context.Background()))
}
func TestGitSource_Validate_NonExistentRepo(t *testing.T) {
s := New("/nonexistent/repo")
- assert.Error(t, s.Validate())
+ assert.Error(t, s.Validate(context.Background()))
}
func TestGitSource_Validate_NotAGitRepo(t *testing.T) {
dir := t.TempDir()
s := New(dir)
- assert.Error(t, s.Validate())
+ assert.Error(t, s.Validate(context.Background()))
}
func TestGitSource_Chunks_ReadsCommitHistory(t *testing.T) {
@@ -102,7 +118,7 @@ func TestGitSource_Chunks_ReadsCommitHistory(t *testing.T) {
})
s := New(dir)
- require.NoError(t, s.Validate())
+ require.NoError(t, s.Validate(context.Background()))
ctx := context.Background()
var chunks []string
@@ -121,7 +137,7 @@ func TestGitSource_Chunks_IncludesCommitMetadata(t *testing.T) {
})
s := New(dir)
- require.NoError(t, s.Validate())
+ require.NoError(t, s.Validate(context.Background()))
ctx := context.Background()
for chunk := range s.Chunks(ctx) {
@@ -149,7 +165,7 @@ func TestGitSource_Chunks_MultipleCommits(t *testing.T) {
}, "third commit")
s := New(dir)
- require.NoError(t, s.Validate())
+ require.NoError(t, s.Validate(context.Background()))
ctx := context.Background()
seen := make(map[string]bool)
@@ -183,7 +199,7 @@ func TestGitSource_Chunks_SinceCommit(t *testing.T) {
}, "new commit 2")
s := New(dir, WithSinceCommit(sinceHash))
- require.NoError(t, s.Validate())
+ require.NoError(t, s.Validate(context.Background()))
ctx := context.Background()
var files []string
@@ -202,7 +218,7 @@ func TestGitSource_Validate_SinceCommitNotFound_ReturnsError(t *testing.T) {
// A well-formed but non-existent commit hash.
s := New(dir, WithSinceCommit("0123456789abcdef0123456789abcdef01234567"))
- err := s.Validate()
+ err := s.Validate(context.Background())
require.Error(t, err)
assert.Contains(t, err.Error(), "not found")
}
@@ -232,7 +248,7 @@ func TestGitSource_Validate_SinceCommitNotAncestor_ReturnsError(t *testing.T) {
// sideHash lives only on sidebranch, so it is not an ancestor of HEAD (main).
s := New(dir, WithSinceCommit(sideHash))
- err = s.Validate()
+ err = s.Validate(context.Background())
require.Error(t, err)
assert.Contains(t, err.Error(), "not an ancestor")
}
@@ -247,7 +263,7 @@ func TestGitSource_Validate_SinceCommitIsAncestor_ReturnsNil(t *testing.T) {
addCommit(t, dir, repo, map[string]string{"b.txt": "more"}, "second commit")
s := New(dir, WithSinceCommit(baseHash))
- assert.NoError(t, s.Validate())
+ assert.NoError(t, s.Validate(context.Background()))
}
func TestGitSource_Validate_SinceCommitEqualsHead_ReturnsNil(t *testing.T) {
@@ -258,7 +274,7 @@ func TestGitSource_Validate_SinceCommitEqualsHead_ReturnsNil(t *testing.T) {
headHash := headRef.Hash().String()
s := New(dir, WithSinceCommit(headHash))
- assert.NoError(t, s.Validate())
+ assert.NoError(t, s.Validate(context.Background()))
}
func TestGitSource_Chunks_WithExcludePaths_SkipsMatching(t *testing.T) {
@@ -269,7 +285,7 @@ func TestGitSource_Chunks_WithExcludePaths_SkipsMatching(t *testing.T) {
})
s := New(dir, WithExcludePaths([]string{"vendor/**", "node_modules/**"}))
- require.NoError(t, s.Validate())
+ require.NoError(t, s.Validate(context.Background()))
ctx := context.Background()
var files []string
@@ -321,7 +337,7 @@ func TestGitSource_Chunks_WithSince(t *testing.T) {
require.NoError(t, err)
s := New(dir, WithSince(cutoff))
- require.NoError(t, s.Validate())
+ require.NoError(t, s.Validate(context.Background()))
ctx := context.Background()
var files []string
@@ -336,7 +352,7 @@ func TestGitSource_Err_SuccessfulChunks_ReturnsNil(t *testing.T) {
dir, _ := initTestRepo(t, map[string]string{"README.md": "hello"})
s := New(dir)
- require.NoError(t, s.Validate())
+ require.NoError(t, s.Validate(context.Background()))
for range s.Chunks(context.Background()) {
}
@@ -346,7 +362,7 @@ func TestGitSource_Err_SuccessfulChunks_ReturnsNil(t *testing.T) {
func TestGitSource_Err_HistoryWalkFailure_ReturnsError(t *testing.T) {
// A freshly-initialized repository has no commits, so HEAD does not resolve.
- // Validate() succeeds (the repo opens), but the history walk in Chunks fails
+ // Validate(ctx) succeeds (the repo opens), but the history walk in Chunks fails
// to resolve a start commit — a terminal error that must be reported via Err
// rather than silently yielding a clean, empty scan.
dir := t.TempDir()
@@ -354,7 +370,7 @@ func TestGitSource_Err_HistoryWalkFailure_ReturnsError(t *testing.T) {
require.NoError(t, err)
s := New(dir)
- require.NoError(t, s.Validate())
+ require.NoError(t, s.Validate(context.Background()))
var count int
for range s.Chunks(context.Background()) {
@@ -397,7 +413,7 @@ func TestGitSource_Chunks_ContextCancellation(t *testing.T) {
})
s := New(dir)
- require.NoError(t, s.Validate())
+ require.NoError(t, s.Validate(context.Background()))
ctx, cancel := context.WithCancel(context.Background())
cancel() // Cancel immediately.
@@ -422,7 +438,7 @@ func TestGitSource_Chunks_SkipsLargeFiles(t *testing.T) {
})
s := New(dir, WithMaxFileSize(512))
- require.NoError(t, s.Validate())
+ require.NoError(t, s.Validate(context.Background()))
ctx := context.Background()
var files []string
@@ -440,7 +456,9 @@ func TestGitSource_IsRemote(t *testing.T) {
expected bool
}{
{"https://github.com/org/repo.git", true},
+ {"HTTPS://github.com/org/repo.git", true},
{"http://github.com/org/repo.git", true},
+ {"SSH://git@github.com/org/repo.git", true},
{"git@github.com:org/repo.git", true},
{"ssh://git@github.com/org/repo.git", true},
{"/local/path/to/repo", false},
@@ -455,6 +473,53 @@ func TestGitSource_IsRemote(t *testing.T) {
}
}
+func TestClassifyGitTarget_RejectsUnsupportedScheme(t *testing.T) {
+ remote, normalized, err := classifyGitTarget("ftp://user:wave9-canary@example.test/repo.git")
+ require.Error(t, err)
+ assert.False(t, remote)
+ assert.Empty(t, normalized)
+ assert.NotContains(t, err.Error(), "wave9-canary")
+}
+
+func TestClassifyGitTarget_LocalPathContainingColonRemainsLocal(t *testing.T) {
+ remote, normalized, err := classifyGitTarget("repo:with-colon")
+ require.NoError(t, err)
+ assert.False(t, remote)
+ assert.Equal(t, "repo:with-colon", normalized)
+}
+
+func TestGitSource_Validate_NonexistentRelativeTargetDoesNotOpenParentRepo(t *testing.T) {
+ dir, _ := initTestRepo(t, map[string]string{"README.md": "parent repo"})
+ t.Chdir(dir)
+
+ s := New("definitely-not-existing-wave9")
+ err := s.Validate(context.Background())
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "failed to inspect git target")
+ assert.Nil(t, s.repo, "a typo must never silently select and scan the parent repository")
+}
+
+func TestGitSource_Validate_LocalFileTargetIsRejected(t *testing.T) {
+ dir, _ := initTestRepo(t, map[string]string{"README.md": "parent repo"})
+ s := New(filepath.Join(dir, "README.md"))
+
+ err := s.Validate(context.Background())
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "not a directory")
+ assert.Nil(t, s.repo)
+}
+
+func TestGitSource_Validate_UppercaseHTTPSSchemeNeverLeaksUserinfo(t *testing.T) {
+ const canary = "wave9-canary-not-a-secret"
+ s := New("HTTPS://user:" + canary + "@127.0.0.1:1/repo.git")
+
+ err := s.Validate(context.Background())
+ require.Error(t, err)
+ assert.NotContains(t, err.Error(), canary)
+ assert.NotContains(t, err.Error(), "user:")
+ assert.Empty(t, s.tmpDir, "failed remote clone must clean its temporary directory")
+}
+
func TestGitSource_Close_RemovesTmpDir(t *testing.T) {
// Simulate a cloned repo by setting tmpDir manually.
tmpDir := t.TempDir()
@@ -488,6 +553,99 @@ func TestGitSource_Close_NoTmpDir(t *testing.T) {
assert.NoError(t, err)
}
+func TestGitSource_Close_FailedCleanupRemainsRetryable(t *testing.T) {
+ tmpDir := t.TempDir()
+ cleanupErr := errors.New("synthetic cleanup failure")
+ s := &GitSource{
+ tmpDir: tmpDir,
+ removeAll: func(string) error { return cleanupErr },
+ }
+
+ err := s.Close()
+ require.ErrorIs(t, err, cleanupErr)
+ assert.Equal(t, tmpDir, s.tmpDir, "failed cleanup must retain ownership for a retry")
+
+ s.removeAll = os.RemoveAll
+ require.NoError(t, s.Close())
+ assert.Empty(t, s.tmpDir)
+ _, err = os.Stat(tmpDir)
+ assert.True(t, os.IsNotExist(err))
+}
+
+func TestGitSource_ResolveCommitHash_CancellationStopsEnumeration(t *testing.T) {
+ dir, repo := initTestRepo(t, map[string]string{"base.txt": "base"})
+ head, err := repo.Head()
+ require.NoError(t, err)
+ basePrefix := head.Hash().String()[:8]
+ for i := range 20 {
+ addCommit(t, dir, repo, map[string]string{
+ fmt.Sprintf("file-%d.txt", i): fmt.Sprintf("content-%d", i),
+ }, fmt.Sprintf("commit-%d", i))
+ }
+
+ s := New(dir)
+ require.NoError(t, s.openLocal())
+ ctx := &cancelAfterErrChecks{Context: context.Background(), remaining: 2}
+
+ _, err = s.resolveCommitHash(ctx, basePrefix)
+ require.ErrorIs(t, err, context.Canceled)
+}
+
+func TestIsAncestorContext_CancellationStopsHistoryWalk(t *testing.T) {
+ dir, repo := initTestRepo(t, map[string]string{"base.txt": "base"})
+ baseRef, err := repo.Head()
+ require.NoError(t, err)
+ base, err := repo.CommitObject(baseRef.Hash())
+ require.NoError(t, err)
+
+ for i := range 20 {
+ addCommit(t, dir, repo, map[string]string{
+ fmt.Sprintf("history-%d.txt", i): fmt.Sprintf("content-%d", i),
+ }, fmt.Sprintf("history-%d", i))
+ }
+ headRef, err := repo.Head()
+ require.NoError(t, err)
+ head, err := repo.CommitObject(headRef.Hash())
+ require.NoError(t, err)
+
+ ctx := &cancelAfterErrChecks{Context: context.Background(), remaining: 2}
+ _, err = isAncestorContext(ctx, base, head)
+ require.ErrorIs(t, err, context.Canceled)
+}
+
+func TestGitSource_Validate_RemoteCloneCancellationIsPromptAndCleansUp(t *testing.T) {
+ requestStarted := make(chan struct{})
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ select {
+ case <-requestStarted:
+ default:
+ close(requestStarted)
+ }
+ <-r.Context().Done()
+ }))
+ defer server.Close()
+
+ s := New(server.URL + "/repo.git")
+ ctx, cancel := context.WithCancel(context.Background())
+ done := make(chan error, 1)
+ go func() { done <- s.Validate(ctx) }()
+
+ select {
+ case <-requestStarted:
+ case <-time.After(2 * time.Second):
+ t.Fatal("clone did not start an HTTP request")
+ }
+ cancel()
+
+ select {
+ case err := <-done:
+ require.ErrorIs(t, err, context.Canceled)
+ assert.Empty(t, s.tmpDir, "cancelled clone must clean up its temporary directory")
+ case <-time.After(2 * time.Second):
+ t.Fatal("remote validation did not stop promptly after cancellation")
+ }
+}
+
func TestSafeDisplayURL_StripsCredentials(t *testing.T) {
// fakeToken is a non-secret placeholder used only to prove redaction.
const fakeToken = "ghp_FAKEtoken1234567890"
@@ -609,7 +767,7 @@ func TestGitSource_Chunks_RepositoryMetadataHasNoCredential(t *testing.T) {
// credential-stripped remote form that should appear in metadata.
s := New(dir)
s.displayTarget = SafeDisplayURL("https://user:" + fakeToken + "@github.com/org/repo.git")
- require.NoError(t, s.Validate())
+ require.NoError(t, s.Validate(context.Background()))
ctx := context.Background()
var sawChunk bool
@@ -637,7 +795,7 @@ func TestGitSource_Chunks_FullHistory_AttributesIntroducingCommit(t *testing.T)
addCommit(t, dir, repo, map[string]string{"b.txt": "bbb"}, "third commit")
s := New(dir)
- require.NoError(t, s.Validate())
+ require.NoError(t, s.Validate(context.Background()))
ctx := context.Background()
var configCommit string
@@ -662,7 +820,7 @@ func TestGitSource_Chunks_FullHistory_ReportsBothPathsForIdenticalContent(t *tes
}, "add duplicate content under a different path")
s := New(dir)
- require.NoError(t, s.Validate())
+ require.NoError(t, s.Validate(context.Background()))
ctx := context.Background()
var files []string
@@ -699,7 +857,7 @@ func TestGitSource_Chunks_Branch_ScansRequestedBranchLocally(t *testing.T) {
addCommit(t, dir, repo, map[string]string{"main-only.txt": "main content"}, "main commit")
s := New(dir, WithBranch("feature"))
- require.NoError(t, s.Validate())
+ require.NoError(t, s.Validate(context.Background()))
ctx := context.Background()
var files []string
@@ -718,7 +876,7 @@ func TestGitSource_Validate_Branch_NotFound_ReturnsError(t *testing.T) {
dir, _ := initTestRepo(t, map[string]string{"a.txt": "content"})
s := New(dir, WithBranch("does-not-exist"))
- err := s.Validate()
+ err := s.Validate(context.Background())
require.Error(t, err)
assert.Contains(t, err.Error(), "does-not-exist")
}
@@ -733,7 +891,7 @@ func TestGitSource_Validate_SinceCommit_AbbreviatedHash_Resolves(t *testing.T) {
addCommit(t, dir, repo, map[string]string{"b.txt": "more"}, "second commit")
s := New(dir, WithSinceCommit(shortHash))
- require.NoError(t, s.Validate(), "an abbreviated since-commit hash should resolve, not fail as 'not found'")
+ require.NoError(t, s.Validate(context.Background()), "an abbreviated since-commit hash should resolve, not fail as 'not found'")
ctx := context.Background()
var files []string
@@ -749,7 +907,7 @@ func TestGitSource_Validate_SinceCommit_TooShort_ReturnsClearError(t *testing.T)
dir, _ := initTestRepo(t, map[string]string{"a.txt": "content"})
s := New(dir, WithSinceCommit("ab"))
- err := s.Validate()
+ err := s.Validate(context.Background())
require.Error(t, err)
assert.Contains(t, err.Error(), "since-commit")
assert.Contains(t, err.Error(), "too short")
@@ -759,7 +917,7 @@ func TestGitSource_Validate_SinceCommit_NonHex_ReturnsClearError(t *testing.T) {
dir, _ := initTestRepo(t, map[string]string{"a.txt": "content"})
s := New(dir, WithSinceCommit("zzzzzzz"))
- err := s.Validate()
+ err := s.Validate(context.Background())
require.Error(t, err)
assert.Contains(t, err.Error(), "not a valid commit hash")
}
@@ -791,7 +949,7 @@ func TestGitSource_Chunks_SinceCommitWithSince_HonorsSince(t *testing.T) {
// since-commit excludes base; --since additionally drops the 2024 commit.
cutoff := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)
s := New(dir, WithSinceCommit(base.String()), WithSince(cutoff))
- require.NoError(t, s.Validate())
+ require.NoError(t, s.Validate(context.Background()))
ctx := context.Background()
var files []string
@@ -817,7 +975,7 @@ func TestGitSource_ResolveBranch_DetachedHead_ReportsShortCommit(t *testing.T) {
require.NoError(t, wt.Checkout(&gogit.CheckoutOptions{Hash: baseHash}))
s := New(dir)
- require.NoError(t, s.Validate())
+ require.NoError(t, s.Validate(context.Background()))
branch := s.resolveBranch()
assert.True(t, strings.HasPrefix(branch, "detached@"),
diff --git a/internal/source/s3/s3.go b/internal/source/s3/s3.go
index 3db14c4..6a119fc 100644
--- a/internal/source/s3/s3.go
+++ b/internal/source/s3/s3.go
@@ -23,11 +23,8 @@ import (
// defaultMaxFileSize is the maximum object size to scan (10 MB).
const defaultMaxFileSize int64 = 10 * 1024 * 1024
-// validateTimeout bounds the network calls made by Validate. The
-// source.Source interface's Validate() method takes no context.Context
-// parameter, so the caller's own cancellation cannot be threaded through
-// here; a bounded timeout at least prevents an unreachable/misconfigured
-// bucket from hanging Validate indefinitely.
+// validateTimeout bounds validation when the caller provides no earlier
+// deadline. An earlier caller deadline or cancellation always wins.
const validateTimeout = 30 * time.Second
// s3Client defines the subset of the S3 API used by S3Source.
@@ -100,12 +97,12 @@ func (s *S3Source) captureErr(err error) {
// Validate checks that the S3 bucket is accessible.
// It initializes the AWS client if not already set and performs a HeadBucket call.
-func (s *S3Source) Validate() error {
+func (s *S3Source) Validate(ctx context.Context) error {
if s.bucket == "" {
return fmt.Errorf("s3 bucket name is required")
}
- ctx, cancel := context.WithTimeout(context.Background(), validateTimeout)
+ ctx, cancel := context.WithTimeout(ctx, validateTimeout)
defer cancel()
if err := s.ensureClient(ctx); err != nil {
diff --git a/internal/source/s3/s3_test.go b/internal/source/s3/s3_test.go
index 952776e..1833a8f 100644
--- a/internal/source/s3/s3_test.go
+++ b/internal/source/s3/s3_test.go
@@ -7,6 +7,7 @@ import (
"strconv"
"strings"
"testing"
+ "time"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
@@ -18,10 +19,13 @@ import (
// mockS3Client is a minimal mock for the s3Client interface.
type mockS3Client struct {
- objects []types.Object
- data map[string]string
- headErr error
- listErr error
+ objects []types.Object
+ data map[string]string
+ headErr error
+ listErr error
+ headBlock bool
+ headDeadline time.Time
+ headHasDeadline bool
}
func (m *mockS3Client) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input, _ ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) {
@@ -51,7 +55,12 @@ func (m *mockS3Client) GetObject(_ context.Context, input *s3.GetObjectInput, _
}, nil
}
-func (m *mockS3Client) HeadBucket(_ context.Context, _ *s3.HeadBucketInput, _ ...func(*s3.Options)) (*s3.HeadBucketOutput, error) {
+func (m *mockS3Client) HeadBucket(ctx context.Context, _ *s3.HeadBucketInput, _ ...func(*s3.Options)) (*s3.HeadBucketOutput, error) {
+ m.headDeadline, m.headHasDeadline = ctx.Deadline()
+ if m.headBlock {
+ <-ctx.Done()
+ return nil, ctx.Err()
+ }
return &s3.HeadBucketOutput{}, m.headErr
}
@@ -104,7 +113,7 @@ func TestWithMaxFileSize_ValidValue_Applied(t *testing.T) {
func TestS3Source_Validate_EmptyBucket_ReturnsError(t *testing.T) {
s := New("")
- err := s.Validate()
+ err := s.Validate(context.Background())
require.Error(t, err)
assert.Contains(t, err.Error(), "bucket name is required")
}
@@ -112,7 +121,32 @@ func TestS3Source_Validate_EmptyBucket_ReturnsError(t *testing.T) {
func TestS3Source_Validate_AccessibleBucket_ReturnsNil(t *testing.T) {
s := New("my-bucket")
s.client = &mockS3Client{}
- assert.NoError(t, s.Validate())
+ assert.NoError(t, s.Validate(context.Background()))
+}
+
+func TestS3Source_Validate_AppliesDefaultDeadline(t *testing.T) {
+ mock := &mockS3Client{}
+ s := New("my-bucket")
+ s.client = mock
+ started := time.Now()
+
+ require.NoError(t, s.Validate(context.Background()))
+ require.True(t, mock.headHasDeadline)
+ assert.WithinDuration(t, started.Add(validateTimeout), mock.headDeadline, time.Second)
+}
+
+func TestS3Source_Validate_CallerDeadlineCancelsRequest(t *testing.T) {
+ mock := &mockS3Client{headBlock: true}
+ s := New("my-bucket")
+ s.client = mock
+ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
+ defer cancel()
+ started := time.Now()
+
+ err := s.Validate(ctx)
+ require.ErrorIs(t, err, context.DeadlineExceeded)
+ assert.Less(t, time.Since(started), time.Second)
+ assert.WithinDuration(t, started.Add(20*time.Millisecond), mock.headDeadline, 50*time.Millisecond)
}
func TestS3Source_Chunks_SendsTextObjects(t *testing.T) {
diff --git a/internal/source/slack/client.go b/internal/source/slack/client.go
index e045392..3f231a8 100644
--- a/internal/source/slack/client.go
+++ b/internal/source/slack/client.go
@@ -7,6 +7,7 @@ package slack
import (
"context"
+ "io"
"github.com/slack-go/slack"
)
@@ -16,5 +17,7 @@ import (
type slackClient interface {
GetConversationsContext(ctx context.Context, params *slack.GetConversationsParameters) ([]slack.Channel, string, error)
GetConversationHistoryContext(ctx context.Context, params *slack.GetConversationHistoryParameters) (*slack.GetConversationHistoryResponse, error)
+ GetFileInfoContext(ctx context.Context, fileID string, count, page int) (*slack.File, []slack.Comment, *slack.Paging, error)
+ GetFileContext(ctx context.Context, downloadURL string, writer io.Writer) error
AuthTestContext(ctx context.Context) (*slack.AuthTestResponse, error)
}
diff --git a/internal/source/slack/options.go b/internal/source/slack/options.go
index 2246c32..5c6f993 100644
--- a/internal/source/slack/options.go
+++ b/internal/source/slack/options.go
@@ -38,22 +38,32 @@ func WithIncludeDMs(include bool) Option {
}
}
-// WithIncludeFiles requests scanning of file content.
-//
-// NOTE: Slack file scanning is not yet implemented. This option is currently a
-// no-op for actual scanning behavior (only message text is scanned); enabling
-// it causes a warning to be logged. See ROADMAP and the planned-feature note in slack.go.
+// WithIncludeFiles enables bounded scanning of text-like files attached to
+// messages. The token needs Slack's files:read scope.
func WithIncludeFiles(include bool) Option {
return func(s *SlackSource) {
s.includeFiles = include
}
}
-// WithRateLimit sets the Slack API rate limit in requests per second.
+// WithMaxFileSize sets the maximum attachment size buffered for scanning.
+func WithMaxFileSize(size int64) Option {
+ return func(s *SlackSource) {
+ if size > 0 {
+ s.maxFileSize = size
+ }
+ }
+}
+
+// WithRateLimit overrides each operation-scoped Slack limiter with the same
+// requests-per-second ceiling. Defaults remain operation-specific.
func WithRateLimit(rps float64) Option {
return func(s *SlackSource) {
if rps > 0 {
- s.rateLimit = rps
+ s.historyRateLimit = rps
+ s.listRateLimit = rps
+ s.fileInfoRateLimit = rps
+ s.fileDownloadRateLimit = rps
}
}
}
diff --git a/internal/source/slack/slack.go b/internal/source/slack/slack.go
index 4c9b190..385d257 100644
--- a/internal/source/slack/slack.go
+++ b/internal/source/slack/slack.go
@@ -2,13 +2,21 @@
package slack
import (
+ "bytes"
"context"
"errors"
"fmt"
+ "io"
"log/slog"
+ "mime"
+ "net/http"
+ "net/url"
+ "path"
"strconv"
"strings"
"time"
+ "unicode"
+ "unicode/utf8"
"github.com/slack-go/slack"
"golang.org/x/time/rate"
@@ -18,41 +26,47 @@ import (
)
const (
- // defaultRateLimit is a conservative default request rate for the Slack
- // Web API's conversations.list/conversations.history methods. Slack's
- // actual enforced tiers for these endpoints are commonly much stricter
- // than a naive guess, so this default favors reliability over speed;
- // callers with a higher-tier app/token can raise it via --rate-limit /
- // WithRateLimit. Combined with the 429 retry handling below, this keeps
- // scans from silently truncating on the first rate-limit response.
- defaultRateLimit = 1.0
- defaultBufferSize = 100
+ // DefaultRateLimit matches Slack's lowest published conversations.history
+ // contract for newly distributed non-Marketplace apps: one request/minute.
+ // Marketplace and internal customer-built apps can raise it explicitly.
+ DefaultRateLimit = 1.0 / 60.0
+ defaultListRateLimit = 20.0 / 60.0
+ defaultFileRateLimit = 100.0 / 60.0
+ defaultBufferSize = 100
+ defaultMaxFileSize = 10 * 1024 * 1024
+ defaultHistoryLimit = 15
+ validationTimeout = 10 * time.Second
+ slackRequestTimeout = 30 * time.Second
// maxRateLimitRetries bounds how many consecutive HTTP 429 responses a
// single page fetch will absorb before giving up, preventing an
// unbounded retry loop against a workspace that never recovers.
maxRateLimitRetries = 5
+ maxSlackPages = 10_000
// defaultRetryAfter is used when a *slack.RateLimitedError reports a
// non-positive RetryAfter duration, as a safe fallback backoff.
- defaultRetryAfter = time.Second
+ defaultRetryAfter = time.Second
+ maximumRetryAfterWait = 2 * time.Minute
)
// SlackSource scans messages in a Slack workspace for leaked secrets.
type SlackSource struct {
- token string
- channels []string
- excludeChannels []string
- since time.Time
- includeDMs bool
- // includeFiles is accepted for forward-compatibility but is currently a
- // no-op: Slack file scanning is not yet implemented (only message text is
- // scanned). See the planned-feature note in processChannel below.
- includeFiles bool
- rateLimit float64
- bufferSize int
- client slackClient
- newClient func(token string) slackClient
+ token string
+ channels []string
+ excludeChannels []string
+ since time.Time
+ includeDMs bool
+ includeFiles bool
+ maxFileSize int64
+ historyRateLimit float64
+ listRateLimit float64
+ fileInfoRateLimit float64
+ fileDownloadRateLimit float64
+ bufferSize int
+ client slackClient
+ newClient func(token string) slackClient
+ skippedAttachments int
// err records the first terminal failure that aborted scanning (channel
// listing / auth / pagination). It is written only by the Chunks goroutine,
@@ -65,21 +79,34 @@ type SlackSource struct {
// defaultNewClient creates a real Slack API client.
func defaultNewClient(token string) slackClient {
- return slack.New(token)
+ return slack.New(token, slack.OptionHTTPClient(newSlackHTTPClient()))
+}
+
+func newSlackHTTPClient() *http.Client {
+ return &http.Client{
+ CheckRedirect: rejectSlackRedirect,
+ Timeout: slackRequestTimeout,
+ }
+}
+
+func rejectSlackRedirect(_ *http.Request, _ []*http.Request) error {
+ return http.ErrUseLastResponse
}
// New creates a new SlackSource for the given workspace token.
// Use functional options to configure channel filtering, rate limits, etc.
func New(token string, opts ...Option) *SlackSource {
s := &SlackSource{
- token: token,
- includeDMs: false,
- // includeFiles defaults to false: file scanning is not implemented yet,
- // so advertising it on by default would be misleading.
- includeFiles: false,
- rateLimit: defaultRateLimit,
- bufferSize: defaultBufferSize,
- newClient: defaultNewClient,
+ token: token,
+ includeDMs: false,
+ includeFiles: false,
+ maxFileSize: defaultMaxFileSize,
+ historyRateLimit: DefaultRateLimit,
+ listRateLimit: defaultListRateLimit,
+ fileInfoRateLimit: defaultFileRateLimit,
+ fileDownloadRateLimit: defaultFileRateLimit,
+ bufferSize: defaultBufferSize,
+ newClient: defaultNewClient,
}
for _, opt := range opts {
opt(s)
@@ -104,10 +131,10 @@ func (s *SlackSource) Err() error {
// is called only from the single Chunks goroutine, before close(ch), so a plain
// field write is safe (the channel close/drain publishes it to Err's reader).
// As defense-in-depth against a client library that echoes the token, any
-// occurrence of the workspace token is stripped from the stored message (and,
-// when present, the error is flattened so the token cannot survive in the unwrap
-// chain either). Context cancellation is never recorded because it is reported
-// through the context, not Err.
+// occurrence of the workspace token is stripped from the stored message. Every
+// provider error is flattened so a wrapper cannot hide credentials in Unwrap.
+// Context cancellation is never recorded because it is reported through the
+// context, not Err.
func (s *SlackSource) captureErr(err error) {
if err == nil || s.err != nil {
return
@@ -115,24 +142,44 @@ func (s *SlackSource) captureErr(err error) {
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return
}
- if s.token != "" && strings.Contains(err.Error(), s.token) {
- s.err = errors.New(strings.ReplaceAll(err.Error(), s.token, "***"))
- return
+ s.err = s.safeSlackError(err)
+}
+
+func (s *SlackSource) safeSlackError(err error) error {
+ if err == nil {
+ return nil
+ }
+ message := err.Error()
+ if s.token != "" {
+ message = strings.ReplaceAll(message, s.token, "***")
}
- s.err = err
+ return errors.New(message)
}
-// Validate checks that the Slack token is valid by calling AuthTest.
-func (s *SlackSource) Validate() error {
+// Validate checks that the Slack token is valid by calling AuthTest. The
+// operation is bounded even when the caller supplies no deadline, while an
+// earlier caller deadline or cancellation always wins.
+func (s *SlackSource) Validate(ctx context.Context) error {
if s.token == "" {
return fmt.Errorf("slack token is required")
}
+ if err := ctx.Err(); err != nil {
+ return err
+ }
s.ensureClient()
- _, err := s.client.AuthTestContext(context.Background())
+ validateCtx, cancel := context.WithTimeout(ctx, validationTimeout)
+ defer cancel()
+ _, err := s.client.AuthTestContext(validateCtx)
if err != nil {
- return fmt.Errorf("slack auth test failed: %w", err)
+ if errors.Is(err, context.Canceled) {
+ return fmt.Errorf("slack auth test failed: %w", context.Canceled)
+ }
+ if errors.Is(err, context.DeadlineExceeded) {
+ return fmt.Errorf("slack auth test failed: %w", context.DeadlineExceeded)
+ }
+ return fmt.Errorf("slack auth test failed: %w", s.safeSlackError(err))
}
return nil
@@ -141,24 +188,32 @@ func (s *SlackSource) Validate() error {
// Chunks lists channels in the workspace and sends message contents over a channel.
// The channel is closed when all messages have been processed or the context is cancelled.
func (s *SlackSource) Chunks(ctx context.Context) <-chan source.Chunk {
- ch := make(chan source.Chunk, s.bufferSize)
+ bufferSize := s.bufferSize
+ if s.includeFiles {
+ bufferSize = 0
+ }
+ ch := make(chan source.Chunk, bufferSize)
go func() {
defer close(ch)
+ s.skippedAttachments = 0
+ defer func() {
+ if s.skippedAttachments > 0 {
+ slog.Warn("Slack scan completed with skipped attachments",
+ slog.Int("skipped_attachments", s.skippedAttachments))
+ }
+ }()
s.ensureClient()
- // File scanning is advertised via WithIncludeFiles but is not yet
- // implemented. Warn loudly instead of silently ignoring the request so
- // the behavior is honest. See the planned-feature note in processChannel.
- if s.includeFiles {
- slog.Warn("slack file scanning requested but not yet implemented; scanning message text only")
- }
-
- limiter := rate.NewLimiter(rate.Limit(s.rateLimit), 1)
+ listLimiter := rate.NewLimiter(rate.Limit(s.listRateLimit), 1)
+ historyLimiter := rate.NewLimiter(rate.Limit(s.historyRateLimit), 1)
+ fileInfoLimiter := rate.NewLimiter(rate.Limit(s.fileInfoRateLimit), 1)
+ fileDownloadLimiter := rate.NewLimiter(rate.Limit(s.fileDownloadRateLimit), 1)
+ seenFiles := make(map[string]struct{})
- channels, err := s.listChannels(ctx, limiter)
+ channels, err := s.listChannels(ctx, listLimiter)
if err != nil {
- slog.Error("slack channel listing failed", "error", err)
+ slog.Error("slack channel listing failed", "error", s.safeSlackError(err))
s.captureErr(fmt.Errorf("slack channel listing failed: %w", err))
return
}
@@ -172,7 +227,10 @@ func (s *SlackSource) Chunks(ctx context.Context) <-chan source.Chunk {
default:
}
- s.processChannel(ctx, ch, limiter, channel)
+ s.processChannel(ctx, ch, historyLimiter, fileInfoLimiter, fileDownloadLimiter, channel, seenFiles)
+ if s.err != nil {
+ return
+ }
}
}()
return ch
@@ -192,7 +250,10 @@ func (s *SlackSource) listChannels(ctx context.Context, limiter *rate.Limiter) (
cursor := ""
retries := 0
- for {
+ for page := 1; ; page++ {
+ if page > maxSlackPages {
+ return allChannels, fmt.Errorf("slack list conversations exceeded %d pages", maxSlackPages)
+ }
select {
case <-ctx.Done():
return allChannels, ctx.Err()
@@ -234,9 +295,13 @@ func (s *SlackSource) listChannels(ctx context.Context, limiter *rate.Limiter) (
allChannels = append(allChannels, channels...)
+ nextCursor = strings.TrimSpace(nextCursor)
if nextCursor == "" {
break
}
+ if nextCursor == cursor {
+ return allChannels, fmt.Errorf("slack list conversations returned a stalled cursor")
+ }
cursor = nextCursor
}
@@ -260,6 +325,9 @@ func waitRetryAfter(ctx context.Context, d time.Duration) error {
if d <= 0 {
d = defaultRetryAfter
}
+ if d > maximumRetryAfterWait {
+ return fmt.Errorf("slack Retry-After exceeds the bounded wait limit")
+ }
timer := time.NewTimer(d)
defer timer.Stop()
select {
@@ -308,105 +376,371 @@ func (s *SlackSource) filterChannels(channels []slack.Channel) []slack.Channel {
}
// processChannel reads message history for a single channel and emits chunks.
-func (s *SlackSource) processChannel(ctx context.Context, ch chan<- source.Chunk, limiter *rate.Limiter, channel slack.Channel) {
+func (s *SlackSource) processChannel(
+ ctx context.Context,
+ ch chan<- source.Chunk,
+ historyLimiter *rate.Limiter,
+ fileInfoLimiter *rate.Limiter,
+ fileDownloadLimiter *rate.Limiter,
+ channel slack.Channel,
+ seenFiles ...map[string]struct{},
+) {
+ seen := make(map[string]struct{})
+ if len(seenFiles) > 0 && seenFiles[0] != nil {
+ seen = seenFiles[0]
+ }
cursor := ""
- retries := 0
- for {
- select {
- case <-ctx.Done():
+ for page := 1; ; page++ {
+ if page > maxSlackPages {
+ s.captureErr(fmt.Errorf("slack conversation history for channel %s exceeded %d pages", channel.ID, maxSlackPages))
+ return
+ }
+ resp, ok := s.fetchConversationHistoryPage(ctx, historyLimiter, channel, cursor)
+ if !ok || !s.processSlackMessages(ctx, ch, fileInfoLimiter, fileDownloadLimiter, channel, resp.Messages, seen) {
return
- default:
}
- if err := limiter.Wait(ctx); err != nil {
- slog.Warn("slack rate limiter wait failed", "channel", channel.ID, "error", err)
+ if !resp.HasMore {
return
}
+ nextCursor := strings.TrimSpace(resp.ResponseMetaData.NextCursor)
+ if nextCursor == "" || nextCursor == cursor {
+ s.captureErr(fmt.Errorf("slack conversation history for channel %s returned a stalled cursor", channel.ID))
+ return
+ }
+ cursor = nextCursor
+ }
+}
- params := &slack.GetConversationHistoryParameters{
- ChannelID: channel.ID,
- Cursor: cursor,
- Limit: 200,
+func (s *SlackSource) fetchConversationHistoryPage(
+ ctx context.Context,
+ limiter *rate.Limiter,
+ channel slack.Channel,
+ cursor string,
+) (*slack.GetConversationHistoryResponse, bool) {
+ params := s.historyParameters(channel.ID, cursor)
+ for retries := 0; ; retries++ {
+ if err := limiter.Wait(ctx); err != nil {
+ slog.Warn("slack rate limiter wait failed", "channel", channel.ID, "error", s.safeSlackError(err))
+ s.captureErr(fmt.Errorf("slack conversation history for channel %s rate limiter wait: %w", channel.ID, err))
+ return nil, false
+ }
+ response, err := s.client.GetConversationHistoryContext(ctx, params)
+ if err == nil {
+ return response, true
}
+ if !s.retryConversationHistory(ctx, channel.ID, retries+1, err) {
+ return nil, false
+ }
+ }
+}
+
+func (s *SlackSource) historyParameters(channelID, cursor string) *slack.GetConversationHistoryParameters {
+ params := &slack.GetConversationHistoryParameters{ChannelID: channelID, Cursor: cursor, Limit: defaultHistoryLimit}
+ if !s.since.IsZero() {
+ params.Oldest = formatSlackTimestamp(s.since)
+ }
+ return params
+}
- // Push the since filter down to the API via the "oldest" parameter so
- // older messages are never transferred. The client-side check below
- // remains as a correctness backstop for boundary timestamps.
- if !s.since.IsZero() {
- params.Oldest = formatSlackTimestamp(s.since)
+func (s *SlackSource) retryConversationHistory(ctx context.Context, channelID string, attempt int, err error) bool {
+ retryAfter, rateLimited := rateLimitRetryAfter(err)
+ if !rateLimited {
+ slog.Warn("slack conversation history failed", "channel", channelID, "error", s.safeSlackError(err))
+ s.captureErr(fmt.Errorf("slack conversation history for channel %s: %w", channelID, err))
+ return false
+ }
+ if attempt > maxRateLimitRetries {
+ slog.Warn("slack conversation history rate limited, giving up",
+ "channel", channelID, "attempts", attempt, "error", s.safeSlackError(err))
+ s.captureErr(fmt.Errorf("slack conversation history for channel %s: rate limited after %d retries: %w", channelID, maxRateLimitRetries, err))
+ return false
+ }
+ slog.Warn("slack conversation history rate limited, retrying",
+ "channel", channelID, "retry_after", retryAfter, "attempt", attempt)
+ if waitErr := waitRetryAfter(ctx, retryAfter); waitErr != nil {
+ slog.Warn("slack rate limiter wait failed", "channel", channelID, "error", s.safeSlackError(waitErr))
+ s.captureErr(fmt.Errorf("slack conversation history for channel %s retry wait: %w", channelID, waitErr))
+ return false
+ }
+ return true
+}
+
+func (s *SlackSource) processSlackMessages(
+ ctx context.Context,
+ output chan<- source.Chunk,
+ fileInfoLimiter, fileDownloadLimiter *rate.Limiter,
+ channel slack.Channel,
+ messages []slack.Message,
+ seen map[string]struct{},
+) bool {
+ for _, message := range messages {
+ if ctx.Err() != nil {
+ return false
+ }
+ if !s.since.IsZero() && parseSlackTimestamp(message.Timestamp).Before(s.since) {
+ continue
}
+ if !s.processSlackMessage(ctx, output, fileInfoLimiter, fileDownloadLimiter, channel, message, seen) {
+ return false
+ }
+ }
+ return true
+}
+
+func (s *SlackSource) processSlackMessage(
+ ctx context.Context,
+ output chan<- source.Chunk,
+ fileInfoLimiter, fileDownloadLimiter *rate.Limiter,
+ channel slack.Channel,
+ message slack.Message,
+ seen map[string]struct{},
+) bool {
+ metadata := finding.SourceMetadata{
+ SourceType: "slack", Channel: channel.ID, ChannelName: channel.Name,
+ MessageUser: message.User, MessageTS: message.Timestamp, ThreadTS: message.ThreadTimestamp,
+ }
+ if message.Text != "" && !sendSlackChunk(ctx, output, source.Chunk{Data: []byte(message.Text), SourceMetadata: metadata}) {
+ return false
+ }
+ if !s.includeFiles {
+ return true
+ }
+ return s.processSlackFiles(ctx, output, fileInfoLimiter, fileDownloadLimiter, channel, message, metadata, seen)
+}
- resp, err := s.client.GetConversationHistoryContext(ctx, params)
+func (s *SlackSource) processSlackFiles(
+ ctx context.Context,
+ output chan<- source.Chunk,
+ fileInfoLimiter, fileDownloadLimiter *rate.Limiter,
+ channel slack.Channel,
+ message slack.Message,
+ metadata finding.SourceMetadata,
+ seen map[string]struct{},
+) bool {
+ for _, attached := range message.Files {
+ if attached.ID == "" {
+ continue
+ }
+ if _, duplicate := seen[attached.ID]; duplicate {
+ continue
+ }
+ seen[attached.ID] = struct{}{}
+ chunk, ok, err := s.fileChunk(ctx, fileInfoLimiter, fileDownloadLimiter, channel, message, attached, metadata)
if err != nil {
- if retryAfter, ok := rateLimitRetryAfter(err); ok {
- retries++
- if retries > maxRateLimitRetries {
- slog.Warn("slack conversation history rate limited, giving up",
- "channel", channel.ID, "attempts", retries, "error", err)
- return
- }
- slog.Warn("slack conversation history rate limited, retrying",
- "channel", channel.ID, "retry_after", retryAfter, "attempt", retries)
- if waitErr := waitRetryAfter(ctx, retryAfter); waitErr != nil {
- slog.Warn("slack rate limiter wait failed", "channel", channel.ID, "error", waitErr)
- return
- }
- continue
+ if ctx.Err() != nil {
+ return false
}
- slog.Warn("slack conversation history failed", "channel", channel.ID, "error", err)
- return
+ s.skippedAttachments++
+ slog.Warn("skipping unreadable Slack attachment",
+ "channel", channel.ID,
+ "message_ts", message.Timestamp,
+ "file_id", attached.ID,
+ "error", s.safeSlackError(err))
+ continue
}
- retries = 0
+ if ok && !sendSlackChunk(ctx, output, chunk) {
+ return false
+ }
+ }
+ return true
+}
- for _, msg := range resp.Messages {
- select {
- case <-ctx.Done():
- return
- default:
- }
+func sendSlackChunk(ctx context.Context, output chan<- source.Chunk, chunk source.Chunk) bool {
+ select {
+ case output <- chunk:
+ return true
+ case <-ctx.Done():
+ return false
+ }
+}
- // Apply since filter by parsing the message timestamp.
- if !s.since.IsZero() {
- msgTime := parseSlackTimestamp(msg.Timestamp)
- if msgTime.Before(s.since) {
- continue
- }
- }
+func (s *SlackSource) fileChunk(
+ ctx context.Context,
+ fileInfoLimiter *rate.Limiter,
+ fileDownloadLimiter *rate.Limiter,
+ channel slack.Channel,
+ message slack.Message,
+ attached slack.File,
+ metadata finding.SourceMetadata,
+) (source.Chunk, bool, error) {
+ info, err := s.fileInfo(ctx, fileInfoLimiter, attached)
+ if err != nil {
+ return source.Chunk{}, false, fmt.Errorf("slack file %s metadata: %w", attached.ID, err)
+ }
+ if info.Size > 0 && int64(info.Size) > s.maxFileSize {
+ slog.Debug("skipping oversized Slack attachment", "file_id", info.ID, "size", info.Size, "limit", s.maxFileSize)
+ return source.Chunk{}, false, nil
+ }
+ if !isTextLikeSlackFile(*info) {
+ return source.Chunk{}, false, nil
+ }
+ downloadURL := info.URLPrivateDownload
+ if downloadURL == "" {
+ downloadURL = info.URLPrivate
+ }
+ if err := validateSlackDownloadURL(downloadURL); err != nil {
+ return source.Chunk{}, false, fmt.Errorf("slack file %s download URL rejected: %w", info.ID, err)
+ }
- if msg.Text == "" {
- continue
- }
+ contents, err := s.downloadFile(ctx, fileDownloadLimiter, downloadURL)
+ if err != nil {
+ if errors.Is(err, errSlackFileTooLarge) {
+ return source.Chunk{}, false, nil
+ }
+ return source.Chunk{}, false, fmt.Errorf("slack file %s download: %w", info.ID, err)
+ }
+ if len(contents) == 0 || !isProbablyText(contents) {
+ return source.Chunk{}, false, nil
+ }
- // Planned (see ROADMAP): Slack file scanning. When s.includeFiles is
- // honored, msg.Files (and each File.URLPrivate) should be downloaded
- // and emitted as additional chunks here. Currently only msg.Text is
- // scanned.
+ fileName := safeSlackPathSegment(info.Name, "attachment")
+ channelName := safeSlackPathSegment(strings.TrimPrefix(channel.Name, "#"), "channel")
+ metadata.FilePath = path.Join("slack", channelName, fileName)
+ metadata.MessageUser = message.User
+ return source.Chunk{Data: contents, SourceMetadata: metadata}, true, nil
+}
- select {
- case ch <- source.Chunk{
- Data: []byte(msg.Text),
- SourceMetadata: finding.SourceMetadata{
- SourceType: "slack",
- Channel: channel.ID,
- ChannelName: channel.Name,
- MessageUser: msg.User,
- MessageTS: msg.Timestamp,
- ThreadTS: msg.ThreadTimestamp,
- },
- }:
- case <-ctx.Done():
- return
+func safeSlackPathSegment(value, fallback string) string {
+ segment := path.Base(strings.ReplaceAll(value, "\\", "/"))
+ if segment == "." || segment == ".." || segment == "/" || segment == "" {
+ return fallback
+ }
+ var cleaned strings.Builder
+ for _, char := range segment {
+ if char < 0x20 || char == 0x7f {
+ cleaned.WriteByte('_')
+ continue
+ }
+ cleaned.WriteRune(char)
+ if cleaned.Len() >= 255 {
+ break
+ }
+ }
+ if cleaned.Len() == 0 {
+ return fallback
+ }
+ return cleaned.String()
+}
+
+func (s *SlackSource) downloadFile(ctx context.Context, limiter *rate.Limiter, downloadURL string) ([]byte, error) {
+ for attempt := 0; attempt <= maxRateLimitRetries; attempt++ {
+ if err := limiter.Wait(ctx); err != nil {
+ return nil, fmt.Errorf("slack file download limiter wait: %w", err)
+ }
+ buffer := &boundedFileBuffer{remaining: s.maxFileSize}
+ err := s.client.GetFileContext(ctx, downloadURL, buffer)
+ if err == nil {
+ return buffer.Bytes(), nil
+ }
+ if errors.Is(err, errSlackFileTooLarge) {
+ return nil, err
+ }
+ retryAfter, limited := rateLimitRetryAfter(err)
+ if !limited || attempt == maxRateLimitRetries {
+ return nil, err
+ }
+ if err := waitRetryAfter(ctx, retryAfter); err != nil {
+ return nil, err
+ }
+ }
+ return nil, fmt.Errorf("slack file download retry budget exhausted")
+}
+
+func (s *SlackSource) fileInfo(ctx context.Context, limiter *rate.Limiter, attached slack.File) (*slack.File, error) {
+ for attempt := 0; attempt <= maxRateLimitRetries; attempt++ {
+ if err := limiter.Wait(ctx); err != nil {
+ return nil, fmt.Errorf("slack file metadata limiter wait: %w", err)
+ }
+ info, _, _, err := s.client.GetFileInfoContext(ctx, attached.ID, 0, 0)
+ if err == nil {
+ if info == nil {
+ return nil, fmt.Errorf("slack returned empty file metadata")
}
+ return info, nil
+ }
+ retryAfter, limited := rateLimitRetryAfter(err)
+ if !limited || attempt == maxRateLimitRetries {
+ return nil, err
}
+ if err := waitRetryAfter(ctx, retryAfter); err != nil {
+ return nil, err
+ }
+ }
+ return nil, fmt.Errorf("slack file metadata retry budget exhausted")
+}
- if !resp.HasMore {
- return
+func isTextLikeSlackFile(file slack.File) bool {
+ rawType := strings.TrimSpace(file.Mimetype)
+ if rawType == "" {
+ return true
+ }
+ mimeType, _, err := mime.ParseMediaType(rawType)
+ if err != nil {
+ return false
+ }
+ mimeType = strings.ToLower(mimeType)
+ if strings.HasPrefix(mimeType, "text/") {
+ return true
+ }
+ switch mimeType {
+ case "application/json", "application/ld+json", "application/xml", "application/x-yaml",
+ "application/yaml", "application/toml", "application/javascript", "application/x-sh":
+ return true
+ default:
+ return false
+ }
+}
+
+func isProbablyText(data []byte) bool {
+ if len(data) == 0 || !utf8.Valid(data) || bytes.IndexByte(data, 0) >= 0 {
+ return false
+ }
+ total, nonText := 0, 0
+ for _, char := range string(data) {
+ total++
+ if unicode.IsPrint(char) || char == '\n' || char == '\r' || char == '\t' {
+ continue
+ }
+ nonText++
+ }
+ return total > 0 && nonText*100 <= total*5
+}
+
+func validateSlackDownloadURL(raw string) error {
+ parsed, err := url.Parse(raw)
+ if err != nil || parsed.Scheme != "https" || parsed.User != nil || parsed.Hostname() == "" {
+ return fmt.Errorf("expected an absolute credential-free HTTPS URL")
+ }
+ if !strings.EqualFold(parsed.Hostname(), "files.slack.com") || (parsed.Port() != "" && parsed.Port() != "443") {
+ return fmt.Errorf("host is not Slack's file-download endpoint")
+ }
+ return nil
+}
+
+var errSlackFileTooLarge = errors.New("slack attachment exceeds configured maximum")
+
+type boundedFileBuffer struct {
+ bytes.Buffer
+ remaining int64
+}
+
+func (w *boundedFileBuffer) Write(input []byte) (int, error) {
+ if int64(len(input)) > w.remaining {
+ written := 0
+ if w.remaining > 0 {
+ written, _ = w.Buffer.Write(input[:int(w.remaining)])
+ w.remaining = 0
}
- cursor = resp.ResponseMetaData.NextCursor
+ return written, errSlackFileTooLarge
}
+ w.remaining -= int64(len(input))
+ return w.Buffer.Write(input)
}
+var _ io.Writer = (*boundedFileBuffer)(nil)
+
// formatSlackTimestamp converts a time.Time to the Slack "oldest" parameter
// format (Unix seconds with a fractional component, e.g. "1234567890.000000").
func formatSlackTimestamp(t time.Time) string {
diff --git a/internal/source/slack/slack_test.go b/internal/source/slack/slack_test.go
index 51320d4..c72a61b 100644
--- a/internal/source/slack/slack_test.go
+++ b/internal/source/slack/slack_test.go
@@ -1,8 +1,14 @@
package slack
import (
+ "bytes"
"context"
+ "errors"
"fmt"
+ "io"
+ "log/slog"
+ "net/http"
+ "net/http/httptest"
"testing"
"time"
@@ -29,13 +35,25 @@ type mockHistoryPage struct {
nextCursor string
}
+type hiddenSlackError struct{ cause error }
+
+func (e hiddenSlackError) Error() string { return "outer SDK failure" }
+func (e hiddenSlackError) Unwrap() error { return e.cause }
+
// mockSlackClient is a minimal mock for the slackClient interface.
type mockSlackClient struct {
- channels []slack.Channel
- messages map[string][]slack.Message
- authErr error
- listErr error
- historyErr error
+ channels []slack.Channel
+ messages map[string][]slack.Message
+ authErr error
+ authWait bool
+ authCtx context.Context
+ listErr error
+ historyErr error
+ fileInfoErr error
+ downloadErr error
+ files map[string]slack.File
+ fileBodies map[string][]byte
+ downloadStarted chan string
// listRateLimitedCalls, when > 0, makes that many upcoming calls to
// GetConversationsContext return a *slack.RateLimitedError before
@@ -58,15 +76,23 @@ type mockSlackClient struct {
historyPages map[string]mockHistoryPage
// listCalls and historyCalls count invocations, for assertions.
- listCalls int
- historyCalls int
+ listCalls int
+ historyCalls int
+ fileInfoCalls int
+ downloadCalls int
// lastHistoryOldest records the Oldest parameter from the most recent
// GetConversationHistoryContext call, for assertions.
lastHistoryOldest string
+ lastHistoryLimit int
}
-func (m *mockSlackClient) AuthTestContext(_ context.Context) (*slack.AuthTestResponse, error) {
+func (m *mockSlackClient) AuthTestContext(ctx context.Context) (*slack.AuthTestResponse, error) {
+ m.authCtx = ctx
+ if m.authWait {
+ <-ctx.Done()
+ return nil, ctx.Err()
+ }
if m.authErr != nil {
return nil, m.authErr
}
@@ -103,6 +129,7 @@ func (m *mockSlackClient) GetConversationsContext(_ context.Context, params *sla
func (m *mockSlackClient) GetConversationHistoryContext(_ context.Context, params *slack.GetConversationHistoryParameters) (*slack.GetConversationHistoryResponse, error) {
m.historyCalls++
m.lastHistoryOldest = params.Oldest
+ m.lastHistoryLimit = params.Limit
if m.historyRateLimitedCalls > 0 {
m.historyRateLimitedCalls--
@@ -143,6 +170,41 @@ func (m *mockSlackClient) GetConversationHistoryContext(_ context.Context, param
}, nil
}
+func (m *mockSlackClient) GetFileInfoContext(_ context.Context, fileID string, _, _ int) (*slack.File, []slack.Comment, *slack.Paging, error) {
+ m.fileInfoCalls++
+ if m.fileInfoErr != nil {
+ return nil, nil, nil, m.fileInfoErr
+ }
+ file, ok := m.files[fileID]
+ if !ok {
+ return nil, nil, nil, fmt.Errorf("file_not_found")
+ }
+ return &file, nil, &slack.Paging{}, nil
+}
+
+func (m *mockSlackClient) GetFileContext(ctx context.Context, downloadURL string, writer io.Writer) error {
+ m.downloadCalls++
+ if m.downloadStarted != nil {
+ select {
+ case m.downloadStarted <- downloadURL:
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+ }
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ if m.downloadErr != nil {
+ return m.downloadErr
+ }
+ body, ok := m.fileBodies[downloadURL]
+ if !ok {
+ return fmt.Errorf("download_not_found")
+ }
+ _, err := io.Copy(writer, bytes.NewReader(body))
+ return err
+}
+
func TestSlackSource_Type_ReturnsSlack(t *testing.T) {
s := New("xoxb-test-token")
assert.Equal(t, "slack", s.Type())
@@ -152,7 +214,7 @@ func TestSlackSource_Validate_ValidToken_ReturnsNoError(t *testing.T) {
s := New("xoxb-test-token")
s.client = &mockSlackClient{}
- err := s.Validate()
+ err := s.Validate(context.Background())
assert.NoError(t, err)
}
@@ -162,7 +224,7 @@ func TestSlackSource_Validate_InvalidToken_ReturnsError(t *testing.T) {
authErr: fmt.Errorf("invalid_auth"),
}
- err := s.Validate()
+ err := s.Validate(context.Background())
require.Error(t, err)
assert.Contains(t, err.Error(), "slack auth test failed")
assert.Contains(t, err.Error(), "invalid_auth")
@@ -171,11 +233,49 @@ func TestSlackSource_Validate_InvalidToken_ReturnsError(t *testing.T) {
func TestSlackSource_Validate_EmptyToken_ReturnsError(t *testing.T) {
s := New("")
- err := s.Validate()
+ err := s.Validate(context.Background())
require.Error(t, err)
assert.Contains(t, err.Error(), "slack token is required")
}
+func TestSlackSource_Validate_IsBoundedWithoutCallerDeadline(t *testing.T) {
+ mock := &mockSlackClient{}
+ s := New("xoxb-test-token")
+ s.client = mock
+
+ started := time.Now()
+ require.NoError(t, s.Validate(context.Background()))
+ deadline, ok := mock.authCtx.Deadline()
+ require.True(t, ok, "AuthTest validation must always carry a deadline")
+ assert.Greater(t, deadline, started)
+ assert.LessOrEqual(t, deadline.Sub(started), validationTimeout+time.Second)
+}
+
+func TestSlackSource_Validate_HonorsCallerCancellationAndRedactsToken(t *testing.T) {
+ const token = "xoxb-synthetic-validation-canary"
+ mock := &mockSlackClient{authWait: true}
+ s := New(token)
+ s.client = mock
+
+ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
+ defer cancel()
+ started := time.Now()
+ err := s.Validate(ctx)
+ require.Error(t, err)
+ assert.ErrorIs(t, err, context.DeadlineExceeded)
+ assert.Less(t, time.Since(started), time.Second)
+ assert.NotContains(t, err.Error(), token)
+
+ // A provider/library error that echoes the raw token must also be flattened
+ // before it can reach CLI stderr.
+ mock = &mockSlackClient{authErr: fmt.Errorf("invalid token %s", token)}
+ s.client = mock
+ err = s.Validate(context.Background())
+ require.Error(t, err)
+ assert.False(t, errors.Is(err, context.Canceled))
+ assert.NotContains(t, err.Error(), token)
+}
+
func TestSlackSource_Chunks_SingleChannel_EmitsMessages(t *testing.T) {
mock := &mockSlackClient{
channels: []slack.Channel{
@@ -205,6 +305,7 @@ func TestSlackSource_Chunks_SingleChannel_EmitsMessages(t *testing.T) {
assert.Len(t, chunks, 2)
assert.Contains(t, chunks, "here is my API_KEY=sk-abc123")
assert.Contains(t, chunks, "another message with SECRET=xyz")
+ assert.Equal(t, defaultHistoryLimit, mock.lastHistoryLimit)
// All chunks should reference channel C001.
for _, m := range metas {
assert.Equal(t, "C001", m)
@@ -377,16 +478,22 @@ func TestSlackSource_New_IncludeFilesDefaultsFalse(t *testing.T) {
assert.False(t, s.includeFiles)
}
-func TestSlackSource_Chunks_IncludeFiles_ScansTextOnly(t *testing.T) {
- // File scanning is not implemented; enabling WithIncludeFiles must not
- // change the emitted chunks (still text-only) and must not error.
+func TestSlackSource_Chunks_IncludeFiles_ScansMessageAndBoundedTextAttachment(t *testing.T) {
+ const downloadURL = "https://files.slack.com/files-pri/T1-F1/config.env"
mock := &mockSlackClient{
channels: []slack.Channel{
{GroupConversation: slack.GroupConversation{Conversation: slack.Conversation{ID: "C001"}, Name: "general"}},
},
messages: map[string][]slack.Message{
- "C001": {{Msg: slack.Msg{Text: "hello", User: "U001", Timestamp: "1700000001.000000"}}},
+ "C001": {{Msg: slack.Msg{
+ Text: "hello", User: "U001", Timestamp: "1700000001.000000",
+ Files: []slack.File{{ID: "F1"}},
+ }}},
+ },
+ files: map[string]slack.File{
+ "F1": {ID: "F1", Name: "config.env", Mimetype: "text/plain", Size: 20, URLPrivateDownload: downloadURL},
},
+ fileBodies: map[string][]byte{downloadURL: []byte("TOKEN=synthetic-value")},
}
s := New("xoxb-test-token", WithIncludeFiles(true), WithRateLimit(1000))
@@ -398,7 +505,328 @@ func TestSlackSource_Chunks_IncludeFiles_ScansTextOnly(t *testing.T) {
chunks = append(chunks, string(chunk.Data))
}
- assert.Equal(t, []string{"hello"}, chunks)
+ assert.Equal(t, []string{"hello", "TOKEN=synthetic-value"}, chunks)
+ assert.Equal(t, 1, mock.fileInfoCalls)
+ assert.Equal(t, 1, mock.downloadCalls)
+}
+
+func TestSlackSource_Chunks_IncludeFiles_RejectsHostileDownloadURLBeforeClientCall(t *testing.T) {
+ const hostileURL = "https://files.slack.com.attacker.example/files-pri/T1-F1/config.env"
+ mock := &mockSlackClient{
+ channels: []slack.Channel{
+ {GroupConversation: slack.GroupConversation{Conversation: slack.Conversation{ID: "C001"}, Name: "general"}},
+ },
+ messages: map[string][]slack.Message{
+ "C001": {{Msg: slack.Msg{Timestamp: "1700000001.000000", Files: []slack.File{{ID: "F1"}}}}},
+ },
+ files: map[string]slack.File{
+ "F1": {ID: "F1", Name: "config.env", Mimetype: "text/plain", URLPrivateDownload: hostileURL},
+ },
+ fileBodies: map[string][]byte{hostileURL: []byte("TOKEN=must-not-be-downloaded")},
+ }
+
+ s := New("xoxb-test-token", WithIncludeFiles(true), WithRateLimit(1000))
+ s.client = mock
+ for range s.Chunks(context.Background()) {
+ }
+
+ assert.Equal(t, 1, mock.fileInfoCalls)
+ assert.Zero(t, mock.downloadCalls, "an untrusted attachment URL must be rejected before the authenticated download client is called")
+ assert.Equal(t, 1, s.skippedAttachments)
+ assert.NoError(t, s.Err(), "one rejected attachment must not abort the remaining workspace scan")
+}
+
+func TestSlackSource_Chunks_IncludeFiles_SkipsOversizedBinaryAndDuplicateFiles(t *testing.T) {
+ const (
+ textURL = "https://files.slack.com/files-pri/T1-F1/text"
+ binaryURL = "https://files.slack.com/files-pri/T1-F2/binary"
+ )
+ mock := &mockSlackClient{
+ channels: []slack.Channel{{GroupConversation: slack.GroupConversation{Conversation: slack.Conversation{ID: "C001"}, Name: "general"}}},
+ messages: map[string][]slack.Message{"C001": {
+ {Msg: slack.Msg{Timestamp: "1700000001.000000", Files: []slack.File{{ID: "F1"}, {ID: "F2"}, {ID: "F3"}}}},
+ {Msg: slack.Msg{Timestamp: "1700000002.000000", Files: []slack.File{{ID: "F1"}}}},
+ }},
+ files: map[string]slack.File{
+ "F1": {ID: "F1", Name: "large.txt", Mimetype: "text/plain", Size: 100, URLPrivateDownload: textURL},
+ "F2": {ID: "F2", Name: "image.bin", Mimetype: "application/octet-stream", Size: 3, URLPrivateDownload: binaryURL},
+ "F3": {ID: "F3", Name: "nul.txt", Mimetype: "text/plain", Size: 3, URLPrivateDownload: textURL},
+ },
+ fileBodies: map[string][]byte{textURL: {'a', 0, 'b'}, binaryURL: {1, 2, 3}},
+ }
+ s := New("xoxb-test-token", WithIncludeFiles(true), WithMaxFileSize(16), WithRateLimit(1000))
+ s.client = mock
+
+ var chunks []source.Chunk
+ for chunk := range s.Chunks(context.Background()) {
+ chunks = append(chunks, chunk)
+ }
+ assert.Empty(t, chunks)
+ assert.Equal(t, 3, mock.fileInfoCalls, "duplicate F1 must not be fetched twice")
+ assert.Equal(t, 1, mock.downloadCalls, "only the text-like, in-limit F3 reaches download")
+}
+
+func TestSlackSource_Chunks_IncludeFiles_DownloadFailureIsSkippedAndLaterChannelsContinue(t *testing.T) {
+ const downloadURL = "https://files.slack.com/files-pri/T1-F1/config.env"
+ token := "xoxb-sensitive-fixture"
+ mock := &mockSlackClient{
+ channels: []slack.Channel{
+ {GroupConversation: slack.GroupConversation{Conversation: slack.Conversation{ID: "C001"}, Name: "general"}},
+ {GroupConversation: slack.GroupConversation{Conversation: slack.Conversation{ID: "C002"}, Name: "security"}},
+ },
+ messages: map[string][]slack.Message{
+ "C001": {{Msg: slack.Msg{Timestamp: "1700000001.000000", Files: []slack.File{{ID: "F1"}}}}},
+ "C002": {{Msg: slack.Msg{Timestamp: "1700000002.000000", Text: "later channel secret"}}},
+ },
+ files: map[string]slack.File{"F1": {ID: "F1", Name: "config.env", Mimetype: "text/plain", URLPrivateDownload: downloadURL}},
+ downloadErr: fmt.Errorf("download failed for %s", token),
+ }
+ var logs bytes.Buffer
+ previous := slog.Default()
+ slog.SetDefault(slog.New(slog.NewTextHandler(&logs, nil)))
+ t.Cleanup(func() { slog.SetDefault(previous) })
+
+ s := New(token, WithIncludeFiles(true), WithRateLimit(1000))
+ s.client = mock
+ var chunks []source.Chunk
+ for chunk := range s.Chunks(context.Background()) {
+ chunks = append(chunks, chunk)
+ }
+ require.NoError(t, s.Err())
+ require.Len(t, chunks, 1)
+ assert.Equal(t, "later channel secret", string(chunks[0].Data))
+ assert.Equal(t, "C002", chunks[0].SourceMetadata.Channel)
+ assert.Equal(t, 2, mock.historyCalls)
+ assert.Equal(t, 1, s.skippedAttachments)
+ assert.NotContains(t, logs.String(), token)
+ assert.Contains(t, logs.String(), "***")
+}
+
+func TestSlackSource_Chunks_ProviderErrorsAreRedactedBeforeLogging(t *testing.T) {
+ const token = "xoxb-sensitive-log-canary"
+ tests := []struct {
+ name string
+ mock *mockSlackClient
+ }{
+ {
+ name: "channel list",
+ mock: &mockSlackClient{listErr: fmt.Errorf("SDK echoed Bearer %s", token)},
+ },
+ {
+ name: "history",
+ mock: &mockSlackClient{
+ channels: []slack.Channel{{GroupConversation: slack.GroupConversation{Conversation: slack.Conversation{ID: "C1"}, Name: "general"}}},
+ historyErr: fmt.Errorf("SDK echoed %s", token),
+ },
+ },
+ {
+ name: "rate retry exhaustion",
+ mock: &mockSlackClient{
+ channels: []slack.Channel{{GroupConversation: slack.GroupConversation{Conversation: slack.Conversation{ID: "C1"}, Name: "general"}}},
+ historyErr: fmt.Errorf("SDK echoed %s: %w", token, &slack.RateLimitedError{RetryAfter: time.Millisecond}),
+ },
+ },
+ }
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ var logs bytes.Buffer
+ previous := slog.Default()
+ slog.SetDefault(slog.New(slog.NewTextHandler(&logs, nil)))
+ t.Cleanup(func() { slog.SetDefault(previous) })
+
+ s := New(token, WithRateLimit(1000))
+ s.client = tc.mock
+ for range s.Chunks(context.Background()) {
+ }
+ require.Error(t, s.Err())
+ assert.NotContains(t, logs.String(), token)
+ assert.NotContains(t, s.Err().Error(), token)
+ assert.Contains(t, logs.String(), "***")
+ assert.Contains(t, s.Err().Error(), "***")
+ })
+ }
+}
+
+func TestSlackSource_CaptureErrAlwaysFlattensUnwrapChain(t *testing.T) {
+ secretCause := errors.New("xoxb-hidden-in-unwrap")
+ s := New("xoxb-hidden-in-unwrap")
+ s.captureErr(hiddenSlackError{cause: secretCause})
+ require.Error(t, s.Err())
+ assert.Equal(t, "outer SDK failure", s.Err().Error())
+ assert.False(t, errors.Is(s.Err(), secretCause))
+}
+
+func TestSlackSource_Chunks_IncludeFilesUsesBackpressureBeforeSecondDownload(t *testing.T) {
+ const (
+ firstURL = "https://files.slack.com/files-pri/T1-F1/first"
+ secondURL = "https://files.slack.com/files-pri/T1-F2/second"
+ )
+ started := make(chan string, 2)
+ mock := &mockSlackClient{
+ channels: []slack.Channel{{GroupConversation: slack.GroupConversation{Conversation: slack.Conversation{ID: "C1"}, Name: "general"}}},
+ messages: map[string][]slack.Message{"C1": {{Msg: slack.Msg{Files: []slack.File{{ID: "F1"}, {ID: "F2"}}}}}},
+ files: map[string]slack.File{
+ "F1": {ID: "F1", Name: "first.env", Mimetype: "text/plain", URLPrivateDownload: firstURL},
+ "F2": {ID: "F2", Name: "second.env", Mimetype: "text/plain", URLPrivateDownload: secondURL},
+ },
+ fileBodies: map[string][]byte{firstURL: []byte("KEY=first"), secondURL: []byte("KEY=second")},
+ downloadStarted: started,
+ }
+ s := New("xoxb-test-token", WithIncludeFiles(true), WithBufferSize(100), WithRateLimit(1000))
+ s.client = mock
+ ctx, cancel := context.WithCancel(context.Background())
+ chunks := s.Chunks(ctx)
+ assert.Zero(t, cap(chunks), "attachment mode must transfer engine backpressure directly")
+
+ select {
+ case got := <-started:
+ assert.Equal(t, firstURL, got)
+ case <-time.After(time.Second):
+ t.Fatal("first attachment download did not start")
+ }
+ select {
+ case got := <-started:
+ t.Fatalf("second attachment downloaded before the first chunk was consumed: %s", got)
+ case <-time.After(50 * time.Millisecond):
+ }
+ cancel()
+ for range chunks {
+ }
+}
+
+func TestSlackSource_DefaultMethodRateLimitsAreIndependent(t *testing.T) {
+ s := New("xoxb-test-token")
+ assert.Equal(t, DefaultRateLimit, s.historyRateLimit)
+ assert.Equal(t, defaultListRateLimit, s.listRateLimit)
+ assert.Equal(t, defaultFileRateLimit, s.fileInfoRateLimit)
+ assert.Equal(t, defaultFileRateLimit, s.fileDownloadRateLimit)
+ assert.NotEqual(t, s.historyRateLimit, s.listRateLimit)
+ assert.NotEqual(t, s.historyRateLimit, s.fileInfoRateLimit)
+
+ WithRateLimit(7)(s)
+ assert.Equal(t, 7.0, s.historyRateLimit)
+ assert.Equal(t, 7.0, s.listRateLimit)
+ assert.Equal(t, 7.0, s.fileInfoRateLimit)
+ assert.Equal(t, 7.0, s.fileDownloadRateLimit)
+}
+
+func TestSlackSource_DefaultListAndHistoryFirstRequestsDoNotBlockEachOther(t *testing.T) {
+ mock := &mockSlackClient{
+ channels: []slack.Channel{{GroupConversation: slack.GroupConversation{Conversation: slack.Conversation{ID: "C1"}, Name: "general"}}},
+ messages: map[string][]slack.Message{"C1": nil},
+ }
+ s := New("xoxb-test-token")
+ s.client = mock
+ done := make(chan struct{})
+ go func() {
+ for range s.Chunks(context.Background()) {
+ }
+ close(done)
+ }()
+ select {
+ case <-done:
+ case <-time.After(2 * time.Second):
+ t.Fatal("method-scoped first requests blocked one another")
+ }
+ assert.Equal(t, 1, mock.listCalls)
+ assert.Equal(t, 1, mock.historyCalls)
+}
+
+func TestSlackAttachmentTextClassification(t *testing.T) {
+ assert.True(t, isTextLikeSlackFile(slack.File{Mimetype: "application/json; charset=utf-8"}))
+ assert.True(t, isTextLikeSlackFile(slack.File{Mimetype: "text/plain; charset=UTF-8"}))
+ assert.False(t, isTextLikeSlackFile(slack.File{Mimetype: "text/plain; broken"}))
+ assert.False(t, isTextLikeSlackFile(slack.File{Mimetype: "application/octet-stream"}))
+
+ assert.True(t, isProbablyText([]byte("{\"secret\":\"value\"}\n")))
+ assert.True(t, isProbablyText([]byte("anahtar=değer\n")))
+ assert.False(t, isProbablyText([]byte{0xff, 0xd8, 0xff, 0xe0, 1, 2, 3}))
+ assert.False(t, isProbablyText([]byte{1, 2, 3, 4, 5}))
+ assert.False(t, isProbablyText([]byte("text\x00binary")))
+}
+
+func TestSlackSource_Chunks_RejectsSpoofedOrMissingMIMEBinaryContent(t *testing.T) {
+ const (
+ spoofedURL = "https://files.slack.com/files-pri/T1-F1/spoofed"
+ missingURL = "https://files.slack.com/files-pri/T1-F2/missing"
+ )
+ mock := &mockSlackClient{
+ channels: []slack.Channel{{GroupConversation: slack.GroupConversation{Conversation: slack.Conversation{ID: "C1"}, Name: "general"}}},
+ messages: map[string][]slack.Message{"C1": {{Msg: slack.Msg{Files: []slack.File{{ID: "F1"}, {ID: "F2"}}}}}},
+ files: map[string]slack.File{
+ "F1": {ID: "F1", Name: "image.txt", Mimetype: "text/plain", URLPrivateDownload: spoofedURL},
+ "F2": {ID: "F2", Name: "archive", Mimetype: "", URLPrivateDownload: missingURL},
+ },
+ fileBodies: map[string][]byte{
+ spoofedURL: {0xff, 0xd8, 0xff, 0xe0, 1, 2, 3},
+ missingURL: {1, 2, 3, 4, 5},
+ },
+ }
+ s := New("xoxb-test-token", WithIncludeFiles(true), WithRateLimit(1000))
+ s.client = mock
+ var chunks []source.Chunk
+ for chunk := range s.Chunks(context.Background()) {
+ chunks = append(chunks, chunk)
+ }
+ assert.Empty(t, chunks)
+ assert.Equal(t, 2, mock.downloadCalls)
+}
+
+func TestBoundedFileBuffer_MaxAndMaxPlusOne(t *testing.T) {
+ exact := &boundedFileBuffer{remaining: 3}
+ n, err := exact.Write([]byte("abc"))
+ require.NoError(t, err)
+ assert.Equal(t, 3, n)
+ assert.Equal(t, "abc", exact.String())
+
+ over := &boundedFileBuffer{remaining: 3}
+ n, err = over.Write([]byte("abcd"))
+ assert.ErrorIs(t, err, errSlackFileTooLarge)
+ assert.Equal(t, 3, n)
+ assert.Equal(t, "abc", over.String())
+}
+
+func TestValidateSlackDownloadURL_RejectsCredentialAndNonSlackDestinations(t *testing.T) {
+ assert.NoError(t, validateSlackDownloadURL("https://files.slack.com/files-pri/T1-F1/file"))
+ assert.NoError(t, validateSlackDownloadURL("https://files.slack.com:443/files-pri/T1-F1/file"))
+ for _, raw := range []string{
+ "http://files.slack.com/file",
+ "https://token@files.slack.com/file",
+ "https://files.slack.com:8443/file",
+ "https://downloads.slack.com/file",
+ "https://files.slack.com.attacker.example/file",
+ "https://127.0.0.1/file",
+ } {
+ assert.Error(t, validateSlackDownloadURL(raw), raw)
+ }
+}
+
+func TestRejectSlackRedirect_PreventsCredentialForwarding(t *testing.T) {
+ redirectTargetCalls := 0
+ target := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
+ redirectTargetCalls++
+ }))
+ defer target.Close()
+ origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ http.Redirect(w, r, target.URL, http.StatusFound)
+ }))
+ defer origin.Close()
+
+ request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, origin.URL, nil)
+ require.NoError(t, err)
+ response, err := newSlackHTTPClient().Do(request)
+ require.NoError(t, err)
+ require.NotNil(t, response)
+ defer response.Body.Close()
+ assert.Equal(t, http.StatusFound, response.StatusCode)
+ assert.Zero(t, redirectTargetCalls, "the authenticated Slack HTTP client must never follow a redirect")
+}
+
+func TestSafeSlackPathSegment_RejectsTraversalAndControls(t *testing.T) {
+ assert.Equal(t, "config.env", safeSlackPathSegment("../config.env", "attachment"))
+ assert.Equal(t, "attachment", safeSlackPathSegment("..", "attachment"))
+ assert.Equal(t, "bad_name.txt", safeSlackPathSegment("bad\nname.txt", "attachment"))
+ assert.Equal(t, "秘密.env", safeSlackPathSegment("秘密.env", "attachment"))
}
func TestSlackSource_Chunks_ContextCancellation_Stops(t *testing.T) {
@@ -535,6 +963,30 @@ func TestSlackSource_ListChannels_RateLimitedBeyondMaxRetries_ReturnsError(t *te
assert.Equal(t, maxRateLimitRetries+1, mock.listCalls)
}
+func TestWaitRetryAfter_RejectsUnboundedProviderDelay(t *testing.T) {
+ start := time.Now()
+ err := waitRetryAfter(context.Background(), maximumRetryAfterWait+time.Second)
+ require.Error(t, err)
+ assert.Less(t, time.Since(start), 100*time.Millisecond)
+ assert.NotContains(t, err.Error(), (maximumRetryAfterWait + time.Second).String())
+}
+
+func TestSlackSource_Chunks_UnboundedRetryAfter_MarksScanFailed(t *testing.T) {
+ mock := &mockSlackClient{
+ channels: []slack.Channel{{GroupConversation: slack.GroupConversation{Conversation: slack.Conversation{ID: "C001"}, Name: "general"}}},
+ historyRateLimitedCalls: 1,
+ rateLimitRetryAfter: maximumRetryAfterWait + time.Second,
+ }
+ s := New("xoxb-test-token", WithRateLimit(1000))
+ s.client = mock
+
+ for range s.Chunks(context.Background()) {
+ }
+ require.Error(t, s.Err())
+ assert.Contains(t, s.Err().Error(), "retry wait")
+ assert.Equal(t, 1, mock.historyCalls)
+}
+
func TestSlackSource_ProcessChannel_RateLimited_RetriesAndEmits(t *testing.T) {
mock := &mockSlackClient{
messages: map[string][]slack.Message{
@@ -551,7 +1003,7 @@ func TestSlackSource_ProcessChannel_RateLimited_RetriesAndEmits(t *testing.T) {
limiter := rate.NewLimiter(rate.Inf, 1)
channel := slack.Channel{GroupConversation: slack.GroupConversation{Conversation: slack.Conversation{ID: "C001"}, Name: "general"}}
- s.processChannel(context.Background(), ch, limiter, channel)
+ s.processChannel(context.Background(), ch, limiter, limiter, limiter, channel)
close(ch)
var texts []string
@@ -583,7 +1035,7 @@ func TestSlackSource_ProcessChannel_RateLimitedBeyondMaxRetries_GivesUpGracefull
// Must not panic and must not hang; give it a bounded time budget.
done := make(chan struct{})
go func() {
- s.processChannel(context.Background(), ch, limiter, channel)
+ s.processChannel(context.Background(), ch, limiter, limiter, limiter, channel)
close(ch)
close(done)
}()
@@ -620,7 +1072,7 @@ func TestSlackSource_Chunks_ListError_ReturnsNoChunksWithoutPanic(t *testing.T)
assert.Equal(t, 0, count)
}
-func TestSlackSource_Chunks_HistoryError_DegradesGracefullyWithoutPanic(t *testing.T) {
+func TestSlackSource_Chunks_HistoryError_IsReportedAndStops(t *testing.T) {
mock := &mockSlackClient{
channels: []slack.Channel{
{GroupConversation: slack.GroupConversation{Conversation: slack.Conversation{ID: "C001"}, Name: "general"}},
@@ -639,9 +1091,9 @@ func TestSlackSource_Chunks_HistoryError_DegradesGracefullyWithoutPanic(t *testi
}
assert.Equal(t, 0, count)
- // Both channels must have been attempted; a history error on one
- // channel must not abort the scan of subsequent channels.
- assert.Equal(t, 2, mock.historyCalls)
+ require.Error(t, s.Err())
+ assert.Contains(t, s.Err().Error(), "conversation history")
+ assert.Equal(t, 1, mock.historyCalls, "a terminal source error must stop partial scanning")
}
// --- Multi-page cursor pagination tests ---
@@ -695,7 +1147,7 @@ func TestSlackSource_ProcessChannel_MultiPageCursorPagination_EmitsAllPages(t *t
limiter := rate.NewLimiter(rate.Inf, 1)
channel := slack.Channel{GroupConversation: slack.GroupConversation{Conversation: slack.Conversation{ID: "C001"}, Name: "general"}}
- s.processChannel(context.Background(), ch, limiter, channel)
+ s.processChannel(context.Background(), ch, limiter, limiter, limiter, channel)
close(ch)
var texts []string
@@ -707,6 +1159,37 @@ func TestSlackSource_ProcessChannel_MultiPageCursorPagination_EmitsAllPages(t *t
assert.Equal(t, 2, mock.historyCalls)
}
+func TestSlackSource_StalledPaginationFailsInsteadOfLooping(t *testing.T) {
+ t.Run("channel list repeated cursor", func(t *testing.T) {
+ mock := &mockSlackClient{listPages: map[string]mockChannelPage{
+ "": {nextCursor: "repeat"},
+ "repeat": {nextCursor: "repeat"},
+ }}
+ s := New("xoxb-test-token")
+ s.client = mock
+ _, err := s.listChannels(context.Background(), rate.NewLimiter(rate.Inf, 1))
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "stalled cursor")
+ assert.Equal(t, 2, mock.listCalls)
+ })
+
+ t.Run("history empty cursor with has more", func(t *testing.T) {
+ mock := &mockSlackClient{
+ channels: []slack.Channel{{GroupConversation: slack.GroupConversation{Conversation: slack.Conversation{ID: "C001"}, Name: "general"}}},
+ historyPages: map[string]mockHistoryPage{
+ "C001|": {hasMore: true, nextCursor: ""},
+ },
+ }
+ s := New("xoxb-test-token", WithRateLimit(1000))
+ s.client = mock
+ for range s.Chunks(context.Background()) {
+ }
+ require.Error(t, s.Err())
+ assert.Contains(t, s.Err().Error(), "stalled cursor")
+ assert.Equal(t, 1, mock.historyCalls)
+ })
+}
+
func TestSlackSource_Chunks_MultiPageChannelsAndHistory_EmitsAllAcrossPages(t *testing.T) {
mock := &mockSlackClient{
listPages: map[string]mockChannelPage{
diff --git a/internal/source/source.go b/internal/source/source.go
index 372ce3f..7717e5b 100644
--- a/internal/source/source.go
+++ b/internal/source/source.go
@@ -39,8 +39,10 @@ type Source interface {
// Err before the channel closes is a data race.
Err() error
- // Validate checks that the source is accessible and valid.
- Validate() error
+ // Validate checks that the source is accessible and valid. Implementations
+ // must honor cancellation and deadlines for every network or blocking
+ // operation performed during validation.
+ Validate(ctx context.Context) error
}
// Chunk is the smallest unit of data to be scanned.
diff --git a/internal/verifier/airtable/airtable_shared_test.go b/internal/verifier/airtable/airtable_shared_test.go
new file mode 100644
index 0000000..ae3d53d
--- /dev/null
+++ b/internal/verifier/airtable/airtable_shared_test.go
@@ -0,0 +1,16 @@
+package airtable
+
+import (
+ "net/http"
+ "testing"
+
+ "github.com/HodeTech/leakwatch/internal/detector"
+ "github.com/HodeTech/leakwatch/internal/verifier"
+ "github.com/HodeTech/leakwatch/internal/verifier/internal/vtest"
+)
+
+func TestVerify_SharedSafetySuite(t *testing.T) {
+ vtest.Run(t, vtest.Case{Name: "airtable", New: func(url string, client *http.Client) verifier.Verifier {
+ return &Verifier{apiURL: url, httpClient: client}
+ }, Raw: detector.RawFinding{DetectorID: detectorID, Raw: []byte("patSyntheticFixture.TokenValue")}})
+}
diff --git a/internal/verifier/auth0/auth0_shared_test.go b/internal/verifier/auth0/auth0_shared_test.go
new file mode 100644
index 0000000..a79634f
--- /dev/null
+++ b/internal/verifier/auth0/auth0_shared_test.go
@@ -0,0 +1,16 @@
+package auth0
+
+import (
+ "net/http"
+ "testing"
+
+ "github.com/HodeTech/leakwatch/internal/detector"
+ "github.com/HodeTech/leakwatch/internal/verifier"
+ "github.com/HodeTech/leakwatch/internal/verifier/internal/vtest"
+)
+
+func TestVerify_SharedSafetySuite(t *testing.T) {
+ vtest.Run(t, vtest.Case{Name: "auth0", New: func(url string, client *http.Client) verifier.Verifier {
+ return &Verifier{apiURL: url, httpClient: client}
+ }, Raw: detector.RawFinding{DetectorID: detectorID, Raw: []byte("aaaaaaaaaaaaaaaa.bbbbbbbbbbbbbbbb.cccccccc")}})
+}
diff --git a/internal/verifier/auth0/auth0_verifier.go b/internal/verifier/auth0/auth0_verifier.go
index 2bea998..deaf86c 100644
--- a/internal/verifier/auth0/auth0_verifier.go
+++ b/internal/verifier/auth0/auth0_verifier.go
@@ -1,15 +1,16 @@
// Package auth0 provides a verifier for Auth0 Management API tokens.
// It uses the Auth0 Management API GET /api/v2/ endpoint with Bearer auth
// to check token validity. The Auth0 Management API is tenant-scoped, so the
-// target host is derived from the token's own iss claim rather than a fixed host.
+// target host must be explicitly trusted by the operator. Repository content
+// and unverified JWT claims never select the request destination.
package auth0
import (
"context"
- "encoding/base64"
"encoding/json"
+ "fmt"
+ "io"
"net/http"
- "strings"
"github.com/HodeTech/leakwatch/internal/detector"
"github.com/HodeTech/leakwatch/internal/verifier"
@@ -19,6 +20,8 @@ import (
const detectorID = "auth0-management-token"
+const clientsProbePath = "/api/v2/clients?per_page=1&include_fields=true&fields=client_id"
+
// Verifier checks whether an Auth0 Management API token is active by calling
// the Auth0 Management API. It NEVER logs or persists raw token values.
type Verifier struct {
@@ -28,6 +31,22 @@ type Verifier struct {
httpClient *http.Client
}
+// NewForTrustedInstance constructs an Auth0 verifier for an origin explicitly
+// supplied by the operator. It must never receive an issuer decoded from an
+// unverified token or a URL extracted from scanned repository content.
+func NewForTrustedInstance(instanceURL string) (*Verifier, error) {
+ normalized, err := verifier.NormalizeTrustedHTTPSOrigin(instanceURL)
+ if err != nil {
+ return nil, err
+ }
+ return &Verifier{apiURL: normalized}, nil
+}
+
+// WithTrustedInstance implements verifier.TrustedInstanceConfigurer.
+func (*Verifier) WithTrustedInstance(instanceURL string) (verifier.Verifier, error) {
+ return NewForTrustedInstance(instanceURL)
+}
+
func init() {
verifier.Register(&Verifier{})
}
@@ -39,11 +58,8 @@ func (v *Verifier) Type() string {
// Verify checks if the detected Auth0 Management API token is valid/active.
//
-// An Auth0 Management API token is a JWT that is only valid against its own
-// tenant's domain. The tenant host is taken from the token's iss claim (when
-// apiURL is not overridden for testing). A non-JWT input, or a JWT without an
-// iss claim, is indeterminate: the token cannot be routed to a tenant, so the
-// result is StatusUnverified rather than a false "invalid".
+// Auth0 Management API tokens are tenant-scoped. Without an operator-trusted
+// tenant origin the result is indeterminate and no request is sent.
func (v *Verifier) Verify(ctx context.Context, raw detector.RawFinding) finding.VerificationResult {
token := string(raw.Raw)
if token == "" {
@@ -55,53 +71,35 @@ func (v *Verifier) Verify(ctx context.Context, raw detector.RawFinding) finding.
apiURL := v.apiURL
if apiURL == "" {
- issuer, ok := issuerFromJWT(token)
- if !ok {
- return finding.VerificationResult{
- Status: finding.StatusUnverified,
- Message: "Auth0 tenant domain could not be determined from token",
- }
+ return finding.VerificationResult{
+ Status: finding.StatusUnverified,
+ Message: "trusted Auth0 tenant origin is not configured",
}
- apiURL = strings.TrimSuffix(issuer, "/")
}
return httpx.VerifyToken(ctx, v.httpClient, token, httpx.TokenSpec{
Name: "auth0",
Request: httpx.Request{
- URL: apiURL + "/api/v2/",
+ URL: apiURL + clientsProbePath,
Header: map[string]string{"Authorization": "Bearer " + token},
},
- ActiveMessage: "Auth0 management token is active",
- InactiveMessage: "Auth0 management token is invalid or expired",
+ ActiveMessage: "Auth0 management token is active",
+ InactiveMessage: "Auth0 management token is invalid or expired",
+ Decode: decodeClients,
+ RequireCompleteBody: true,
+ RequireJSONContentType: true,
})
}
-// issuerFromJWT decodes the unverified payload segment of a JWT and returns its
-// iss claim, which for an Auth0 Management token is the tenant domain URL
-// (for example "https://acme.eu.auth0.com/"). The signature is intentionally
-// NOT validated: the claim is used only to route the verification request to
-// the correct tenant host, and validity is decided by the live API call. It
-// returns ok=false for a non-JWT input or a missing/empty iss claim.
-func issuerFromJWT(token string) (issuer string, ok bool) {
- parts := strings.Split(token, ".")
- if len(parts) != 3 {
- return "", false
- }
-
- // JWT segments use base64url without padding, but tolerate padded input.
- payload, err := base64.RawURLEncoding.DecodeString(strings.TrimRight(parts[1], "="))
- if err != nil {
- return "", false
- }
-
- var claims struct {
- Iss string `json:"iss"`
+func decodeClients(body io.Reader) (map[string]string, string, error) {
+ var clients []struct {
+ ClientID string `json:"client_id"`
}
- if err := json.Unmarshal(payload, &claims); err != nil {
- return "", false
+ if err := json.NewDecoder(body).Decode(&clients); err != nil {
+ return nil, "", err
}
- if claims.Iss == "" {
- return "", false
+ if clients == nil {
+ return nil, "", fmt.Errorf("Auth0 clients response is not an array")
}
- return claims.Iss, true
+ return nil, "", nil
}
diff --git a/internal/verifier/auth0/auth0_verifier_test.go b/internal/verifier/auth0/auth0_verifier_test.go
index ada5e3a..8b98141 100644
--- a/internal/verifier/auth0/auth0_verifier_test.go
+++ b/internal/verifier/auth0/auth0_verifier_test.go
@@ -25,12 +25,13 @@ func makeJWT(iss string) string {
func TestVerify_ValidToken_ReturnsActive(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- assert.Equal(t, "/api/v2/", r.URL.Path)
+ assert.Equal(t, "/api/v2/clients", r.URL.Path)
+ assert.Equal(t, "1", r.URL.Query().Get("per_page"))
assert.Contains(t, r.Header.Get("Authorization"), "Bearer ")
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
- _, _ = w.Write([]byte(`{}`))
+ _, _ = w.Write([]byte(`[]`))
}))
defer server.Close()
@@ -53,6 +54,7 @@ func TestVerify_ValidToken_ReturnsActive(t *testing.T) {
func TestVerify_InvalidToken_ReturnsInactive(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusUnauthorized)
_, _ = w.Write([]byte(`{"statusCode":401,"error":"Unauthorized","message":"Invalid token"}`))
}))
@@ -118,10 +120,7 @@ func TestVerify_EmptyToken_ReturnsUnverified(t *testing.T) {
assert.Equal(t, "empty token", result.Message)
}
-func TestVerify_NonJWTToken_ReturnsUnverified(t *testing.T) {
- // No apiURL override: the verifier must derive the tenant from the token
- // itself. A non-JWT token cannot be routed, so the result is indeterminate
- // (StatusUnverified) — never a false "invalid".
+func TestVerify_MissingTrustedOrigin_ReturnsUnverified(t *testing.T) {
v := &Verifier{}
raw := detector.RawFinding{
@@ -133,21 +132,17 @@ func TestVerify_NonJWTToken_ReturnsUnverified(t *testing.T) {
result := v.Verify(context.Background(), raw)
assert.Equal(t, finding.StatusUnverified, result.Status)
- assert.Contains(t, result.Message, "tenant domain")
+ assert.Contains(t, result.Message, "trusted Auth0 tenant origin")
}
-func TestVerify_TenantHostDerivedFromIssClaim(t *testing.T) {
- var gotHost string
+func TestVerify_UntrustedIssuerClaimCannotSelectDestination(t *testing.T) {
+ called := false
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- gotHost = r.Host
- assert.Equal(t, "/api/v2/", r.URL.Path)
- w.WriteHeader(http.StatusOK)
- _, _ = w.Write([]byte(`{}`))
+ called = true
+ t.Fatalf("untrusted JWT issuer triggered request to %s", r.URL)
}))
defer server.Close()
- // The iss claim points the verifier at the test server; apiURL is left
- // unset so the iss-decoding path (not the test override) is exercised.
v := &Verifier{httpClient: server.Client()}
raw := detector.RawFinding{
@@ -158,50 +153,20 @@ func TestVerify_TenantHostDerivedFromIssClaim(t *testing.T) {
result := v.Verify(context.Background(), raw)
- require.Equal(t, finding.StatusVerifiedActive, result.Status)
- assert.NotEmpty(t, gotHost)
+ require.Equal(t, finding.StatusUnverified, result.Status)
+ assert.False(t, called)
}
-func TestIssuerFromJWT(t *testing.T) {
- tests := []struct {
- name string
- token string
- wantIss string
- wantOK bool
- }{
- {
- name: "valid JWT with iss",
- token: makeJWT("https://acme.eu.auth0.com/"),
- wantIss: "https://acme.eu.auth0.com/",
- wantOK: true,
- },
- {
- name: "not a JWT (single segment)",
- token: "plaintextsecret",
- wantOK: false,
- },
- {
- name: "wrong segment count",
- token: "a.b.c.d",
- wantOK: false,
- },
- {
- name: "JWT without iss claim",
- token: makeJWT(""),
- wantOK: false,
- },
- {
- name: "invalid base64 payload",
- token: "aaa.!!!not-base64!!!.ccc",
- wantOK: false,
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- iss, ok := issuerFromJWT(tt.token)
- assert.Equal(t, tt.wantOK, ok)
- assert.Equal(t, tt.wantIss, iss)
- })
+func TestNewForTrustedInstance_ValidatesOrigin(t *testing.T) {
+ configured, err := NewForTrustedInstance("https://fixture.eu.auth0.com/")
+ require.NoError(t, err)
+ assert.Equal(t, "https://fixture.eu.auth0.com", configured.apiURL)
+
+ for _, origin := range []string{
+ "http://fixture.auth0.com", "https://127.0.0.1", "https://localhost",
+ "https://user:pass@fixture.auth0.com", "https://fixture.auth0.com/path",
+ } {
+ _, err := NewForTrustedInstance(origin)
+ assert.Error(t, err, origin)
}
}
diff --git a/internal/verifier/aws/aws_verifier.go b/internal/verifier/aws/aws_verifier.go
index f586aaa..3dcbc57 100644
--- a/internal/verifier/aws/aws_verifier.go
+++ b/internal/verifier/aws/aws_verifier.go
@@ -33,6 +33,8 @@ type Verifier struct {
client stsClient
}
+var _ verifier.RequestGatedVerifier = (*Verifier)(nil)
+
func init() {
verifier.Register(&Verifier{})
}
@@ -46,6 +48,27 @@ func (v *Verifier) Type() string {
// Raw contains the Access Key ID and RawV2 contains the Secret Access Key.
// Both are required for verification.
func (v *Verifier) Verify(ctx context.Context, raw detector.RawFinding) finding.VerificationResult {
+ return v.verify(ctx, raw, nil)
+}
+
+// VerificationRequestBudget declares the single STS probe this verifier can
+// issue for a complete key pair.
+func (v *Verifier) VerificationRequestBudget() int { return 1 }
+
+// VerifyWithRequestGate admits the STS request at its actual SDK send point.
+func (v *Verifier) VerifyWithRequestGate(
+ ctx context.Context,
+ raw detector.RawFinding,
+ gate verifier.RequestGate,
+) finding.VerificationResult {
+ return v.verify(ctx, raw, gate)
+}
+
+func (v *Verifier) verify(
+ ctx context.Context,
+ raw detector.RawFinding,
+ gate verifier.RequestGate,
+) finding.VerificationResult {
if len(raw.RawV2) == 0 {
slog.DebugContext(ctx, "aws verifier: secret access key not found, skipping verification")
return finding.VerificationResult{
@@ -65,6 +88,11 @@ func (v *Verifier) Verify(ctx context.Context, raw detector.RawFinding) finding.
),
})
}
+ if gate != nil {
+ if rejection := gate(); rejection != nil {
+ return *rejection
+ }
+ }
output, err := client.GetCallerIdentity(ctx, &sts.GetCallerIdentityInput{})
if err != nil {
diff --git a/internal/verifier/aws/aws_verifier_test.go b/internal/verifier/aws/aws_verifier_test.go
index 4056fb9..3e5dd7e 100644
--- a/internal/verifier/aws/aws_verifier_test.go
+++ b/internal/verifier/aws/aws_verifier_test.go
@@ -13,16 +13,34 @@ import (
"github.com/stretchr/testify/require"
"github.com/HodeTech/leakwatch/internal/detector"
+ awsdetector "github.com/HodeTech/leakwatch/internal/detector/aws"
+ "github.com/HodeTech/leakwatch/internal/detector/testutil"
"github.com/HodeTech/leakwatch/pkg/finding"
)
+func TestVerify_RealDetectorOutput_ReachesSTSContract(t *testing.T) {
+ fixture := testutil.RegisteredDetectorFixtures()[detectorID]
+ findings := testutil.ScanViaMatcher(&awsdetector.AccessKeyID{}, fixture.Input)
+ require.Len(t, findings, 1)
+ require.NotEmpty(t, findings[0].RawV2)
+
+ v := &Verifier{client: &mockSTSClient{output: &sts.GetCallerIdentityOutput{
+ Account: aws.String("123456789012"), Arn: aws.String("arn:aws:iam::123456789012:user/fixture"),
+ UserId: aws.String("AIDAFIXTURE"),
+ }}}
+ result := v.Verify(t.Context(), findings[0])
+ require.Equal(t, finding.StatusVerifiedActive, result.Status)
+}
+
// mockSTSClient implements stsClient for testing.
type mockSTSClient struct {
output *sts.GetCallerIdentityOutput
err error
+ calls int
}
func (m *mockSTSClient) GetCallerIdentity(_ context.Context, _ *sts.GetCallerIdentityInput, _ ...func(*sts.Options)) (*sts.GetCallerIdentityOutput, error) {
+ m.calls++
return m.output, m.err
}
@@ -42,6 +60,47 @@ func TestVerify_NoSecretKey_ReturnsUnverified(t *testing.T) {
assert.Equal(t, "secret access key not found", result.Message)
}
+func TestVerifyWithRequestGate_AdmitsOnlyActualSTSCall(t *testing.T) {
+ t.Run("missing companion key consumes no admission", func(t *testing.T) {
+ gateCalls := 0
+ result := (&Verifier{}).VerifyWithRequestGate(context.Background(), detector.RawFinding{
+ Raw: []byte("AKIAIOSFODNN7EXAMPLE"),
+ }, func() *finding.VerificationResult {
+ gateCalls++
+ return nil
+ })
+ assert.Equal(t, finding.StatusUnverified, result.Status)
+ assert.Zero(t, gateCalls)
+ })
+
+ t.Run("complete pair gets one admission", func(t *testing.T) {
+ mock := &mockSTSClient{output: &sts.GetCallerIdentityOutput{}}
+ v := &Verifier{client: mock}
+ gateCalls := 0
+ result := v.VerifyWithRequestGate(context.Background(), detector.RawFinding{
+ Raw: []byte("AKIAIOSFODNN7EXAMPLE"), RawV2: []byte("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"),
+ }, func() *finding.VerificationResult {
+ gateCalls++
+ return nil
+ })
+ assert.Equal(t, finding.StatusVerifiedActive, result.Status)
+ assert.Equal(t, 1, gateCalls)
+ assert.Equal(t, 1, mock.calls)
+ })
+
+ t.Run("rejection prevents SDK call", func(t *testing.T) {
+ mock := &mockSTSClient{output: &sts.GetCallerIdentityOutput{}}
+ v := &Verifier{client: mock}
+ result := v.VerifyWithRequestGate(context.Background(), detector.RawFinding{
+ Raw: []byte("AKIAIOSFODNN7EXAMPLE"), RawV2: []byte("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"),
+ }, func() *finding.VerificationResult {
+ return &finding.VerificationResult{Status: finding.StatusVerifyError, Message: "admission rejected"}
+ })
+ assert.Equal(t, "admission rejected", result.Message)
+ assert.Zero(t, mock.calls)
+ })
+}
+
func TestVerify_Type_ReturnsCorrectID(t *testing.T) {
v := &Verifier{}
assert.Equal(t, "aws-access-key-id", v.Type())
diff --git a/internal/verifier/bitbucket/bitbucket_shared_test.go b/internal/verifier/bitbucket/bitbucket_shared_test.go
new file mode 100644
index 0000000..5841211
--- /dev/null
+++ b/internal/verifier/bitbucket/bitbucket_shared_test.go
@@ -0,0 +1,16 @@
+package bitbucket
+
+import (
+ "net/http"
+ "testing"
+
+ "github.com/HodeTech/leakwatch/internal/detector"
+ "github.com/HodeTech/leakwatch/internal/verifier"
+ "github.com/HodeTech/leakwatch/internal/verifier/internal/vtest"
+)
+
+func TestVerify_SharedSafetySuite(t *testing.T) {
+ vtest.Run(t, vtest.Case{Name: "bitbucket", New: func(url string, client *http.Client) verifier.Verifier {
+ return &Verifier{apiURL: url, httpClient: client}
+ }, Raw: detector.RawFinding{DetectorID: detectorID, Raw: []byte("SyntheticAppPassword12"), ExtraData: map[string]string{"username": "fixture-user"}}})
+}
diff --git a/internal/verifier/bitbucket/bitbucket_verifier_test.go b/internal/verifier/bitbucket/bitbucket_verifier_test.go
index 95384d3..1b21f57 100644
--- a/internal/verifier/bitbucket/bitbucket_verifier_test.go
+++ b/internal/verifier/bitbucket/bitbucket_verifier_test.go
@@ -12,9 +12,31 @@ import (
"github.com/stretchr/testify/require"
"github.com/HodeTech/leakwatch/internal/detector"
+ bitbucketdetector "github.com/HodeTech/leakwatch/internal/detector/bitbucket"
+ "github.com/HodeTech/leakwatch/internal/detector/testutil"
"github.com/HodeTech/leakwatch/pkg/finding"
)
+func TestVerify_RealDetectorOutput_UsesPairedUsername(t *testing.T) {
+ fixture := testutil.RegisteredDetectorFixtures()[detectorID]
+ findings := testutil.ScanViaMatcher(&bitbucketdetector.Detector{}, fixture.Input)
+ require.Len(t, findings, 1)
+ require.Equal(t, "fixture-user", findings[0].ExtraData["username"])
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ username, password, ok := r.BasicAuth()
+ require.True(t, ok)
+ assert.Equal(t, findings[0].ExtraData["username"], username)
+ assert.Equal(t, string(findings[0].Raw), password)
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write([]byte(`{"display_name":"Fixture User"}`))
+ }))
+ defer server.Close()
+
+ result := (&Verifier{apiURL: server.URL, httpClient: server.Client()}).Verify(t.Context(), findings[0])
+ require.Equal(t, finding.StatusVerifiedActive, result.Status)
+}
+
func TestVerify_ValidPassword_ReturnsActive(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, "/2.0/user", r.URL.Path)
diff --git a/internal/verifier/cloudflare/cloudflare_shared_test.go b/internal/verifier/cloudflare/cloudflare_shared_test.go
new file mode 100644
index 0000000..302a266
--- /dev/null
+++ b/internal/verifier/cloudflare/cloudflare_shared_test.go
@@ -0,0 +1,16 @@
+package cloudflare
+
+import (
+ "net/http"
+ "testing"
+
+ "github.com/HodeTech/leakwatch/internal/detector"
+ "github.com/HodeTech/leakwatch/internal/verifier"
+ "github.com/HodeTech/leakwatch/internal/verifier/internal/vtest"
+)
+
+func TestVerify_SharedSafetySuite(t *testing.T) {
+ vtest.Run(t, vtest.Case{Name: "cloudflare", New: func(url string, client *http.Client) verifier.Verifier {
+ return &Verifier{apiURL: url, httpClient: client}
+ }, Raw: detector.RawFinding{DetectorID: detectorID, Raw: []byte("SyntheticCloudflareTokenFixture123456")}})
+}
diff --git a/internal/verifier/configuration.go b/internal/verifier/configuration.go
new file mode 100644
index 0000000..9c8b5a5
--- /dev/null
+++ b/internal/verifier/configuration.go
@@ -0,0 +1,73 @@
+package verifier
+
+import (
+ "fmt"
+ "reflect"
+)
+
+// TrustedInstanceConfigurer is implemented by verifiers whose live endpoint
+// must be selected from explicit operator context rather than scanned content.
+// Implementations must validate the supplied origin and return an independent
+// verifier value; they must not mutate the process-global registry instance.
+type TrustedInstanceConfigurer interface {
+ WithTrustedInstance(instanceURL string) (Verifier, error)
+}
+
+// ConfigureTrustedInstance returns a per-run verifier slice in which detectorID
+// is configured for an explicit operator-supplied instance. The input slice and
+// the process-global registry objects are never mutated.
+func ConfigureTrustedInstance(vs []Verifier, detectorID, instanceURL string) ([]Verifier, error) {
+ configured := append([]Verifier(nil), vs...)
+ if detectorID == "" {
+ return nil, fmt.Errorf("trusted instance verifier ID must not be empty")
+ }
+
+ target := -1
+ seen := make(map[string]struct{}, len(configured))
+ for i, candidate := range configured {
+ if isNilVerifier(candidate) {
+ return nil, fmt.Errorf("verifier list contains nil entry at index %d", i)
+ }
+ id := candidate.Type()
+ if id == "" {
+ return nil, fmt.Errorf("verifier list contains empty type at index %d", i)
+ }
+ if _, duplicate := seen[id]; duplicate {
+ return nil, fmt.Errorf("verifier list contains duplicate type %q", id)
+ }
+ seen[id] = struct{}{}
+ if id == detectorID {
+ target = i
+ }
+ }
+ if target < 0 {
+ return nil, fmt.Errorf("verifier %q is not registered", detectorID)
+ }
+
+ configurer, ok := configured[target].(TrustedInstanceConfigurer)
+ if !ok {
+ return nil, fmt.Errorf("verifier %q does not accept trusted instance configuration", detectorID)
+ }
+ replacement, err := configurer.WithTrustedInstance(instanceURL)
+ if err != nil {
+ return nil, err
+ }
+ if isNilVerifier(replacement) || replacement.Type() != detectorID {
+ return nil, fmt.Errorf("verifier %q returned an invalid configured replacement", detectorID)
+ }
+ configured[target] = replacement
+ return configured, nil
+}
+
+func isNilVerifier(v Verifier) bool {
+ if v == nil {
+ return true
+ }
+ value := reflect.ValueOf(v)
+ switch value.Kind() {
+ case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice:
+ return value.IsNil()
+ default:
+ return false
+ }
+}
diff --git a/internal/verifier/configuration_test.go b/internal/verifier/configuration_test.go
new file mode 100644
index 0000000..03faa1f
--- /dev/null
+++ b/internal/verifier/configuration_test.go
@@ -0,0 +1,77 @@
+package verifier
+
+import (
+ "context"
+ "fmt"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "github.com/HodeTech/leakwatch/internal/detector"
+ "github.com/HodeTech/leakwatch/pkg/finding"
+)
+
+type trustedInstanceMock struct {
+ id string
+ instance string
+ err error
+ replacement Verifier
+}
+
+func (m *trustedInstanceMock) Type() string { return m.id }
+
+func (m *trustedInstanceMock) Verify(context.Context, detector.RawFinding) finding.VerificationResult {
+ return finding.VerificationResult{}
+}
+
+func (m *trustedInstanceMock) WithTrustedInstance(instanceURL string) (Verifier, error) {
+ if m.err != nil {
+ return nil, m.err
+ }
+ if m.replacement != nil {
+ return m.replacement, nil
+ }
+ return &trustedInstanceMock{id: m.id, instance: instanceURL}, nil
+}
+
+func TestConfigureTrustedInstance_ReturnsIndependentReplacement(t *testing.T) {
+ original := &trustedInstanceMock{id: "contextual"}
+ other := newMock("other")
+ input := []Verifier{original, other}
+
+ got, err := ConfigureTrustedInstance(input, "contextual", "https://operator.example")
+ require.NoError(t, err)
+ require.Len(t, got, 2)
+ assert.Same(t, original, input[0])
+ assert.Same(t, other, input[1])
+ assert.NotSame(t, original, got[0])
+ assert.Equal(t, "https://operator.example", got[0].(*trustedInstanceMock).instance)
+ assert.Same(t, other, got[1])
+}
+
+func TestConfigureTrustedInstance_RejectsInvalidContracts(t *testing.T) {
+ var typedNil *trustedInstanceMock
+ tests := []struct {
+ name string
+ vs []Verifier
+ id string
+ }{
+ {name: "missing", vs: []Verifier{newMock("other")}, id: "missing"},
+ {name: "not configurable", vs: []Verifier{newMock("plain")}, id: "plain"},
+ {name: "configuration error", vs: []Verifier{&trustedInstanceMock{id: "bad", err: fmt.Errorf("invalid origin")}}, id: "bad"},
+ {name: "nil entry", vs: []Verifier{nil}, id: "nil"},
+ {name: "typed nil entry", vs: []Verifier{typedNil}, id: "nil"},
+ {name: "duplicate type", vs: []Verifier{newMock("duplicate"), newMock("duplicate")}, id: "duplicate"},
+ {name: "mismatched replacement", vs: []Verifier{&trustedInstanceMock{id: "contextual", replacement: newMock("other")}}, id: "contextual"},
+ {name: "typed nil replacement", vs: []Verifier{&trustedInstanceMock{id: "contextual", replacement: typedNil}}, id: "contextual"},
+ {name: "empty target ID", vs: []Verifier{newMock("other")}, id: ""},
+ }
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ got, err := ConfigureTrustedInstance(tc.vs, tc.id, "https://operator.example")
+ require.Error(t, err)
+ assert.Nil(t, got)
+ })
+ }
+}
diff --git a/internal/verifier/databricks/databricks_shared_test.go b/internal/verifier/databricks/databricks_shared_test.go
new file mode 100644
index 0000000..a274e33
--- /dev/null
+++ b/internal/verifier/databricks/databricks_shared_test.go
@@ -0,0 +1,16 @@
+package databricks
+
+import (
+ "net/http"
+ "testing"
+
+ "github.com/HodeTech/leakwatch/internal/detector"
+ "github.com/HodeTech/leakwatch/internal/verifier"
+ "github.com/HodeTech/leakwatch/internal/verifier/internal/vtest"
+)
+
+func TestVerify_SharedSafetySuite(t *testing.T) {
+ vtest.Run(t, vtest.Case{Name: "databricks", New: func(url string, client *http.Client) verifier.Verifier {
+ return &Verifier{apiURL: url, httpClient: client}
+ }, Raw: detector.RawFinding{DetectorID: detectorID, Raw: []byte("dapi0123456789abcdef0123456789abcdef")}})
+}
diff --git a/internal/verifier/databricks/databricks_verifier_test.go b/internal/verifier/databricks/databricks_verifier_test.go
index 7b11f33..052d258 100644
--- a/internal/verifier/databricks/databricks_verifier_test.go
+++ b/internal/verifier/databricks/databricks_verifier_test.go
@@ -2,17 +2,47 @@ package databricks
import (
"context"
+ "io"
"net/http"
"net/http/httptest"
+ "strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/HodeTech/leakwatch/internal/detector"
+ databricksdetector "github.com/HodeTech/leakwatch/internal/detector/databricks"
+ "github.com/HodeTech/leakwatch/internal/detector/testutil"
"github.com/HodeTech/leakwatch/pkg/finding"
)
+func TestVerify_RealDetectorOutput_UsesWorkspaceHost(t *testing.T) {
+ fixture := testutil.RegisteredDetectorFixtures()[detectorID]
+ findings := testutil.ScanViaMatcher(&databricksdetector.Detector{}, fixture.Input)
+ require.Len(t, findings, 1)
+ wantOrigin := findings[0].ExtraData["host"]
+ require.NotEmpty(t, wantOrigin)
+
+ var gotURL string
+ client := &http.Client{Transport: databricksRoundTrip(func(r *http.Request) (*http.Response, error) {
+ gotURL = r.URL.String()
+ return &http.Response{
+ StatusCode: http.StatusOK, Header: http.Header{"Content-Type": []string{"application/json"}},
+ Body: io.NopCloser(strings.NewReader(`{"userName":"fixture@example.com"}`)), Request: r,
+ }, nil
+ })}
+ result := (&Verifier{httpClient: client}).Verify(t.Context(), findings[0])
+ require.Equal(t, finding.StatusVerifiedActive, result.Status)
+ assert.Equal(t, wantOrigin+"/api/2.0/preview/scim/v2/Me", gotURL)
+}
+
+type databricksRoundTrip func(*http.Request) (*http.Response, error)
+
+func (f databricksRoundTrip) RoundTrip(request *http.Request) (*http.Response, error) {
+ return f(request)
+}
+
func TestVerify_ValidToken_ReturnsActive(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, "/api/2.0/preview/scim/v2/Me", r.URL.Path)
diff --git a/internal/verifier/datadog/datadog_verifier.go b/internal/verifier/datadog/datadog_verifier.go
index 7d498d0..617a669 100644
--- a/internal/verifier/datadog/datadog_verifier.go
+++ b/internal/verifier/datadog/datadog_verifier.go
@@ -5,8 +5,11 @@ package datadog
import (
"context"
"encoding/json"
+ "fmt"
"io"
"net/http"
+ "net/url"
+ "strings"
"github.com/HodeTech/leakwatch/internal/detector"
"github.com/HodeTech/leakwatch/internal/verifier"
@@ -16,18 +19,46 @@ import (
const detectorID = "datadog-api-key"
-// defaultAPIURL is the base URL for the Datadog API.
-const defaultAPIURL = "https://api.datadoghq.com"
-
// Verifier checks whether a Datadog API key is active by calling the
// Datadog validation API. It NEVER logs or persists raw key values.
type Verifier struct {
- // apiURL overrides the Datadog API base URL (for testing).
+ // apiURL is reserved for a validated, trusted Datadog site origin. The
+ // production registry value stays empty and each scan may install an
+ // independently configured copy; tests inject a local server.
apiURL string
// httpClient overrides the default HTTP client (for testing).
httpClient *http.Client
}
+var trustedDatadogAPIHosts = map[string]struct{}{
+ "api.datadoghq.com": {}, "api.us3.datadoghq.com": {}, "api.us5.datadoghq.com": {},
+ "api.datadoghq.eu": {}, "api.ap1.datadoghq.com": {}, "api.ap2.datadoghq.com": {},
+ "api.datad0g.com": {}, "api.ddog-gov.com": {}, "api.us2.ddog-gov.com": {},
+}
+
+// NewForTrustedInstance accepts only canonical official Datadog API site
+// origins. Datadog keys are site-bound; arbitrary repository or custom hosts
+// are never valid routing authorities.
+func NewForTrustedInstance(instanceURL string) (*Verifier, error) {
+ normalized, err := verifier.NormalizeTrustedHTTPSOrigin(instanceURL)
+ if err != nil {
+ return nil, err
+ }
+ u, err := url.Parse(normalized)
+ if err != nil || u.Port() != "" {
+ return nil, fmt.Errorf("invalid Datadog API origin")
+ }
+ if _, ok := trustedDatadogAPIHosts[u.Hostname()]; !ok {
+ return nil, fmt.Errorf("invalid Datadog API origin: host is not an official Datadog site")
+ }
+ return &Verifier{apiURL: normalized}, nil
+}
+
+// WithTrustedInstance implements verifier.TrustedInstanceConfigurer.
+func (*Verifier) WithTrustedInstance(instanceURL string) (verifier.Verifier, error) {
+ return NewForTrustedInstance(instanceURL)
+}
+
func init() {
verifier.Register(&Verifier{})
}
@@ -42,7 +73,16 @@ func (v *Verifier) Type() string {
// 403 (not 401) for a rejected key.
func (v *Verifier) Verify(ctx context.Context, raw detector.RawFinding) finding.VerificationResult {
token := string(raw.Raw)
- apiURL := httpx.BaseURL(v.apiURL, defaultAPIURL)
+ if token == "" {
+ return finding.VerificationResult{Status: finding.StatusUnverified, Message: "empty token"}
+ }
+ if v.apiURL == "" {
+ return finding.VerificationResult{
+ Status: finding.StatusUnverified,
+ Message: "trusted Datadog site is not configured",
+ }
+ }
+ apiURL := v.apiURL
return httpx.VerifyToken(ctx, v.httpClient, token, httpx.TokenSpec{
Name: "datadog",
@@ -50,22 +90,45 @@ func (v *Verifier) Verify(ctx context.Context, raw detector.RawFinding) finding.
URL: apiURL + "/api/v1/validate",
Header: map[string]string{"DD-API-KEY": token},
},
- InactiveStatuses: []int{http.StatusForbidden},
- ActiveMessage: "Datadog API key is active",
- InactiveMessage: "Datadog API key is invalid or revoked",
- Decode: decodeValidate,
+ InactiveStatuses: []int{http.StatusForbidden},
+ ActiveMessage: "Datadog API key is active",
+ InactiveMessage: "Datadog API key is invalid or revoked",
+ Decode: decodeValidate,
+ DecodeInactive: decodeInactiveResponse,
+ RequireCompleteBody: true,
+ RequireJSONContentType: true,
})
}
+func decodeInactiveResponse(body io.Reader) error {
+ var response struct {
+ Errors []string `json:"errors"`
+ }
+ decoder := json.NewDecoder(body)
+ if err := decoder.Decode(&response); err != nil {
+ return err
+ }
+ if len(response.Errors) != 1 || strings.TrimSpace(response.Errors[0]) != "Forbidden" {
+ return fmt.Errorf("unexpected Datadog authentication error")
+ }
+ if decoder.Decode(&struct{}{}) != io.EOF {
+ return fmt.Errorf("unexpected trailing Datadog response data")
+ }
+ return nil
+}
+
// decodeValidate downgrades a 200 response to inactive when valid=false.
func decodeValidate(body io.Reader) (map[string]string, string, error) {
var resp struct {
- Valid bool `json:"valid"`
+ Valid *bool `json:"valid"`
}
if err := json.NewDecoder(body).Decode(&resp); err != nil {
return nil, "", err
}
- if !resp.Valid {
+ if resp.Valid == nil {
+ return nil, "", fmt.Errorf("missing Datadog valid field")
+ }
+ if !*resp.Valid {
return nil, "Datadog API key is invalid", nil
}
return nil, "", nil
diff --git a/internal/verifier/datadog/datadog_verifier_test.go b/internal/verifier/datadog/datadog_verifier_test.go
index 8fade22..bffad8f 100644
--- a/internal/verifier/datadog/datadog_verifier_test.go
+++ b/internal/verifier/datadog/datadog_verifier_test.go
@@ -66,8 +66,36 @@ func TestVerify_InvalidKey_200False_ReturnsInactive(t *testing.T) {
assert.Equal(t, "Datadog API key is invalid", result.Message)
}
+func TestVerify_MalformedSuccessResponse_ReturnsVerifyError(t *testing.T) {
+ tests := []struct {
+ name string
+ contentType string
+ body string
+ }{
+ {name: "missing valid", contentType: "application/json", body: `{}`},
+ {name: "null body", contentType: "application/json", body: `null`},
+ {name: "wrong content type", contentType: "text/plain", body: `{"valid":true}`},
+ }
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", tc.contentType)
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write([]byte(tc.body))
+ }))
+ defer server.Close()
+
+ result := (&Verifier{apiURL: server.URL, httpClient: server.Client()}).Verify(
+ context.Background(), detector.RawFinding{Raw: []byte("synthetic-datadog-key")},
+ )
+ assert.Equal(t, finding.StatusVerifyError, result.Status)
+ })
+ }
+}
+
func TestVerify_InvalidKey_403_ReturnsInactive(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusForbidden)
_, _ = w.Write([]byte(`{"errors":["Forbidden"]}`))
}))
@@ -90,6 +118,25 @@ func TestVerify_InvalidKey_403_ReturnsInactive(t *testing.T) {
assert.Equal(t, "Datadog API key is invalid or revoked", result.Message)
}
+func TestVerify_InactiveResponseMustBeDefinitiveJSON(t *testing.T) {
+ tests := []struct{ contentType, body string }{
+ {contentType: "text/html", body: `WAF block`},
+ {contentType: "application/json", body: `{"errors":["challenge"]}`},
+ }
+ for _, tc := range tests {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", tc.contentType)
+ w.WriteHeader(http.StatusForbidden)
+ _, _ = w.Write([]byte(tc.body))
+ }))
+ result := (&Verifier{apiURL: server.URL, httpClient: server.Client()}).Verify(
+ context.Background(), detector.RawFinding{Raw: []byte("synthetic-datadog-key")},
+ )
+ server.Close()
+ assert.Equal(t, finding.StatusVerifyError, result.Status)
+ }
+}
+
func TestVerify_ServerError_ReturnsError(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
@@ -133,3 +180,29 @@ func TestVerify_EmptyToken_ReturnsUnverified(t *testing.T) {
assert.Equal(t, finding.StatusUnverified, result.Status)
assert.Equal(t, "empty token", result.Message)
}
+
+func TestVerify_WithoutTrustedSite_MakesNoRequest(t *testing.T) {
+ v := &Verifier{}
+ result := v.Verify(context.Background(), detector.RawFinding{Raw: []byte("synthetic-datadog-key")})
+
+ assert.Equal(t, finding.StatusUnverified, result.Status)
+ assert.Equal(t, "trusted Datadog site is not configured", result.Message)
+}
+
+func TestNewForTrustedInstance_AcceptsOnlyOfficialDatadogSites(t *testing.T) {
+ for _, origin := range []string{
+ "https://api.datadoghq.com", "https://api.datadoghq.eu", "https://api.ap2.datadoghq.com",
+ "https://api.ddog-gov.com", "https://api.us2.ddog-gov.com",
+ } {
+ configured, err := NewForTrustedInstance(origin)
+ require.NoError(t, err, origin)
+ assert.Equal(t, origin, configured.apiURL)
+ }
+ for _, origin := range []string{
+ "https://datadoghq.com", "https://api.datadoghq.com.attacker.example",
+ "https://api.datadoghq.com:443", "http://api.datadoghq.com",
+ } {
+ _, err := NewForTrustedInstance(origin)
+ assert.Error(t, err, origin)
+ }
+}
diff --git a/internal/verifier/deepseek/deepseek_shared_test.go b/internal/verifier/deepseek/deepseek_shared_test.go
new file mode 100644
index 0000000..ac85b2c
--- /dev/null
+++ b/internal/verifier/deepseek/deepseek_shared_test.go
@@ -0,0 +1,16 @@
+package deepseek
+
+import (
+ "net/http"
+ "testing"
+
+ "github.com/HodeTech/leakwatch/internal/detector"
+ "github.com/HodeTech/leakwatch/internal/verifier"
+ "github.com/HodeTech/leakwatch/internal/verifier/internal/vtest"
+)
+
+func TestVerify_SharedSafetySuite(t *testing.T) {
+ vtest.Run(t, vtest.Case{Name: "deepseek", New: func(url string, client *http.Client) verifier.Verifier {
+ return &Verifier{apiURL: url, httpClient: client}
+ }, Raw: detector.RawFinding{DetectorID: detectorID, Raw: []byte("sk-0123456789abcdef0123456789abcdef")}})
+}
diff --git a/internal/verifier/digitalocean/digitalocean_shared_test.go b/internal/verifier/digitalocean/digitalocean_shared_test.go
new file mode 100644
index 0000000..06bd44a
--- /dev/null
+++ b/internal/verifier/digitalocean/digitalocean_shared_test.go
@@ -0,0 +1,16 @@
+package digitalocean
+
+import (
+ "net/http"
+ "testing"
+
+ "github.com/HodeTech/leakwatch/internal/detector"
+ "github.com/HodeTech/leakwatch/internal/verifier"
+ "github.com/HodeTech/leakwatch/internal/verifier/internal/vtest"
+)
+
+func TestVerify_SharedSafetySuite(t *testing.T) {
+ vtest.Run(t, vtest.Case{Name: "digitalocean", New: func(url string, client *http.Client) verifier.Verifier {
+ return &Verifier{apiURL: url, httpClient: client}
+ }, Raw: detector.RawFinding{DetectorID: detectorID, Raw: []byte("dop_v1_0123456789abcdef0123456789abcdef")}})
+}
diff --git a/internal/verifier/discord/discord_shared_test.go b/internal/verifier/discord/discord_shared_test.go
new file mode 100644
index 0000000..f9259a4
--- /dev/null
+++ b/internal/verifier/discord/discord_shared_test.go
@@ -0,0 +1,16 @@
+package discord
+
+import (
+ "net/http"
+ "testing"
+
+ "github.com/HodeTech/leakwatch/internal/detector"
+ "github.com/HodeTech/leakwatch/internal/verifier"
+ "github.com/HodeTech/leakwatch/internal/verifier/internal/vtest"
+)
+
+func TestVerify_SharedSafetySuite(t *testing.T) {
+ vtest.Run(t, vtest.Case{Name: "discord", New: func(url string, client *http.Client) verifier.Verifier {
+ return &Verifier{apiURL: url, httpClient: client}
+ }, Raw: detector.RawFinding{DetectorID: detectorID, Raw: []byte("MSyntheticDiscord.Fixtur.SyntheticCredentialValue")}})
+}
diff --git a/internal/verifier/doppler/doppler_shared_test.go b/internal/verifier/doppler/doppler_shared_test.go
new file mode 100644
index 0000000..696c5f9
--- /dev/null
+++ b/internal/verifier/doppler/doppler_shared_test.go
@@ -0,0 +1,16 @@
+package doppler
+
+import (
+ "net/http"
+ "testing"
+
+ "github.com/HodeTech/leakwatch/internal/detector"
+ "github.com/HodeTech/leakwatch/internal/verifier"
+ "github.com/HodeTech/leakwatch/internal/verifier/internal/vtest"
+)
+
+func TestVerify_SharedSafetySuite(t *testing.T) {
+ vtest.Run(t, vtest.Case{Name: "doppler", New: func(url string, client *http.Client) verifier.Verifier {
+ return &Verifier{apiURL: url, httpClient: client}
+ }, Raw: detector.RawFinding{DetectorID: detectorID, Raw: []byte("dp.st.SyntheticDopplerCredentialFixture123456")}, SkipMalformed: true})
+}
diff --git a/internal/verifier/engine.go b/internal/verifier/engine.go
index a14d582..03d2772 100644
--- a/internal/verifier/engine.go
+++ b/internal/verifier/engine.go
@@ -11,10 +11,12 @@ import (
"golang.org/x/time/rate"
"github.com/HodeTech/leakwatch/internal/detector"
+ "github.com/HodeTech/leakwatch/internal/verifier/internal/httpx"
"github.com/HodeTech/leakwatch/pkg/finding"
)
-// DefaultTimeout is the default per-request verification timeout.
+// DefaultTimeout is the default timeout for one finding's complete verification
+// operation, including any bounded provider-region fallback requests.
const DefaultTimeout = 10 * time.Second
// DefaultConcurrency is the default number of concurrent verification workers.
@@ -23,12 +25,17 @@ const DefaultConcurrency = 4
// DefaultRateLimit is the default maximum verification requests per second.
const DefaultRateLimit = 10.0
+// maxVerificationRequestBudget prevents a buggy verifier from reserving an
+// unbounded number of limiter tokens for one finding.
+const maxVerificationRequestBudget = 8
+
// Config holds the verification engine configuration.
type Config struct {
// Enabled controls whether verification is performed at all.
Enabled bool
- // Timeout is the maximum duration for a single verification request.
+ // Timeout is the maximum duration for one finding's complete verification
+ // operation, including bounded provider-region fallback requests.
Timeout time.Duration
// Concurrency is the number of concurrent verification workers.
@@ -36,10 +43,10 @@ type Config struct {
// RateLimit is the maximum verification requests per second.
//
- // The rate applies both as a per-detector budget (each distinct detector ID
- // gets its own limiter of this rate) and as a global ceiling shared across
- // all providers, so one high-volume detector type cannot starve the
- // verification of unrelated findings in the same run.
+ // The same rate applies both as a per-detector ceiling (each distinct
+ // detector ID gets its own limiter) and as a global ceiling shared across all
+ // providers. The global bucket bounds aggregate traffic; it does not promise
+ // fair scheduling between providers.
RateLimit float64
}
@@ -55,7 +62,7 @@ func DefaultConfig() Config {
// Engine orchestrates concurrent secret verification with rate limiting.
// It maps findings to the appropriate verifier by detector ID and applies
-// per-request timeouts and global rate limiting.
+// per-finding operation timeouts and global rate limiting.
type Engine struct {
verifiers map[string]Verifier
@@ -90,17 +97,26 @@ func NewEngine(cfg Config, vs []Verifier) *Engine {
}
vMap := make(map[string]Verifier, len(vs))
- for _, v := range vs {
- if _, exists := vMap[v.Type()]; exists {
+ for i, v := range vs {
+ if isNilVerifier(v) {
+ slog.Error("nil verifier ignored while building verification engine", "index", i)
+ continue
+ }
+ id, ok := safeVerifierType(v)
+ if !ok || id == "" {
+ slog.Error("invalid verifier ignored while building verification engine", "index", i)
+ continue
+ }
+ if _, exists := vMap[id]; exists {
// Unlike registry.Register (which panics), NewEngine tolerates a
// duplicate Type() but must not lose it silently: warn so the
// capability loss is observable.
slog.Warn(
"duplicate verifier type; later verifier overwrites earlier",
- "verifier_type", v.Type(),
+ "verifier_type", id,
)
}
- vMap[v.Type()] = v
+ vMap[id] = v
}
burst := int(cfg.RateLimit)
@@ -120,6 +136,18 @@ func NewEngine(cfg Config, vs []Verifier) *Engine {
}
}
+func safeVerifierType(v Verifier) (id string, ok bool) {
+ defer func() {
+ if recover() != nil {
+ // Type implementations are not allowed to crash engine construction;
+ // the recovered value may contain sensitive data and is discarded.
+ id = ""
+ ok = false
+ }
+ }()
+ return v.Type(), true
+}
+
// limiterFor returns the per-detector rate limiter for detectorID, creating it
// on first use. Each detector gets an independent budget at the configured rate.
func (e *Engine) limiterFor(detectorID string) *rate.Limiter {
@@ -214,14 +242,8 @@ func (e *Engine) verifySingle(ctx context.Context, pair VerifyPair) finding.Find
return f
}
- // Apply rate limiting: the per-detector limiter first (independent budget
- // per provider), then the global ceiling shared across all providers.
- if res := e.waitRateLimit(ctx, pair.Raw.DetectorID); res != nil {
- f.Verification = *res
- return f
- }
-
- // Apply per-request timeout.
+ // Apply one timeout to the complete provider verification operation,
+ // including rate-limiter admission and any bounded regional fallback.
verifyCtx, cancel := context.WithTimeout(ctx, e.timeout)
defer cancel()
@@ -231,7 +253,32 @@ func (e *Engine) verifySingle(ctx context.Context, pair VerifyPair) finding.Find
"redacted", pair.Raw.Redacted,
)
- result := e.safeVerify(verifyCtx, v, pair.Raw)
+ var result finding.VerificationResult
+ if gated, ok := v.(RequestGatedVerifier); ok {
+ requestBudget, err := safeVerificationRequestBudget(gated)
+ if err != nil {
+ f.Verification = finding.VerificationResult{
+ Status: finding.StatusVerifyError,
+ Message: "verifier declared an invalid request budget",
+ }
+ return f
+ }
+ result = e.safeVerifyWithRequestGate(
+ verifyCtx,
+ gated,
+ pair.Raw,
+ e.boundedRequestGate(verifyCtx, pair.Raw.DetectorID, requestBudget),
+ )
+ } else {
+ // Standard httpx verifiers receive an admission gate that runs at each
+ // actual client.Do send point. Format-only, empty-input, and missing-
+ // context paths therefore consume no limiter token, while a bounded 429
+ // replay receives a fresh admission automatically.
+ requestCtx := httpx.WithRequestGate(verifyCtx, func() *finding.VerificationResult {
+ return e.waitRateLimit(verifyCtx, pair.Raw.DetectorID)
+ })
+ result = e.safeVerify(requestCtx, v, pair.Raw)
+ }
f.Verification = result
slog.Debug(
@@ -247,21 +294,116 @@ func (e *Engine) verifySingle(ctx context.Context, pair VerifyPair) finding.Find
// admit a request, honoring ctx cancellation. It returns a non-nil result
// describing a StatusVerifyError when the context is cancelled while waiting,
// and nil when the request may proceed.
+
func (e *Engine) waitRateLimit(ctx context.Context, detectorID string) *finding.VerificationResult {
- for _, l := range []*rate.Limiter{e.limiterFor(detectorID), e.rateLimiter} {
- if err := l.Wait(ctx); err != nil {
- slog.Warn(
- "rate limiter wait cancelled",
- "detector_id", detectorID,
- "error", err,
- )
+ if err := ctx.Err(); err != nil {
+ return rateLimitFailure(detectorID, err)
+ }
+
+ now := time.Now()
+ reservations := []*rate.Reservation{
+ e.limiterFor(detectorID).ReserveN(now, 1),
+ e.rateLimiter.ReserveN(now, 1),
+ }
+ for _, reservation := range reservations {
+ if !reservation.OK() {
+ cancelRateReservations(reservations, now)
+ return rateLimitFailure(detectorID, fmt.Errorf("rate limiter reservation rejected"))
+ }
+ }
+
+ delay := time.Duration(0)
+ for _, reservation := range reservations {
+ if candidate := reservation.DelayFrom(now); candidate > delay {
+ delay = candidate
+ }
+ }
+ if deadline, ok := ctx.Deadline(); ok && now.Add(delay).After(deadline) {
+ cancelRateReservations(reservations, now)
+ return rateLimitFailure(detectorID, context.DeadlineExceeded)
+ }
+ if delay <= 0 {
+ if err := ctx.Err(); err != nil {
+ cancelRateReservations(reservations, now)
+ return rateLimitFailure(detectorID, err)
+ }
+ return nil
+ }
+
+ timer := time.NewTimer(delay)
+ defer timer.Stop()
+ select {
+ case <-timer.C:
+ return nil
+ case <-ctx.Done():
+ cancelRateReservations(reservations, time.Now())
+ return rateLimitFailure(detectorID, ctx.Err())
+ }
+}
+
+func cancelRateReservations(reservations []*rate.Reservation, at time.Time) {
+ for _, reservation := range reservations {
+ if reservation != nil && reservation.OK() {
+ reservation.CancelAt(at)
+ }
+ }
+}
+
+func rateLimitFailure(detectorID string, err error) *finding.VerificationResult {
+ slog.Warn(
+ "rate limiter wait cancelled",
+ "detector_id", detectorID,
+ "error", err,
+ )
+ return &finding.VerificationResult{
+ Status: finding.StatusVerifyError,
+ Message: fmt.Sprintf("rate limiter cancelled: %v", err),
+ }
+}
+
+func verificationRequestBudget(v RequestGatedVerifier) (int, error) {
+ budget := v.VerificationRequestBudget()
+ if budget < 1 {
+ return 0, fmt.Errorf("verification request budget %d must be positive", budget)
+ }
+ if budget > maxVerificationRequestBudget {
+ return 0, fmt.Errorf("verification request budget %d exceeds maximum %d", budget, maxVerificationRequestBudget)
+ }
+ return budget, nil
+}
+
+func safeVerificationRequestBudget(v RequestGatedVerifier) (budget int, err error) {
+ defer func() {
+ if recover() != nil {
+ // The recovered value can contain a raw credential. Do not format,
+ // return, or log it.
+ budget = 0
+ err = fmt.Errorf("verifier panicked while declaring its request budget")
+ }
+ }()
+ return verificationRequestBudget(v)
+}
+
+// boundedRequestGate returns a concurrency-safe admission gate tied to one
+// finding. Each successful invocation consumes one unit of the verifier's
+// declared request budget and one token from both the provider and global
+// limiters at the actual request send point.
+func (e *Engine) boundedRequestGate(ctx context.Context, detectorID string, budget int) RequestGate {
+ var mu sync.Mutex
+ used := 0
+ return func() *finding.VerificationResult {
+ mu.Lock()
+ if used >= budget {
+ mu.Unlock()
return &finding.VerificationResult{
Status: finding.StatusVerifyError,
- Message: fmt.Sprintf("rate limiter cancelled: %v", err),
+ Message: "verifier exceeded its declared request budget",
}
}
+ used++
+ mu.Unlock()
+ return e.waitRateLimit(ctx, detectorID)
}
- return nil
}
// safeVerify invokes v.Verify with panic recovery so that a single misbehaving
@@ -270,13 +412,35 @@ func (e *Engine) waitRateLimit(ctx context.Context, detectorID string) *finding.
// findings. A recovered panic is converted to a StatusVerifyError result and
// logged with a stack trace; the raw secret is never logged.
func (e *Engine) safeVerify(ctx context.Context, v Verifier, raw detector.RawFinding) (result finding.VerificationResult) {
+ return e.safeVerification(ctx, raw, func() finding.VerificationResult {
+ return v.Verify(ctx, raw)
+ })
+}
+
+func (e *Engine) safeVerifyWithRequestGate(
+ ctx context.Context,
+ v RequestGatedVerifier,
+ raw detector.RawFinding,
+ gate RequestGate,
+) finding.VerificationResult {
+ return e.safeVerification(ctx, raw, func() finding.VerificationResult {
+ return v.VerifyWithRequestGate(ctx, raw, gate)
+ })
+}
+
+func (e *Engine) safeVerification(
+ ctx context.Context,
+ raw detector.RawFinding,
+ verify func() finding.VerificationResult,
+) (result finding.VerificationResult) {
defer func() {
- if r := recover(); r != nil {
- slog.Error(
+ if recover() != nil {
+ // Never format or log the recovered payload: a defective verifier can
+ // panic with the raw credential itself. debug.Stack contains program
+ // counters/function frames, not panic values or local variables.
+ slog.ErrorContext(ctx,
"verifier panicked; recovered to protect the scan",
- "verifier_type", v.Type(),
"detector_id", raw.DetectorID,
- "panic", fmt.Sprintf("%v", r),
"stack", string(debug.Stack()),
)
result = finding.VerificationResult{
@@ -285,5 +449,5 @@ func (e *Engine) safeVerify(ctx context.Context, v Verifier, raw detector.RawFin
}
}
}()
- return v.Verify(ctx, raw)
+ return verify()
}
diff --git a/internal/verifier/engine_test.go b/internal/verifier/engine_test.go
index 41b9706..16405dd 100644
--- a/internal/verifier/engine_test.go
+++ b/internal/verifier/engine_test.go
@@ -1,12 +1,20 @@
package verifier
import (
+ "bytes"
"context"
+ "log/slog"
+ "net/http"
+ "net/http/httptest"
+ "sync"
"sync/atomic"
"testing"
"time"
+ "golang.org/x/time/rate"
+
"github.com/HodeTech/leakwatch/internal/detector"
+ "github.com/HodeTech/leakwatch/internal/verifier/internal/httpx"
"github.com/HodeTech/leakwatch/pkg/finding"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -26,6 +34,22 @@ type testVerifier struct {
maxInFlight atomic.Int64
}
+type httpRetryTestVerifier struct {
+ detectorID string
+ client *http.Client
+ url string
+}
+
+func (v *httpRetryTestVerifier) Type() string { return v.detectorID }
+func (v *httpRetryTestVerifier) Verify(ctx context.Context, raw detector.RawFinding) finding.VerificationResult {
+ token := string(raw.Raw)
+ return httpx.VerifyToken(ctx, v.client, token, httpx.TokenSpec{
+ Name: "engine-retry-test",
+ Request: httpx.Request{URL: v.url, Header: map[string]string{"Authorization": "Bearer " + token}},
+ ActiveMessage: "active after bounded retry",
+ })
+}
+
func (v *testVerifier) Type() string { return v.detectorID }
func (v *testVerifier) Verify(ctx context.Context, _ detector.RawFinding) finding.VerificationResult {
@@ -53,17 +77,71 @@ func (v *testVerifier) Verify(ctx context.Context, _ detector.RawFinding) findin
return v.result
}
+// gatedTestVerifier models a verifier that may make several outbound attempts.
+// It calls the engine gate immediately before each simulated request and records
+// admission timestamps so request pacing can be asserted without real network
+// calls.
+type gatedTestVerifier struct {
+ *testVerifier
+ requestBudget int
+ budgetPanic string
+ attempts int
+ admittedMu sync.Mutex
+ admittedAt []time.Time
+}
+
+func (v *gatedTestVerifier) VerificationRequestBudget() int {
+ if v.budgetPanic != "" {
+ panic(v.budgetPanic)
+ }
+ return v.requestBudget
+}
+
+func (v *gatedTestVerifier) VerifyWithRequestGate(
+ ctx context.Context,
+ raw detector.RawFinding,
+ gate RequestGate,
+) finding.VerificationResult {
+ for attempt := 0; attempt < v.attempts; attempt++ {
+ if rejection := gate(); rejection != nil {
+ return *rejection
+ }
+ v.admittedMu.Lock()
+ v.admittedAt = append(v.admittedAt, time.Now())
+ v.admittedMu.Unlock()
+ }
+ return v.Verify(ctx, raw)
+}
+
+func (v *gatedTestVerifier) admissionTimes() []time.Time {
+ v.admittedMu.Lock()
+ defer v.admittedMu.Unlock()
+ return append([]time.Time(nil), v.admittedAt...)
+}
+
// panicVerifier is a mock verifier whose Verify always panics, used to prove the
// engine recovers instead of crashing the whole scan.
type panicVerifier struct {
detectorID string
callCount atomic.Int64
+ panicRaw bool
+}
+
+type panicTypeVerifier struct{ payload string }
+
+func (v *panicTypeVerifier) Type() string { panic(v.payload) }
+
+func (v *panicTypeVerifier) Verify(context.Context, detector.RawFinding) finding.VerificationResult {
+ return finding.VerificationResult{Status: finding.StatusVerifiedActive}
}
func (v *panicVerifier) Type() string { return v.detectorID }
-func (v *panicVerifier) Verify(context.Context, detector.RawFinding) finding.VerificationResult {
+func (v *panicVerifier) Verify(_ context.Context, raw detector.RawFinding) finding.VerificationResult {
v.callCount.Add(1)
+ if v.panicRaw {
+ panic(string(raw.Raw))
+ }
panic("boom: simulated verifier defect")
}
@@ -125,6 +203,204 @@ func TestVerifyAll_MatchingVerifier_UpdatesFinding(t *testing.T) {
assert.Equal(t, int64(1), v.callCount.Load())
}
+func TestVerifyAll_NoRequestVerifierConsumesNoLimiterCapacity(t *testing.T) {
+ const detectorID = "format-only-provider"
+ v := &testVerifier{
+ detectorID: detectorID,
+ result: finding.VerificationResult{
+ Status: finding.StatusUnverified,
+ Message: "format-only verification",
+ },
+ }
+ engine := NewEngine(Config{Enabled: true, Timeout: time.Second, Concurrency: 1, RateLimit: 1}, []Verifier{v})
+
+ results := engine.VerifyAll(context.Background(), []VerifyPair{makePair(detectorID, "format****only")})
+ require.Len(t, results, 1)
+ assert.Equal(t, finding.StatusUnverified, results[0].Verification.Status)
+ assert.Equal(t, int64(1), v.callCount.Load())
+ assert.True(t, engine.rateLimiter.Allow(), "a verifier that sends no request must not consume the global token")
+ _, created := engine.perDetector[detectorID]
+ assert.False(t, created, "a verifier that sends no request must not create or consume a provider bucket")
+}
+
+func TestVerifyAll_GatedVerifierDoesNotReserveUnusedFallbackToken(t *testing.T) {
+ base := &testVerifier{
+ detectorID: "regional-provider",
+ result: finding.VerificationResult{Status: finding.StatusVerifiedActive},
+ }
+ v := &gatedTestVerifier{testVerifier: base, requestBudget: 2, attempts: 1}
+ engine := NewEngine(Config{Enabled: true, Timeout: 250 * time.Millisecond, Concurrency: 1, RateLimit: 1}, []Verifier{v})
+
+ results := engine.VerifyAll(context.Background(), []VerifyPair{makePair("regional-provider", "regional****key")})
+ require.Len(t, results, 1)
+ assert.Equal(t, finding.StatusVerifiedActive, results[0].Verification.Status)
+ assert.Equal(t, int64(1), base.callCount.Load())
+ assert.Len(t, v.admissionTimes(), 1, "unused fallback must consume no limiter token")
+}
+
+func TestVerifyAll_GatedFallbackRequestsArePacedAtSendTime(t *testing.T) {
+ base := &testVerifier{
+ detectorID: "regional-provider",
+ result: finding.VerificationResult{Status: finding.StatusVerifiedActive},
+ }
+ v := &gatedTestVerifier{testVerifier: base, requestBudget: 2, attempts: 2}
+ engine := NewEngine(Config{Enabled: true, Timeout: 2 * time.Second, Concurrency: 1, RateLimit: 1}, []Verifier{v})
+
+ results := engine.VerifyAll(context.Background(), []VerifyPair{makePair("regional-provider", "regional****key")})
+ require.Len(t, results, 1)
+ assert.Equal(t, finding.StatusVerifiedActive, results[0].Verification.Status)
+ times := v.admissionTimes()
+ require.Len(t, times, 2)
+ assert.GreaterOrEqual(t, times[1].Sub(times[0]), 900*time.Millisecond,
+ "fallback request must pass the limiter at its actual send point")
+}
+
+func TestVerifyAll_TimeoutIncludesFallbackRateLimitAdmission(t *testing.T) {
+ base := &testVerifier{
+ detectorID: "regional-provider",
+ result: finding.VerificationResult{Status: finding.StatusVerifiedActive},
+ }
+ v := &gatedTestVerifier{testVerifier: base, requestBudget: 2, attempts: 2}
+ engine := NewEngine(Config{Enabled: true, Timeout: 50 * time.Millisecond, Concurrency: 1, RateLimit: 1}, []Verifier{v})
+ started := time.Now()
+
+ results := engine.VerifyAll(context.Background(), []VerifyPair{makePair("regional-provider", "regional****key")})
+ require.Len(t, results, 1)
+ assert.Equal(t, finding.StatusVerifyError, results[0].Verification.Status)
+ assert.Less(t, time.Since(started), 500*time.Millisecond,
+ "configured timeout must bound limiter admission and provider fallback together")
+ assert.Zero(t, base.callCount.Load(), "provider result must not run after fallback admission fails")
+}
+
+func TestVerificationRequestBudget_IsBounded(t *testing.T) {
+ tests := []struct {
+ name string
+ budget int
+ want int
+ wantErr bool
+ }{
+ {name: "zero fails closed", budget: 0, wantErr: true},
+ {name: "negative fails closed", budget: -1, wantErr: true},
+ {name: "declared budget", budget: 2, want: 2},
+ {name: "excessive fails closed", budget: 1000, wantErr: true},
+ }
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ got, err := verificationRequestBudget(&gatedTestVerifier{
+ testVerifier: &testVerifier{detectorID: "regional-provider"},
+ requestBudget: tc.budget,
+ })
+ if tc.wantErr {
+ require.Error(t, err)
+ return
+ }
+ require.NoError(t, err)
+ assert.Equal(t, tc.want, got)
+ })
+ }
+}
+
+func TestVerifyAll_ExcessiveRequestBudgetFailsClosed(t *testing.T) {
+ base := &testVerifier{detectorID: "unsafe-regional-provider", result: finding.VerificationResult{Status: finding.StatusVerifiedActive}}
+ v := &gatedTestVerifier{testVerifier: base, requestBudget: maxVerificationRequestBudget + 1, attempts: 1}
+ engine := NewEngine(Config{Enabled: true, Timeout: time.Second, Concurrency: 1, RateLimit: 100}, []Verifier{v})
+
+ results := engine.VerifyAll(context.Background(), []VerifyPair{makePair(v.detectorID, "unsafe****key")})
+ require.Len(t, results, 1)
+ assert.Equal(t, finding.StatusVerifyError, results[0].Verification.Status)
+ assert.Zero(t, base.callCount.Load(), "verifier with an unsafe request budget must not run")
+}
+
+func TestVerifyAll_GatedVerifierCannotExceedDeclaredBudget(t *testing.T) {
+ base := &testVerifier{detectorID: "runaway-provider", result: finding.VerificationResult{Status: finding.StatusVerifiedActive}}
+ v := &gatedTestVerifier{testVerifier: base, requestBudget: 2, attempts: 3}
+ engine := NewEngine(Config{Enabled: true, Timeout: time.Second, Concurrency: 1, RateLimit: 1000}, []Verifier{v})
+
+ results := engine.VerifyAll(context.Background(), []VerifyPair{makePair(v.detectorID, "runaway****key")})
+ require.Len(t, results, 1)
+ assert.Equal(t, finding.StatusVerifyError, results[0].Verification.Status)
+ assert.Zero(t, base.callCount.Load(), "verifier result must not run after it exceeds the request budget")
+ assert.Len(t, v.admissionTimes(), 2)
+}
+
+func TestVerifyAll_RequestBudgetPanicFailsClosedWithoutPayload(t *testing.T) {
+ const canary = "budget-panic-secret-canary-7rQ2mN9p"
+ base := &testVerifier{detectorID: "panic-budget-provider", result: finding.VerificationResult{Status: finding.StatusVerifiedActive}}
+ v := &gatedTestVerifier{testVerifier: base, requestBudget: 2, attempts: 1, budgetPanic: canary}
+ engine := NewEngine(Config{Enabled: true, Timeout: time.Second, Concurrency: 1, RateLimit: 100}, []Verifier{v})
+
+ results := engine.VerifyAll(context.Background(), []VerifyPair{makePair(v.detectorID, "budget****canary")})
+ require.Len(t, results, 1)
+ assert.Equal(t, finding.StatusVerifyError, results[0].Verification.Status)
+ assert.NotContains(t, results[0].Verification.Message, canary)
+ assert.Zero(t, base.callCount.Load())
+}
+
+func TestWaitRateLimit_GlobalFailureRefundsUnusedProviderReservation(t *testing.T) {
+ engine := NewEngine(Config{Enabled: true, Timeout: time.Second, Concurrency: 1, RateLimit: 1}, nil)
+ require.True(t, engine.rateLimiter.Allow(), "consume the global limiter's initial token")
+ ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
+ defer cancel()
+
+ rejection := engine.waitRateLimit(ctx, "refunded-provider")
+ require.NotNil(t, rejection)
+ assert.Equal(t, finding.StatusVerifyError, rejection.Status)
+
+ // Replace only the global limiter with an immediately available one. The
+ // second admission can succeed under a short deadline only if the failed
+ // combined reservation refunded the provider token.
+ engine.rateLimiter = rate.NewLimiter(rate.Inf, 1)
+ secondCtx, secondCancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+ defer secondCancel()
+ assert.Nil(t, engine.waitRateLimit(secondCtx, "refunded-provider"))
+}
+
+func TestVerifyAll_HTTP429RetryUsesGlobalAndProviderLimiters(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ globalRate rate.Limit
+ providerRate rate.Limit
+ }{
+ {name: "provider limiter is enforced", globalRate: 1000, providerRate: 10},
+ {name: "global limiter is enforced", globalRate: 10, providerRate: 1000},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ var (
+ mu sync.Mutex
+ callsAt []time.Time
+ )
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ mu.Lock()
+ callsAt = append(callsAt, time.Now())
+ call := len(callsAt)
+ mu.Unlock()
+ if call == 1 {
+ w.Header().Set("Retry-After", "0")
+ w.WriteHeader(http.StatusTooManyRequests)
+ return
+ }
+ w.WriteHeader(http.StatusOK)
+ }))
+ defer server.Close()
+
+ const detectorID = "http-retry-provider"
+ v := &httpRetryTestVerifier{detectorID: detectorID, client: server.Client(), url: server.URL}
+ engine := NewEngine(Config{Enabled: true, Timeout: time.Second, Concurrency: 1, RateLimit: 1000}, []Verifier{v})
+ engine.rateLimiter = rate.NewLimiter(tc.globalRate, 1)
+ engine.perDetector[detectorID] = rate.NewLimiter(tc.providerRate, 1)
+
+ results := engine.VerifyAll(context.Background(), []VerifyPair{makePair(detectorID, "http****retry")})
+ require.Len(t, results, 1)
+ assert.Equal(t, finding.StatusVerifiedActive, results[0].Verification.Status)
+ mu.Lock()
+ defer mu.Unlock()
+ require.Len(t, callsAt, 2)
+ assert.GreaterOrEqual(t, callsAt[1].Sub(callsAt[0]), 85*time.Millisecond,
+ "retry must pass the limiting bucket at its actual send point")
+ })
+ }
+}
+
func TestVerifyAll_NoMatchingVerifier_LeavesUnverified(t *testing.T) {
v := &testVerifier{
detectorID: "aws-access-key-id",
@@ -314,6 +590,25 @@ func TestVerifyAll_PanickingVerifier_RecoversAsVerifyError(t *testing.T) {
assert.Equal(t, int64(1), pv.callCount.Load())
}
+func TestVerifyAll_PanicPayloadCannotLeakRawSecretToLogs(t *testing.T) {
+ const canary = "panic-payload-secret-canary-9mN2pQ7r"
+ var logs bytes.Buffer
+ previous := slog.Default()
+ slog.SetDefault(slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug})))
+ t.Cleanup(func() { slog.SetDefault(previous) })
+
+ pv := &panicVerifier{detectorID: "panic-secret-detector", panicRaw: true}
+ engine := NewEngine(Config{Enabled: true, Timeout: time.Second, Concurrency: 1, RateLimit: 100}, []Verifier{pv})
+ pair := makePair(pv.detectorID, "panic****canary")
+ pair.Raw.Raw = []byte(canary)
+
+ results := engine.VerifyAll(context.Background(), []VerifyPair{pair})
+ require.Len(t, results, 1)
+ assert.Equal(t, finding.StatusVerifyError, results[0].Verification.Status)
+ assert.NotContains(t, logs.String(), canary)
+ assert.Contains(t, logs.String(), "verifier panicked; recovered to protect the scan")
+}
+
func TestVerifyAll_PanickingVerifier_OtherFindingsStillComplete(t *testing.T) {
pv := &panicVerifier{detectorID: "panic-detector"}
good := &testVerifier{
@@ -389,3 +684,21 @@ func TestNewEngine_DefaultValues_AppliedForZeroConfig(t *testing.T) {
assert.Equal(t, DefaultTimeout, engine.timeout)
assert.True(t, engine.enabled)
}
+
+func TestNewEngine_InvalidVerifierEntriesCannotCrashConstruction(t *testing.T) {
+ const canary = "type-panic-secret-canary-5pN8qR2m"
+ var typedNil *testVerifier
+ var engine *Engine
+ assert.NotPanics(t, func() {
+ engine = NewEngine(Config{Enabled: true}, []Verifier{
+ typedNil,
+ &panicTypeVerifier{payload: canary},
+ })
+ })
+ require.NotNil(t, engine)
+ assert.Empty(t, engine.verifiers)
+
+ results := engine.VerifyAll(context.Background(), []VerifyPair{makePair("invalid-provider", "invalid****key")})
+ require.Len(t, results, 1)
+ assert.Equal(t, finding.StatusUnverified, results[0].Verification.Status)
+}
diff --git a/internal/verifier/figma/figma_shared_test.go b/internal/verifier/figma/figma_shared_test.go
new file mode 100644
index 0000000..4248d73
--- /dev/null
+++ b/internal/verifier/figma/figma_shared_test.go
@@ -0,0 +1,16 @@
+package figma
+
+import (
+ "net/http"
+ "testing"
+
+ "github.com/HodeTech/leakwatch/internal/detector"
+ "github.com/HodeTech/leakwatch/internal/verifier"
+ "github.com/HodeTech/leakwatch/internal/verifier/internal/vtest"
+)
+
+func TestVerify_SharedSafetySuite(t *testing.T) {
+ vtest.Run(t, vtest.Case{Name: "figma", New: func(url string, client *http.Client) verifier.Verifier {
+ return &Verifier{apiURL: url, httpClient: client}
+ }, Raw: detector.RawFinding{DetectorID: detectorID, Raw: []byte("figd_SyntheticFigmaCredentialFixture123456")}})
+}
diff --git a/internal/verifier/github/github_oauth_verifier.go b/internal/verifier/github/github_oauth_verifier.go
index 49da59c..f1802e3 100644
--- a/internal/verifier/github/github_oauth_verifier.go
+++ b/internal/verifier/github/github_oauth_verifier.go
@@ -5,8 +5,10 @@ package github
import (
"context"
"encoding/json"
+ "fmt"
"io"
"net/http"
+ "strings"
"github.com/HodeTech/leakwatch/internal/detector"
"github.com/HodeTech/leakwatch/internal/verifier"
@@ -19,12 +21,29 @@ const oauthDetectorID = "github-oauth-token"
// OAuthVerifier checks whether a GitHub OAuth token is active by calling the
// GitHub API. It NEVER logs or persists raw token values.
type OAuthVerifier struct {
- // apiURL overrides the GitHub API base URL (for testing).
+ // apiURL is reserved for a validated, trusted GitHub.com or GHES API
+ // origin. The production registration leaves it empty until operator wiring
+ // exists; tests inject a local server.
apiURL string
// httpClient overrides the default HTTP client (for testing).
httpClient *http.Client
}
+// NewOAuthForTrustedInstance constructs a subtype-aware GitHub App/OAuth
+// verifier for an explicit GitHub.com or GHES API origin.
+func NewOAuthForTrustedInstance(instanceURL string) (*OAuthVerifier, error) {
+ normalized, err := verifier.NormalizeTrustedHTTPSOrigin(instanceURL)
+ if err != nil {
+ return nil, err
+ }
+ return &OAuthVerifier{apiURL: normalized}, nil
+}
+
+// WithTrustedInstance implements verifier.TrustedInstanceConfigurer.
+func (*OAuthVerifier) WithTrustedInstance(instanceURL string) (verifier.Verifier, error) {
+ return NewOAuthForTrustedInstance(instanceURL)
+}
+
func init() {
verifier.Register(&OAuthVerifier{})
}
@@ -38,30 +57,75 @@ func (v *OAuthVerifier) Type() string {
// Raw contains the token value.
func (v *OAuthVerifier) Verify(ctx context.Context, raw detector.RawFinding) finding.VerificationResult {
token := string(raw.Raw)
- apiURL := httpx.BaseURL(v.apiURL, defaultAPIURL)
+ if token == "" {
+ return finding.VerificationResult{Status: finding.StatusUnverified, Message: "empty token"}
+ }
+ if v.apiURL == "" {
+ return finding.VerificationResult{
+ Status: finding.StatusUnverified,
+ Message: "trusted GitHub API origin is not configured",
+ }
+ }
+ apiURL := v.apiURL
+ path := "/user"
+ var decode httpx.DecodeFunc
+ var decodeResponse httpx.ResponseDecodeFunc
+ switch {
+ case strings.HasPrefix(token, "gho_"), strings.HasPrefix(token, "ghu_"):
+ // OAuth and GitHub App user access tokens authenticate a user.
+ decodeResponse = decodeUserResponse
+ case strings.HasPrefix(token, "ghs_"):
+ // Installation tokens do not support GET /user. This endpoint is a
+ // read-only identity-safe probe and works even when the installation has
+ // access to zero repositories.
+ path = "/installation/repositories"
+ decode = decodeInstallationRepositories
+ case strings.HasPrefix(token, "ghr_"):
+ // Verifying a refresh token requires exchanging it, which rotates the
+ // token. A verifier must never create that side effect.
+ return finding.VerificationResult{
+ Status: finding.StatusUnverified,
+ Message: "GitHub refresh token cannot be verified without rotating it",
+ }
+ default:
+ return finding.VerificationResult{
+ Status: finding.StatusUnverified,
+ Message: "unsupported GitHub token subtype",
+ }
+ }
return httpx.VerifyToken(ctx, v.httpClient, token, httpx.TokenSpec{
Name: "github oauth",
Request: httpx.Request{
- URL: apiURL + "/user",
+ URL: apiURL + path,
Header: map[string]string{
"Authorization": "Bearer " + token,
"Accept": "application/vnd.github+json",
},
},
- ActiveMessage: "GitHub OAuth token is active",
- InactiveMessage: "GitHub OAuth token is invalid or revoked",
- Decode: decodeOAuthUser,
+ ActiveMessage: "GitHub OAuth or installation token is active",
+ InactiveMessage: "GitHub OAuth or installation token is invalid or revoked",
+ Decode: decode,
+ DecodeResponse: decodeResponse,
+ DecodeInactive: decodeGitHubInactive,
+ RequireCompleteBody: true,
+ RequireJSONContentType: true,
})
}
-// decodeOAuthUser parses the GitHub API response for a valid OAuth token.
-func decodeOAuthUser(body io.Reader) (map[string]string, string, error) {
- var user struct {
- Login string `json:"login"`
+func decodeInstallationRepositories(body io.Reader) (map[string]string, string, error) {
+ var response struct {
+ TotalCount *int `json:"total_count"`
}
- if err := json.NewDecoder(body).Decode(&user); err != nil {
+ decoder := json.NewDecoder(body)
+ if err := decoder.Decode(&response); err != nil {
return nil, "", err
}
- return map[string]string{"login": user.Login}, "", nil
+ if err := requireGitHubJSONEOF(decoder); err != nil {
+ return nil, "", err
+ }
+ if response.TotalCount == nil || *response.TotalCount < 0 {
+ return nil, "", fmt.Errorf("missing GitHub installation repository count")
+ }
+ return map[string]string{"token_subtype": "ghs"}, "", nil
}
diff --git a/internal/verifier/github/github_oauth_verifier_test.go b/internal/verifier/github/github_oauth_verifier_test.go
index 35b8e9a..ea75fed 100644
--- a/internal/verifier/github/github_oauth_verifier_test.go
+++ b/internal/verifier/github/github_oauth_verifier_test.go
@@ -4,6 +4,7 @@ import (
"context"
"net/http"
"net/http/httptest"
+ "strings"
"testing"
"github.com/stretchr/testify/assert"
@@ -20,6 +21,8 @@ func TestOAuthVerify_ValidToken_ReturnsActive(t *testing.T) {
assert.Equal(t, "application/vnd.github+json", r.Header.Get("Accept"))
w.Header().Set("Content-Type", "application/json")
+ w.Header().Set("X-OAuth-Scopes", "user, repo, user")
+ w.Header().Set("GitHub-Authentication-Token-Expiration", "2027-01-02 03:04:05 UTC")
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte(`{"login":"octocat"}`))
}))
@@ -39,12 +42,61 @@ func TestOAuthVerify_ValidToken_ReturnsActive(t *testing.T) {
result := v.Verify(context.Background(), raw)
require.Equal(t, finding.StatusVerifiedActive, result.Status)
- assert.Equal(t, "GitHub OAuth token is active", result.Message)
+ assert.Equal(t, "GitHub OAuth or installation token is active", result.Message)
assert.Equal(t, "octocat", result.ExtraData["login"])
+ assert.Equal(t, "repo,user", result.ExtraData["scopes"])
+ assert.Equal(t, "2", result.ExtraData["scope_count"])
+ assert.Equal(t, "2027-01-02T03:04:05Z", result.ExtraData["expires_at"])
+}
+
+func TestDecodeInstallationRepositories_RejectsMalformedSuccess(t *testing.T) {
+ for _, body := range []string{
+ `{"total_count":-1}`,
+ `{"total_count":0} {"extra":true}`,
+ } {
+ _, _, err := decodeInstallationRepositories(strings.NewReader(body))
+ assert.Error(t, err, body)
+ }
+}
+
+func TestOAuthVerify_InstallationToken_UsesInstallationEndpoint(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ assert.Equal(t, "/installation/repositories", r.URL.Path)
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write([]byte(`{"total_count":0,"repositories":[]}`))
+ }))
+ defer server.Close()
+
+ result := (&OAuthVerifier{apiURL: server.URL, httpClient: server.Client()}).Verify(
+ context.Background(),
+ detector.RawFinding{Raw: []byte("ghs_ABCDEFGHIJKLMNOPQRSTUVWXYZabcdef12")},
+ )
+
+ require.Equal(t, finding.StatusVerifiedActive, result.Status)
+ assert.Equal(t, "ghs", result.ExtraData["token_subtype"])
+}
+
+func TestOAuthVerify_RefreshToken_IsUnverifiedWithoutRequest(t *testing.T) {
+ requests := 0
+ server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
+ requests++
+ }))
+ defer server.Close()
+
+ result := (&OAuthVerifier{apiURL: server.URL, httpClient: server.Client()}).Verify(
+ context.Background(),
+ detector.RawFinding{Raw: []byte("ghr_ABCDEFGHIJKLMNOPQRSTUVWXYZabcdef12")},
+ )
+
+ assert.Equal(t, finding.StatusUnverified, result.Status)
+ assert.Contains(t, result.Message, "cannot be verified without rotating")
+ assert.Zero(t, requests)
}
func TestOAuthVerify_InvalidToken_ReturnsInactive(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusUnauthorized)
_, _ = w.Write([]byte(`{"message":"Bad credentials"}`))
}))
@@ -64,7 +116,20 @@ func TestOAuthVerify_InvalidToken_ReturnsInactive(t *testing.T) {
result := v.Verify(context.Background(), raw)
assert.Equal(t, finding.StatusVerifiedInactive, result.Status)
- assert.Equal(t, "GitHub OAuth token is invalid or revoked", result.Message)
+ assert.Equal(t, "GitHub OAuth or installation token is invalid or revoked", result.Message)
+}
+
+func TestOAuthVerify_InactiveHTMLIsInconclusive(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", "text/html")
+ w.WriteHeader(http.StatusUnauthorized)
+ _, _ = w.Write([]byte(`proxy login`))
+ }))
+ defer server.Close()
+ result := (&OAuthVerifier{apiURL: server.URL, httpClient: server.Client()}).Verify(
+ context.Background(), detector.RawFinding{Raw: []byte("gho_synthetic123456789012345678901234")},
+ )
+ assert.Equal(t, finding.StatusVerifyError, result.Status)
}
func TestOAuthVerify_ServerError_ReturnsError(t *testing.T) {
@@ -143,6 +208,14 @@ func TestOAuthVerify_EmptyToken_ReturnsUnverified(t *testing.T) {
assert.Equal(t, "empty token", result.Message)
}
+func TestOAuthVerify_WithoutTrustedOrigin_MakesNoRequest(t *testing.T) {
+ v := &OAuthVerifier{}
+ result := v.Verify(context.Background(), detector.RawFinding{Raw: []byte("synthetic-github-oauth-token")})
+
+ assert.Equal(t, finding.StatusUnverified, result.Status)
+ assert.Equal(t, "trusted GitHub API origin is not configured", result.Message)
+}
+
func TestOAuthVerify_MalformedJSON_ReturnsVerifyError(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/json")
@@ -169,3 +242,39 @@ func TestOAuthVerify_MalformedJSON_ReturnsVerifyError(t *testing.T) {
assert.Equal(t, finding.StatusVerifyError, result.Status)
assert.Contains(t, result.Message, "failed to decode response body")
}
+
+func TestOAuthVerify_MissingIdentityOrWrongContentType_ReturnsVerifyError(t *testing.T) {
+ tests := []struct {
+ name string
+ contentType string
+ body string
+ }{
+ {name: "missing login", contentType: "application/json", body: `{}`},
+ {name: "wrong content type", contentType: "text/plain", body: `{"login":"octocat"}`},
+ }
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", tc.contentType)
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write([]byte(tc.body))
+ }))
+ defer server.Close()
+
+ result := (&OAuthVerifier{apiURL: server.URL, httpClient: server.Client()}).Verify(
+ context.Background(), detector.RawFinding{Raw: []byte("gho_ABCDEFGHIJKLMNOPQRSTUVWXYZabcdef12")},
+ )
+ assert.Equal(t, finding.StatusVerifyError, result.Status)
+ })
+ }
+}
+
+func TestNewOAuthForTrustedInstance_ValidatesOrigin(t *testing.T) {
+ configured, err := NewOAuthForTrustedInstance("https://api.github.example/")
+ require.NoError(t, err)
+ assert.Equal(t, "https://api.github.example", configured.apiURL)
+ for _, origin := range []string{"http://api.github.com", "https://127.0.0.1", "https://localhost", "https://github.example/path"} {
+ _, err := NewOAuthForTrustedInstance(origin)
+ assert.Error(t, err, origin)
+ }
+}
diff --git a/internal/verifier/github/github_verifier.go b/internal/verifier/github/github_verifier.go
index d4c2707..922db7e 100644
--- a/internal/verifier/github/github_verifier.go
+++ b/internal/verifier/github/github_verifier.go
@@ -5,8 +5,13 @@ package github
import (
"context"
"encoding/json"
+ "fmt"
"io"
"net/http"
+ "sort"
+ "strconv"
+ "strings"
+ "time"
"github.com/HodeTech/leakwatch/internal/detector"
"github.com/HodeTech/leakwatch/internal/verifier"
@@ -16,18 +21,32 @@ import (
const detectorID = "github-token"
-// defaultAPIURL is the base URL for the GitHub API.
-const defaultAPIURL = "https://api.github.com"
-
// Verifier checks whether a GitHub token is active by calling the
// GitHub API. It NEVER logs or persists raw token values.
type Verifier struct {
- // apiURL overrides the GitHub API base URL (for testing).
+ // apiURL is reserved for a validated, trusted GitHub.com or GHES API
+ // origin. The production registration leaves it empty until operator wiring
+ // exists; tests inject a local server.
apiURL string
// httpClient overrides the default HTTP client (for testing).
httpClient *http.Client
}
+// NewForTrustedInstance constructs a GitHub PAT verifier for an origin chosen
+// explicitly by the operator (GitHub.com API or a GHES API origin).
+func NewForTrustedInstance(instanceURL string) (*Verifier, error) {
+ normalized, err := verifier.NormalizeTrustedHTTPSOrigin(instanceURL)
+ if err != nil {
+ return nil, err
+ }
+ return &Verifier{apiURL: normalized}, nil
+}
+
+// WithTrustedInstance implements verifier.TrustedInstanceConfigurer.
+func (*Verifier) WithTrustedInstance(instanceURL string) (verifier.Verifier, error) {
+ return NewForTrustedInstance(instanceURL)
+}
+
func init() {
verifier.Register(&Verifier{})
}
@@ -41,7 +60,16 @@ func (v *Verifier) Type() string {
// Raw contains the token value.
func (v *Verifier) Verify(ctx context.Context, raw detector.RawFinding) finding.VerificationResult {
token := string(raw.Raw)
- apiURL := httpx.BaseURL(v.apiURL, defaultAPIURL)
+ if token == "" {
+ return finding.VerificationResult{Status: finding.StatusUnverified, Message: "empty token"}
+ }
+ if v.apiURL == "" {
+ return finding.VerificationResult{
+ Status: finding.StatusUnverified,
+ Message: "trusted GitHub API origin is not configured",
+ }
+ }
+ apiURL := v.apiURL
return httpx.VerifyToken(ctx, v.httpClient, token, httpx.TokenSpec{
Name: "github",
@@ -52,26 +80,118 @@ func (v *Verifier) Verify(ctx context.Context, raw detector.RawFinding) finding.
"Accept": "application/vnd.github+json",
},
},
- ActiveMessage: "GitHub token is active",
- InactiveMessage: "GitHub token is invalid or revoked",
- Decode: decodeUser,
+ ActiveMessage: "GitHub token is active",
+ InactiveMessage: "GitHub token is invalid or revoked",
+ DecodeResponse: decodeUserResponse,
+ DecodeInactive: decodeGitHubInactive,
+ RequireCompleteBody: true,
+ RequireJSONContentType: true,
})
}
+func decodeGitHubInactive(body io.Reader) error {
+ var response struct {
+ Message string `json:"message"`
+ }
+ decoder := json.NewDecoder(body)
+ if err := decoder.Decode(&response); err != nil {
+ return err
+ }
+ if strings.TrimSpace(response.Message) != "Bad credentials" {
+ return fmt.Errorf("unexpected GitHub authentication error")
+ }
+ return requireGitHubJSONEOF(decoder)
+}
+
+// decodeUserResponse parses the required identity body and enriches classic PAT
+// findings with the provider's documented OAuth-scope and token-expiry response
+// headers. Fine-grained tokens may omit X-OAuth-Scopes; absence is not an error.
+func decodeUserResponse(header http.Header, body io.Reader) (map[string]string, string, error) {
+ extra, downgrade, err := decodeUser(body)
+ if err != nil {
+ return nil, "", err
+ }
+ if scopes := normalizedScopes(header.Get("X-OAuth-Scopes")); len(scopes) > 0 {
+ extra["scopes"] = strings.Join(scopes, ",")
+ extra["scope_count"] = strconv.Itoa(len(scopes))
+ }
+ if expiresAt, ok := normalizedGitHubExpiry(header.Get("GitHub-Authentication-Token-Expiration")); ok {
+ extra["expires_at"] = expiresAt
+ }
+ return extra, downgrade, nil
+}
+
// decodeUser parses the GitHub API response for a valid token.
-//
-// Enhancement note: a classic PAT's granted scopes are exposed by GitHub in
-// the X-OAuth-Scopes response header, not the body, and would be valuable
-// triage signal (a repo-scoped token is materially more critical than a
-// read:user-scoped one). Surfacing it requires httpx.DecodeFunc to receive
-// the response headers, which it currently does not (body-only) — that is a
-// change to internal/verifier/internal/httpx, outside this package.
func decodeUser(body io.Reader) (map[string]string, string, error) {
var user struct {
Login string `json:"login"`
}
- if err := json.NewDecoder(body).Decode(&user); err != nil {
+ decoder := json.NewDecoder(body)
+ if err := decoder.Decode(&user); err != nil {
+ return nil, "", err
+ }
+ user.Login = strings.TrimSpace(user.Login)
+ if user.Login == "" {
+ return nil, "", fmt.Errorf("missing GitHub user login")
+ }
+ if err := requireGitHubJSONEOF(decoder); err != nil {
return nil, "", err
}
return map[string]string{"login": user.Login}, "", nil
}
+
+func requireGitHubJSONEOF(decoder *json.Decoder) error {
+ var trailing any
+ if err := decoder.Decode(&trailing); err != io.EOF {
+ if err == nil {
+ return fmt.Errorf("GitHub response contains trailing JSON values")
+ }
+ return fmt.Errorf("invalid trailing GitHub response: %w", err)
+ }
+ return nil
+}
+
+func normalizedScopes(raw string) []string {
+ if len(raw) > 4096 {
+ return nil
+ }
+ seen := make(map[string]struct{})
+ for _, candidate := range strings.Split(raw, ",") {
+ scope := strings.TrimSpace(candidate)
+ if scope == "" || len(scope) > 64 || !isSafeScope(scope) {
+ continue
+ }
+ seen[scope] = struct{}{}
+ if len(seen) >= 128 {
+ break
+ }
+ }
+ scopes := make([]string, 0, len(seen))
+ for scope := range seen {
+ scopes = append(scopes, scope)
+ }
+ sort.Strings(scopes)
+ return scopes
+}
+
+func isSafeScope(scope string) bool {
+ for _, char := range scope {
+ if (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z') ||
+ (char >= '0' && char <= '9') || char == ':' || char == '_' || char == '-' {
+ continue
+ }
+ return false
+ }
+ return true
+}
+
+func normalizedGitHubExpiry(raw string) (string, bool) {
+ value := strings.TrimSpace(raw)
+ for _, layout := range []string{time.RFC3339, "2006-01-02 15:04:05 MST"} {
+ parsed, err := time.Parse(layout, value)
+ if err == nil {
+ return parsed.UTC().Format(time.RFC3339), true
+ }
+ }
+ return "", false
+}
diff --git a/internal/verifier/github/github_verifier_test.go b/internal/verifier/github/github_verifier_test.go
index 32b2c98..f0caded 100644
--- a/internal/verifier/github/github_verifier_test.go
+++ b/internal/verifier/github/github_verifier_test.go
@@ -4,6 +4,7 @@ import (
"context"
"net/http"
"net/http/httptest"
+ "strings"
"testing"
"github.com/stretchr/testify/assert"
@@ -19,6 +20,8 @@ func TestVerify_ValidToken_ReturnsActive(t *testing.T) {
assert.Contains(t, r.Header.Get("Authorization"), "Bearer ")
w.Header().Set("Content-Type", "application/json")
+ w.Header().Set("X-OAuth-Scopes", "user, repo, user")
+ w.Header().Set("GitHub-Authentication-Token-Expiration", "2027-01-02 03:04:05 UTC")
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte(`{"login":"octocat","id":1}`))
}))
@@ -40,10 +43,25 @@ func TestVerify_ValidToken_ReturnsActive(t *testing.T) {
require.Equal(t, finding.StatusVerifiedActive, result.Status)
assert.Equal(t, "GitHub token is active", result.Message)
assert.Equal(t, "octocat", result.ExtraData["login"])
+ assert.Equal(t, "repo,user", result.ExtraData["scopes"])
+ assert.Equal(t, "2", result.ExtraData["scope_count"])
+ assert.Equal(t, "2027-01-02T03:04:05Z", result.ExtraData["expires_at"])
+}
+
+func TestDecodeUserResponse_IgnoresMalformedUntrustedMetadata(t *testing.T) {
+ header := http.Header{
+ "X-Oauth-Scopes": []string{"repo, scope with spaces, bad\nvalue"},
+ "Github-Authentication-Token-Expiration": []string{"not-a-date"},
+ }
+ extra, _, err := decodeUserResponse(header, strings.NewReader(`{"login":"octocat"}`))
+ require.NoError(t, err)
+ assert.Equal(t, "repo", extra["scopes"])
+ assert.NotContains(t, extra, "expires_at")
}
func TestVerify_InvalidToken_ReturnsInactive(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusUnauthorized)
_, _ = w.Write([]byte(`{"message":"Bad credentials"}`))
}))
@@ -66,6 +84,31 @@ func TestVerify_InvalidToken_ReturnsInactive(t *testing.T) {
assert.Equal(t, "GitHub token is invalid or revoked", result.Message)
}
+func TestVerify_InactiveResponseMustBeDefinitiveJSON(t *testing.T) {
+ tests := []struct {
+ name string
+ contentType string
+ body string
+ }{
+ {name: "proxy html", contentType: "text/html", body: `login`},
+ {name: "ambiguous json", contentType: "application/json", body: `{"message":"Unauthorized"}`},
+ }
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", tc.contentType)
+ w.WriteHeader(http.StatusUnauthorized)
+ _, _ = w.Write([]byte(tc.body))
+ }))
+ defer server.Close()
+ result := (&Verifier{apiURL: server.URL, httpClient: server.Client()}).Verify(
+ context.Background(), detector.RawFinding{Raw: []byte("ghp_synthetic123456789012345678901234")},
+ )
+ assert.Equal(t, finding.StatusVerifyError, result.Status)
+ })
+ }
+}
+
func TestVerify_ServerError_ReturnsError(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
@@ -109,3 +152,47 @@ func TestVerify_EmptyToken_ReturnsUnverified(t *testing.T) {
assert.Equal(t, finding.StatusUnverified, result.Status)
assert.Equal(t, "empty token", result.Message)
}
+
+func TestVerify_WithoutTrustedOrigin_MakesNoRequest(t *testing.T) {
+ v := &Verifier{}
+ result := v.Verify(context.Background(), detector.RawFinding{Raw: []byte("synthetic-github-token")})
+
+ assert.Equal(t, finding.StatusUnverified, result.Status)
+ assert.Equal(t, "trusted GitHub API origin is not configured", result.Message)
+}
+
+func TestVerify_MissingIdentityOrWrongContentType_ReturnsVerifyError(t *testing.T) {
+ tests := []struct {
+ name string
+ contentType string
+ body string
+ }{
+ {name: "missing login", contentType: "application/json", body: `{}`},
+ {name: "wrong content type", contentType: "text/html", body: `{"login":"octocat"}`},
+ }
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", tc.contentType)
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write([]byte(tc.body))
+ }))
+ defer server.Close()
+
+ result := (&Verifier{apiURL: server.URL, httpClient: server.Client()}).Verify(
+ context.Background(), detector.RawFinding{Raw: []byte("ghp_ABCDEFGHIJKLMNOPQRSTUVWXYZabcdef12")},
+ )
+ assert.Equal(t, finding.StatusVerifyError, result.Status)
+ })
+ }
+}
+
+func TestNewForTrustedInstance_ValidatesOrigin(t *testing.T) {
+ configured, err := NewForTrustedInstance("https://api.github.example/")
+ require.NoError(t, err)
+ assert.Equal(t, "https://api.github.example", configured.apiURL)
+ for _, origin := range []string{"http://api.github.com", "https://127.0.0.1", "https://localhost", "https://github.example/path"} {
+ _, err := NewForTrustedInstance(origin)
+ assert.Error(t, err, origin)
+ }
+}
diff --git a/internal/verifier/gitlab/gitlab_verifier.go b/internal/verifier/gitlab/gitlab_verifier.go
index d72012a..4e1bf7e 100644
--- a/internal/verifier/gitlab/gitlab_verifier.go
+++ b/internal/verifier/gitlab/gitlab_verifier.go
@@ -5,8 +5,13 @@ package gitlab
import (
"context"
"encoding/json"
+ "fmt"
"io"
"net/http"
+ "sort"
+ "strconv"
+ "strings"
+ "time"
"github.com/HodeTech/leakwatch/internal/detector"
"github.com/HodeTech/leakwatch/internal/verifier"
@@ -16,9 +21,12 @@ import (
const detectorID = "gitlab-pat"
-// defaultHost is the GitLab SaaS host, used when no self-hosted instance host
-// was captured alongside the token.
-const defaultHost = "gitlab.com"
+type gitLabGranularScope struct {
+ Access string `json:"access"`
+ Permissions []string `json:"permissions"`
+ ProjectID *int64 `json:"project_id"`
+ GroupID *int64 `json:"group_id"`
+}
// Verifier checks whether a GitLab personal access token is active by calling
// the GitLab API. It NEVER logs or persists raw token values.
@@ -29,6 +37,24 @@ type Verifier struct {
httpClient *http.Client
}
+var _ verifier.RequestGatedVerifier = (*Verifier)(nil)
+
+// NewForTrustedInstance constructs a GitLab verifier for an API origin
+// explicitly supplied by the operator. Finding metadata and scanned URLs are
+// deliberately never accepted as routing authority.
+func NewForTrustedInstance(instanceURL string) (*Verifier, error) {
+ normalized, err := verifier.NormalizeTrustedHTTPSOrigin(instanceURL)
+ if err != nil {
+ return nil, err
+ }
+ return &Verifier{apiURL: normalized}, nil
+}
+
+// WithTrustedInstance implements verifier.TrustedInstanceConfigurer.
+func (*Verifier) WithTrustedInstance(instanceURL string) (verifier.Verifier, error) {
+ return NewForTrustedInstance(instanceURL)
+}
+
func init() {
verifier.Register(&Verifier{})
}
@@ -38,48 +64,296 @@ func (v *Verifier) Type() string {
return detectorID
}
+// VerificationRequestBudget covers the identity probe, the optional PAT
+// self-metadata probe, and at most one bounded 429 retry for each GET.
+func (*Verifier) VerificationRequestBudget() int { return 4 }
+
// Verify checks if the detected GitLab personal access token is valid/active.
-// Raw contains the token value. The API host is derived from a co-located
-// self-hosted instance host (ExtraData["host"]) when present, defaulting to
-// gitlab.com — so an active self-hosted token is verified against its true
-// issuer instead of being misreported as invalid by gitlab.com.
+// Raw contains the token value. Only personal access tokens (`glpat-`) can use
+// the read-only `/user` probe. Other GitLab credential families have different
+// authentication contracts and remain unverified.
func (v *Verifier) Verify(ctx context.Context, raw detector.RawFinding) finding.VerificationResult {
+ return v.verify(ctx, raw, nil)
+}
+
+// VerifyWithRequestGate admits every actual identity/metadata request (and any
+// bounded GET retry) at its send point.
+func (v *Verifier) VerifyWithRequestGate(
+ ctx context.Context,
+ raw detector.RawFinding,
+ gate verifier.RequestGate,
+) finding.VerificationResult {
+ return v.verify(ctx, raw, gate)
+}
+
+func (v *Verifier) verify(
+ ctx context.Context,
+ raw detector.RawFinding,
+ gate verifier.RequestGate,
+) finding.VerificationResult {
token := string(raw.Raw)
- apiURL := v.baseURL(raw)
+ if token == "" {
+ return finding.VerificationResult{Status: finding.StatusUnverified, Message: "empty token"}
+ }
+ if !strings.HasPrefix(token, "glpat-") {
+ return finding.VerificationResult{
+ Status: finding.StatusUnverified,
+ Message: "GitLab token subtype has no safe universal verification probe",
+ }
+ }
+ if v.apiURL == "" {
+ return finding.VerificationResult{
+ Status: finding.StatusUnverified,
+ Message: "trusted GitLab API origin is not configured",
+ }
+ }
+ if rejection := admitGitLabRequest(gate); rejection != nil {
+ return *rejection
+ }
+ requestCtx := ctx
+ if gate != nil {
+ requestCtx = httpx.WithRetryGate(ctx, httpx.RetryGate(gate))
+ }
- return httpx.VerifyToken(ctx, v.httpClient, token, httpx.TokenSpec{
+ result := httpx.VerifyToken(requestCtx, v.httpClient, token, httpx.TokenSpec{
Name: "gitlab",
Request: httpx.Request{
- URL: apiURL + "/api/v4/user",
+ URL: v.apiURL + "/api/v4/user",
Header: map[string]string{"PRIVATE-TOKEN": token},
},
- ActiveMessage: "GitLab token is active",
- InactiveMessage: "GitLab token is invalid or revoked",
- Decode: decodeUser,
+ ActiveMessage: "GitLab token is active",
+ InactiveMessage: "GitLab token is invalid or revoked",
+ Decode: decodeUser,
+ DecodeInactive: decodeUnauthorized,
+ RequireCompleteBody: true,
+ RequireJSONContentType: true,
})
-}
+ if result.Status != finding.StatusVerifiedActive {
+ return result
+ }
-// baseURL resolves the GitLab API base URL. A test-injected apiURL wins;
-// otherwise the co-located self-hosted host (ExtraData["host"]) is used, falling
-// back to gitlab.com. raw.ExtraData may be nil (safe to index).
-func (v *Verifier) baseURL(raw detector.RawFinding) string {
- if v.apiURL != "" {
- return v.apiURL
+ // Token metadata is optional enrichment. Older/self-managed instances and
+ // non-personal glpat token families may not expose the PAT self endpoint;
+ // the proven-active /user result remains authoritative in those cases.
+ if rejection := admitGitLabRequest(gate); rejection != nil {
+ return result
}
- host := defaultHost
- if h := raw.ExtraData["host"]; h != "" {
- host = h
+ metadata := httpx.VerifyToken(requestCtx, v.httpClient, token, httpx.TokenSpec{
+ Name: "gitlab-token-metadata",
+ Request: httpx.Request{
+ URL: v.apiURL + "/api/v4/personal_access_tokens/self",
+ Header: map[string]string{"PRIVATE-TOKEN": token},
+ },
+ ActiveMessage: "GitLab token metadata is available",
+ InactiveMessage: "GitLab token metadata is unavailable",
+ Decode: decodeTokenMetadata,
+ RequireCompleteBody: true,
+ RequireJSONContentType: true,
+ })
+ if metadata.Status == finding.StatusVerifiedActive {
+ if result.ExtraData == nil {
+ result.ExtraData = make(map[string]string)
+ }
+ for key, value := range metadata.ExtraData {
+ result.ExtraData[key] = value
+ }
+ }
+ return result
+}
+
+func admitGitLabRequest(gate verifier.RequestGate) *finding.VerificationResult {
+ if gate == nil {
+ return nil
}
- return "https://" + host
+ return gate()
}
// decodeUser parses the GitLab API response for a valid token.
func decodeUser(body io.Reader) (map[string]string, string, error) {
var user struct {
+ ID int64 `json:"id"`
Username string `json:"username"`
}
- if err := json.NewDecoder(body).Decode(&user); err != nil {
+ decoder := json.NewDecoder(body)
+ if err := decoder.Decode(&user); err != nil {
return nil, "", err
}
+ if err := requireJSONEOF(decoder); err != nil {
+ return nil, "", err
+ }
+ if user.ID <= 0 || strings.TrimSpace(user.Username) == "" {
+ return nil, "", fmt.Errorf("GitLab user response is missing required identity fields")
+ }
return map[string]string{"username": user.Username}, "", nil
}
+
+func decodeUnauthorized(body io.Reader) error {
+ var response struct {
+ Message string `json:"message"`
+ }
+ decoder := json.NewDecoder(body)
+ if err := decoder.Decode(&response); err != nil {
+ return err
+ }
+ if err := requireJSONEOF(decoder); err != nil {
+ return err
+ }
+ if response.Message != "401 Unauthorized" {
+ return fmt.Errorf("GitLab response is not a standard invalid-token response")
+ }
+ return nil
+}
+
+func decodeTokenMetadata(body io.Reader) (map[string]string, string, error) {
+ var metadata struct {
+ Active bool `json:"active"`
+ Revoked bool `json:"revoked"`
+ Scopes []string `json:"scopes"`
+ GranularScopes []gitLabGranularScope `json:"granular_scopes"`
+ ExpiresAt *string `json:"expires_at"`
+ }
+ decoder := json.NewDecoder(body)
+ if err := decoder.Decode(&metadata); err != nil {
+ return nil, "", err
+ }
+ if err := requireJSONEOF(decoder); err != nil {
+ return nil, "", err
+ }
+ if !metadata.Active || metadata.Revoked {
+ return nil, "", fmt.Errorf("GitLab self metadata did not describe an active token")
+ }
+ extra := make(map[string]string)
+ scopes, err := normalizedTokenScopes(metadata.Scopes, metadata.GranularScopes)
+ if err != nil {
+ return nil, "", err
+ }
+ if len(scopes) > 0 {
+ extra["scopes"] = strings.Join(scopes, ",")
+ extra["scope_count"] = strconv.Itoa(len(scopes))
+ }
+ if metadata.ExpiresAt != nil && strings.TrimSpace(*metadata.ExpiresAt) != "" {
+ expiresAt := strings.TrimSpace(*metadata.ExpiresAt)
+ if _, err := time.Parse(time.DateOnly, expiresAt); err != nil {
+ return nil, "", fmt.Errorf("GitLab token expiry is not an ISO date")
+ }
+ extra["expires_at"] = expiresAt
+ }
+ return extra, "", nil
+}
+
+func normalizedTokenScopes(standard []string, granular []gitLabGranularScope) ([]string, error) {
+ result := normalizedMetadataScopes(standard)
+ if len(granular) > 128 {
+ return nil, fmt.Errorf("GitLab granular scope list exceeds safe limit")
+ }
+ seen := make(map[string]struct{}, len(result)+len(granular))
+ for _, scope := range result {
+ seen[scope] = struct{}{}
+ }
+ for _, scope := range granular {
+ labels, err := normalizedGranularScopeLabels(scope)
+ if err != nil {
+ return nil, err
+ }
+ for _, label := range labels {
+ seen[label] = struct{}{}
+ if len(seen) > 128 {
+ return nil, fmt.Errorf("GitLab normalized scope list exceeds safe limit")
+ }
+ }
+ }
+ result = result[:0]
+ for scope := range seen {
+ result = append(result, scope)
+ }
+ sort.Strings(result)
+ return result, nil
+}
+
+func normalizedGranularScopeLabels(scope gitLabGranularScope) ([]string, error) {
+ access := strings.TrimSpace(scope.Access)
+ if !isAllowedGranularAccess(access) || len(scope.Permissions) == 0 || len(scope.Permissions) > 128 {
+ return nil, fmt.Errorf("GitLab granular scope has an invalid access contract")
+ }
+ target, err := granularScopeTarget(access, scope.ProjectID, scope.GroupID)
+ if err != nil {
+ return nil, err
+ }
+ labels := make([]string, 0, len(scope.Permissions))
+ for _, candidate := range scope.Permissions {
+ permission := strings.TrimSpace(candidate)
+ if permission == "" || len(permission) > 64 || !isSafeMetadataScope(permission) {
+ return nil, fmt.Errorf("GitLab granular scope has an invalid permission")
+ }
+ labels = append(labels, "granular:"+access+target+":"+permission)
+ }
+ return labels, nil
+}
+
+func granularScopeTarget(access string, projectID, groupID *int64) (string, error) {
+ if access != "selected_memberships" {
+ if projectID != nil || groupID != nil {
+ return "", fmt.Errorf("GitLab granular scope has an unexpected membership target")
+ }
+ return "", nil
+ }
+ switch {
+ case projectID != nil && *projectID > 0 && groupID == nil:
+ return ":project:" + strconv.FormatInt(*projectID, 10), nil
+ case groupID != nil && *groupID > 0 && projectID == nil:
+ return ":group:" + strconv.FormatInt(*groupID, 10), nil
+ default:
+ return "", fmt.Errorf("GitLab selected-membership scope has an invalid target")
+ }
+}
+
+func isAllowedGranularAccess(access string) bool {
+ switch access {
+ case "selected_memberships", "user", "instance", "all_memberships", "personal_projects":
+ return true
+ default:
+ return false
+ }
+}
+
+func normalizedMetadataScopes(input []string) []string {
+ seen := make(map[string]struct{})
+ for _, candidate := range input {
+ scope := strings.TrimSpace(candidate)
+ if scope == "" || len(scope) > 64 || !isSafeMetadataScope(scope) {
+ continue
+ }
+ seen[scope] = struct{}{}
+ if len(seen) >= 128 {
+ break
+ }
+ }
+ result := make([]string, 0, len(seen))
+ for scope := range seen {
+ result = append(result, scope)
+ }
+ sort.Strings(result)
+ return result
+}
+
+func isSafeMetadataScope(scope string) bool {
+ for _, char := range scope {
+ if (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z') ||
+ (char >= '0' && char <= '9') || char == ':' || char == '_' || char == '-' {
+ continue
+ }
+ return false
+ }
+ return true
+}
+
+func requireJSONEOF(decoder *json.Decoder) error {
+ var trailing any
+ if err := decoder.Decode(&trailing); err != io.EOF {
+ if err == nil {
+ return fmt.Errorf("response contains trailing JSON values")
+ }
+ return err
+ }
+ return nil
+}
diff --git a/internal/verifier/gitlab/gitlab_verifier_test.go b/internal/verifier/gitlab/gitlab_verifier_test.go
index 09d25e6..8488fb9 100644
--- a/internal/verifier/gitlab/gitlab_verifier_test.go
+++ b/internal/verifier/gitlab/gitlab_verifier_test.go
@@ -4,6 +4,7 @@ import (
"context"
"net/http"
"net/http/httptest"
+ "strings"
"testing"
"github.com/stretchr/testify/assert"
@@ -15,12 +16,17 @@ import (
func TestVerify_ValidToken_ReturnsActive(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- assert.Equal(t, "/api/v4/user", r.URL.Path)
assert.NotEmpty(t, r.Header.Get("PRIVATE-TOKEN"))
w.Header().Set("Content-Type", "application/json")
- w.WriteHeader(http.StatusOK)
- _, _ = w.Write([]byte(`{"id":1,"username":"johndoe","name":"John Doe"}`))
+ switch r.URL.Path {
+ case "/api/v4/user":
+ _, _ = w.Write([]byte(`{"id":1,"username":"johndoe","name":"John Doe"}`))
+ case "/api/v4/personal_access_tokens/self":
+ _, _ = w.Write([]byte(`{"active":true,"revoked":false,"scopes":["read_api","api","api"],"expires_at":"2027-02-03"}`))
+ default:
+ http.NotFound(w, r)
+ }
}))
defer server.Close()
@@ -40,10 +46,120 @@ func TestVerify_ValidToken_ReturnsActive(t *testing.T) {
require.Equal(t, finding.StatusVerifiedActive, result.Status)
assert.Equal(t, "GitLab token is active", result.Message)
assert.Equal(t, "johndoe", result.ExtraData["username"])
+ assert.Equal(t, "api,read_api", result.ExtraData["scopes"])
+ assert.Equal(t, "2", result.ExtraData["scope_count"])
+ assert.Equal(t, "2027-02-03", result.ExtraData["expires_at"])
+}
+
+func TestVerifyWithRequestGate_MetadataFailureDoesNotEraseProvenActiveIdentity(t *testing.T) {
+ requests := 0
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ requests++
+ w.Header().Set("Content-Type", "application/json")
+ if r.URL.Path == "/api/v4/user" {
+ _, _ = w.Write([]byte(`{"id":1,"username":"johndoe"}`))
+ return
+ }
+ w.WriteHeader(http.StatusNotFound)
+ _, _ = w.Write([]byte(`{"message":"404 Not Found"}`))
+ }))
+ defer server.Close()
+
+ admissions := 0
+ result := (&Verifier{apiURL: server.URL, httpClient: server.Client()}).VerifyWithRequestGate(
+ context.Background(),
+ detector.RawFinding{Raw: []byte("glpat-ABCDEFGHIJKLMNOPQRSTUVWXYZab")},
+ func() *finding.VerificationResult { admissions++; return nil },
+ )
+ require.Equal(t, finding.StatusVerifiedActive, result.Status)
+ assert.Equal(t, 2, requests)
+ assert.Equal(t, 2, admissions)
+}
+
+func TestVerifyWithRequestGate_AdmitsEvery429RetryAndMetadataSend(t *testing.T) {
+ requests := 0
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ requests++
+ w.Header().Set("Content-Type", "application/json")
+ if requests == 1 {
+ w.Header().Set("Retry-After", "0")
+ w.WriteHeader(http.StatusTooManyRequests)
+ return
+ }
+ switch r.URL.Path {
+ case "/api/v4/user":
+ _, _ = w.Write([]byte(`{"id":1,"username":"johndoe"}`))
+ case "/api/v4/personal_access_tokens/self":
+ _, _ = w.Write([]byte(`{"active":true,"revoked":false,"scopes":["read_api"]}`))
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+
+ admissions := 0
+ result := (&Verifier{apiURL: server.URL, httpClient: server.Client()}).VerifyWithRequestGate(
+ context.Background(),
+ detector.RawFinding{Raw: []byte("glpat-ABCDEFGHIJKLMNOPQRSTUVWXYZab")},
+ func() *finding.VerificationResult { admissions++; return nil },
+ )
+ require.Equal(t, finding.StatusVerifiedActive, result.Status)
+ assert.Equal(t, 3, requests)
+ assert.Equal(t, requests, admissions, "each actual HTTP send, including the 429 replay, needs one admission")
+}
+
+func TestDecodeTokenMetadata_GranularScopesAreBoundedAndDeterministic(t *testing.T) {
+ body := `{
+ "active":true,"revoked":false,"scopes":[],"expires_at":"2027-02-03",
+ "granular_scopes":[
+ {"access":"selected_memberships","permissions":["write_job","read_job"],"project_id":6,"group_id":null},
+ {"access":"user","permissions":["read_profile"],"project_id":null,"group_id":null}
+ ]
+ }`
+ extra, _, err := decodeTokenMetadata(strings.NewReader(body))
+ require.NoError(t, err)
+ assert.Equal(t,
+ "granular:selected_memberships:project:6:read_job,granular:selected_memberships:project:6:write_job,granular:user:read_profile",
+ extra["scopes"])
+ assert.Equal(t, "3", extra["scope_count"])
+ assert.Equal(t, "2027-02-03", extra["expires_at"])
+}
+
+func TestVerify_MalformedGranularMetadataDoesNotEraseActiveIdentity(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ if r.URL.Path == "/api/v4/user" {
+ _, _ = w.Write([]byte(`{"id":1,"username":"johndoe"}`))
+ return
+ }
+ _, _ = w.Write([]byte(`{"active":true,"revoked":false,"granular_scopes":[{"access":"selected_memberships","permissions":["read_job"],"project_id":null,"group_id":null}]}`))
+ }))
+ defer server.Close()
+
+ result := (&Verifier{apiURL: server.URL, httpClient: server.Client()}).Verify(
+ context.Background(), detector.RawFinding{Raw: []byte("glpat-ABCDEFGHIJKLMNOPQRSTUVWXYZab")})
+ require.Equal(t, finding.StatusVerifiedActive, result.Status)
+ assert.Equal(t, "johndoe", result.ExtraData["username"])
+ assert.NotContains(t, result.ExtraData, "scopes")
+}
+
+func TestVerifyWithRequestGate_RejectionPreventsAnyRequest(t *testing.T) {
+ called := false
+ v := &Verifier{apiURL: "https://trusted.example", httpClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
+ called = true
+ return nil, assert.AnError
+ })}}
+ rejection := finding.VerificationResult{Status: finding.StatusVerifyError, Message: "admission rejected"}
+ result := v.VerifyWithRequestGate(context.Background(), detector.RawFinding{
+ Raw: []byte("glpat-ABCDEFGHIJKLMNOPQRSTUVWXYZab"),
+ }, func() *finding.VerificationResult { return &rejection })
+ assert.Equal(t, rejection, result)
+ assert.False(t, called)
}
func TestVerify_InvalidToken_ReturnsInactive(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusUnauthorized)
_, _ = w.Write([]byte(`{"message":"401 Unauthorized"}`))
}))
@@ -66,6 +182,52 @@ func TestVerify_InvalidToken_ReturnsInactive(t *testing.T) {
assert.Equal(t, "GitLab token is invalid or revoked", result.Message)
}
+func TestVerify_DPoPChallengeDoesNotProveTokenInactive(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(http.StatusUnauthorized)
+ _, _ = w.Write([]byte(`{"error":"invalid_token","error_description":"DPoP proof required"}`))
+ }))
+ defer server.Close()
+
+ result := (&Verifier{apiURL: server.URL, httpClient: server.Client()}).Verify(
+ context.Background(),
+ detector.RawFinding{DetectorID: detectorID, Raw: []byte("glpat-ABCDEFGHIJKLMNOPQRSTUVWXYZab")},
+ )
+ assert.Equal(t, finding.StatusVerifyError, result.Status)
+ assert.NotContains(t, result.Message, "DPoP")
+}
+
+func TestVerify_ActiveResponseRequiresCompleteJSONIdentity(t *testing.T) {
+ tests := []struct {
+ name string
+ contentType string
+ body string
+ }{
+ {name: "empty object", contentType: "application/json", body: `{}`},
+ {name: "null", contentType: "application/json", body: `null`},
+ {name: "missing username", contentType: "application/json", body: `{"id":1}`},
+ {name: "missing id", contentType: "application/json", body: `{"username":"johndoe"}`},
+ {name: "trailing json", contentType: "application/json", body: `{"id":1,"username":"johndoe"}{}`},
+ {name: "wrong content type", contentType: "text/html", body: `{"id":1,"username":"johndoe"}`},
+ }
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", tc.contentType)
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write([]byte(tc.body))
+ }))
+ defer server.Close()
+ result := (&Verifier{apiURL: server.URL, httpClient: server.Client()}).Verify(
+ context.Background(),
+ detector.RawFinding{DetectorID: detectorID, Raw: []byte("glpat-ABCDEFGHIJKLMNOPQRSTUVWXYZab")},
+ )
+ assert.Equal(t, finding.StatusVerifyError, result.Status)
+ })
+ }
+}
+
func TestVerify_ServerError_ReturnsError(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
@@ -95,69 +257,42 @@ func TestVerify_Type_ReturnsCorrectID(t *testing.T) {
assert.Equal(t, "gitlab-pat", v.Type())
}
-func TestBaseURL_DerivesHostFromExtraData(t *testing.T) {
- tests := []struct {
- name string
- apiURL string
- extra map[string]string
- want string
- }{
- {
- name: "no context defaults to gitlab.com",
- want: "https://gitlab.com",
- },
- {
- name: "self-hosted host from ExtraData",
- extra: map[string]string{"host": "gitlab.example.com"},
- want: "https://gitlab.example.com",
- },
- {
- name: "self-hosted host with port",
- extra: map[string]string{"host": "gitlab.corp.internal:8443"},
- want: "https://gitlab.corp.internal:8443",
- },
- {
- name: "apiURL override wins",
- apiURL: "http://127.0.0.1:1234",
- extra: map[string]string{"host": "gitlab.example.com"},
- want: "http://127.0.0.1:1234",
- },
+func TestVerify_RepositoryHostNeverSelectsDestination(t *testing.T) {
+ called := false
+ v := &Verifier{httpClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
+ called = true
+ return nil, assert.AnError
+ })}}
+ raw := detector.RawFinding{
+ DetectorID: detectorID,
+ Raw: []byte("glpat-selfhostedtoken123456789012"),
+ ExtraData: map[string]string{"host": "gitlab.attacker.example"},
}
+ result := v.Verify(context.Background(), raw)
+ require.Equal(t, finding.StatusUnverified, result.Status)
+ assert.False(t, called)
+}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- v := &Verifier{apiURL: tt.apiURL}
- raw := detector.RawFinding{DetectorID: detectorID, ExtraData: tt.extra}
- assert.Equal(t, tt.want, v.baseURL(raw))
+func TestVerify_NonPATSubtypesRemainUnverified(t *testing.T) {
+ for _, prefix := range []string{
+ "gldt-", "glrt-", "glrtr-", "glcbt-", "glptt-", "glimt-", "glagent-",
+ "glwt-", "glsoat-", "glffct-", "gloas-", "glft-",
+ } {
+ result := (&Verifier{apiURL: "https://trusted.example"}).Verify(context.Background(), detector.RawFinding{
+ DetectorID: detectorID, Raw: []byte(prefix + "abcDEF1234567890xyzW"),
})
+ assert.Equal(t, finding.StatusUnverified, result.Status, prefix)
}
}
-func TestVerify_SelfHostedActiveToken_NotReportedInvalid(t *testing.T) {
- // A live self-hosted token verified against its true issuer must read as
- // active, never as "invalid or revoked". The test server stands in for the
- // self-hosted instance the detector captured into ExtraData["host"].
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- assert.Equal(t, "/api/v4/user", r.URL.Path)
- w.Header().Set("Content-Type", "application/json")
- w.WriteHeader(http.StatusOK)
- _, _ = w.Write([]byte(`{"username":"selfhosted-user"}`))
- }))
- defer server.Close()
-
- v := &Verifier{apiURL: server.URL, httpClient: server.Client()}
-
- raw := detector.RawFinding{
- DetectorID: detectorID,
- Raw: []byte("glpat-selfhostedtoken123456789012"),
- Redacted: "glpat-****9012",
- ExtraData: map[string]string{"host": "gitlab.example.com"},
+func TestNewForTrustedInstance_ValidatesOrigin(t *testing.T) {
+ configured, err := NewForTrustedInstance("https://gitlab.example.com/")
+ require.NoError(t, err)
+ assert.Equal(t, "https://gitlab.example.com", configured.apiURL)
+ for _, origin := range []string{"http://gitlab.com", "https://127.0.0.1", "https://localhost", "https://gitlab.com/path"} {
+ _, err := NewForTrustedInstance(origin)
+ assert.Error(t, err, origin)
}
-
- result := v.Verify(context.Background(), raw)
-
- require.Equal(t, finding.StatusVerifiedActive, result.Status)
- assert.Equal(t, "selfhosted-user", result.ExtraData["username"])
}
func TestVerify_EmptyToken_ReturnsUnverified(t *testing.T) {
@@ -174,3 +309,9 @@ func TestVerify_EmptyToken_ReturnsUnverified(t *testing.T) {
assert.Equal(t, finding.StatusUnverified, result.Status)
assert.Equal(t, "empty token", result.Message)
}
+
+type roundTripFunc func(*http.Request) (*http.Response, error)
+
+func (f roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) {
+ return f(request)
+}
diff --git a/internal/verifier/grafana/grafana_shared_test.go b/internal/verifier/grafana/grafana_shared_test.go
index ec824d4..44fca1c 100644
--- a/internal/verifier/grafana/grafana_shared_test.go
+++ b/internal/verifier/grafana/grafana_shared_test.go
@@ -13,15 +13,12 @@ func TestVerify_SharedSafetySuite(t *testing.T) {
vtest.Run(t, vtest.Case{
Name: "grafana",
New: func(apiURL string, client *http.Client) verifier.Verifier {
- return &Verifier{apiURL: apiURL, httpClient: client}
+ return &Verifier{instanceURL: apiURL, httpClient: client}
},
Raw: detector.RawFinding{
DetectorID: detectorID,
- Raw: []byte("glsa_ABCDEFGHIJKLMNOPQRSTUVWXYZabcdef12345678"),
+ Raw: []byte("glsa_sharedsafetyfixture12345678901234567_12345678"),
Redacted: "glsa_****5678",
},
- // The Grafana verifier checks only the status code on success; it has
- // no Decode func to fail on a malformed body.
- SkipMalformed: true,
})
}
diff --git a/internal/verifier/grafana/grafana_verifier.go b/internal/verifier/grafana/grafana_verifier.go
index bf8797b..6c7686b 100644
--- a/internal/verifier/grafana/grafana_verifier.go
+++ b/internal/verifier/grafana/grafana_verifier.go
@@ -1,10 +1,15 @@
-// Package grafana provides a verifier for Grafana API keys.
-// It uses the Grafana HTTP API GET /api/user endpoint to check key validity.
+// Package grafana provides a verifier for Grafana service-account tokens.
+// It uses the issuing instance's read-only access-control permissions endpoint.
package grafana
import (
"context"
+ "encoding/json"
+ "fmt"
+ "io"
"net/http"
+ "net/url"
+ "strings"
"github.com/HodeTech/leakwatch/internal/detector"
"github.com/HodeTech/leakwatch/internal/verifier"
@@ -14,14 +19,16 @@ import (
const detectorID = "grafana-api-key"
-// defaultAPIURL is the base URL for the Grafana Cloud API.
-const defaultAPIURL = "https://grafana.com"
+const permissionsPath = "/api/access-control/user/permissions"
-// Verifier checks whether a Grafana API key is active by calling the
-// Grafana Cloud API. It NEVER logs or persists raw key values.
+// Verifier checks whether a Grafana service-account token is active against
+// its issuing instance. It NEVER logs or persists raw token values.
type Verifier struct {
- // apiURL overrides the Grafana Cloud API base URL (for testing).
- apiURL string
+ // instanceURL is either empty or normalized by NewForTrustedInstance. Tests
+ // in this package may set it directly to reach a hermetic HTTP server.
+ // Repository content and RawFinding.ExtraData must never populate it because
+ // doing so would turn verification into an SSRF primitive.
+ instanceURL string
// httpClient overrides the default HTTP client (for testing).
httpClient *http.Client
}
@@ -30,33 +37,120 @@ func init() {
verifier.Register(&Verifier{})
}
+// NewForTrustedInstance constructs a Grafana verifier for an instance origin
+// explicitly supplied by the operator. Callers MUST NOT pass a URL extracted
+// from scanned repository content. The CLI intentionally exposes this only as
+// a command-line flag, not as a project-config or finding field.
+func NewForTrustedInstance(instanceURL string) (*Verifier, error) {
+ normalized, err := normalizeTrustedInstanceURL(instanceURL)
+ if err != nil {
+ return nil, err
+ }
+ return &Verifier{instanceURL: normalized}, nil
+}
+
+// WithTrustedInstance implements verifier.TrustedInstanceConfigurer.
+func (*Verifier) WithTrustedInstance(instanceURL string) (verifier.Verifier, error) {
+ return NewForTrustedInstance(instanceURL)
+}
+
// Type returns the detector ID this verifier handles.
func (v *Verifier) Type() string {
return detectorID
}
-// Verify checks if the detected Grafana API key is valid/active.
+// Verify checks if the detected Grafana service-account token is valid/active.
// Raw contains the key value.
//
-// This calls GET /api/user, matching docs/architecture/05-VERIFIER-ANALYSIS.md
-// and docs/guides/secret-verification.md (the code previously called the
-// nonexistent /api/viewer path). Note the caveat those docs already record:
-// glsa_ tokens are Grafana Cloud service-account tokens scoped to the stack
-// that issued them, so a token minted on a self-hosted or per-stack instance
-// will not authenticate against the central grafana.com host used here —
-// there is currently no per-instance URL captured for this detector to
-// verify against, the same limitation documented for Okta.
+// Grafana service-account tokens are scoped to the instance that issued them.
+// The detector does not currently have a trusted instance URL, so the
+// production-registered verifier deliberately performs no request and returns
+// unverified. In particular, it never trusts an arbitrary URL extracted from
+// repository content or falls back to the central grafana.com portal.
func (v *Verifier) Verify(ctx context.Context, raw detector.RawFinding) finding.VerificationResult {
token := string(raw.Raw)
- apiURL := httpx.BaseURL(v.apiURL, defaultAPIURL)
+ if strings.TrimSpace(token) == "" {
+ return finding.VerificationResult{
+ Status: finding.StatusUnverified,
+ Message: "empty token",
+ }
+ }
+ if v.instanceURL == "" {
+ return finding.VerificationResult{
+ Status: finding.StatusUnverified,
+ Message: "Grafana instance URL required",
+ }
+ }
return httpx.VerifyToken(ctx, v.httpClient, token, httpx.TokenSpec{
Name: "grafana",
Request: httpx.Request{
- URL: apiURL + "/api/user",
- Header: map[string]string{"Authorization": "Bearer " + token},
+ URL: v.instanceURL + permissionsPath,
+ Header: map[string]string{
+ "Accept": "application/json",
+ "Authorization": "Bearer " + token,
+ },
},
- ActiveMessage: "Grafana API key is active",
- InactiveMessage: "Grafana API key is invalid or revoked",
+ ActiveMessage: "Grafana API key is active",
+ InactiveMessage: "Grafana API key is invalid or revoked",
+ Decode: decodePermissionsObject,
+ DecodeInactive: decodeInactiveResponse,
+ RequireCompleteBody: true,
+ RequireJSONContentType: true,
})
}
+
+func decodeInactiveResponse(body io.Reader) error {
+ var response struct {
+ Message string `json:"message"`
+ }
+ decoder := json.NewDecoder(body)
+ if err := decoder.Decode(&response); err != nil {
+ return err
+ }
+ switch strings.ToLower(strings.TrimSpace(response.Message)) {
+ case "invalid api key", "unauthorized":
+ default:
+ return fmt.Errorf("unexpected Grafana authentication error")
+ }
+ if decoder.Decode(&struct{}{}) != io.EOF {
+ return fmt.Errorf("unexpected trailing Grafana response data")
+ }
+ return nil
+}
+
+func normalizeTrustedInstanceURL(rawURL string) (string, error) {
+ normalized, err := verifier.NormalizeTrustedHTTPSOrigin(rawURL)
+ if err != nil {
+ return "", fmt.Errorf("invalid Grafana instance URL: %w", err)
+ }
+ u, err := url.Parse(normalized)
+ if err != nil {
+ return "", fmt.Errorf("invalid Grafana instance URL after normalization: %w", err)
+ }
+ hostname := u.Hostname()
+ if hostname == "grafana.com" || hostname == "www.grafana.com" {
+ return "", fmt.Errorf("invalid Grafana instance URL: the central grafana.com portal is not an issuing instance")
+ }
+ return normalized, nil
+}
+
+func decodePermissionsObject(body io.Reader) (map[string]string, string, error) {
+ decoder := json.NewDecoder(body)
+ var permissions map[string]json.RawMessage
+ if err := decoder.Decode(&permissions); err != nil {
+ return nil, "", fmt.Errorf("invalid Grafana permissions response: %w", err)
+ }
+ if permissions == nil {
+ return nil, "", fmt.Errorf("invalid Grafana permissions response: expected JSON object")
+ }
+
+ var trailing json.RawMessage
+ if err := decoder.Decode(&trailing); err != io.EOF {
+ if err == nil {
+ return nil, "", fmt.Errorf("invalid Grafana permissions response: trailing JSON value")
+ }
+ return nil, "", fmt.Errorf("invalid Grafana permissions response: %w", err)
+ }
+ return nil, "", nil
+}
diff --git a/internal/verifier/grafana/grafana_verifier_test.go b/internal/verifier/grafana/grafana_verifier_test.go
index d0e65b6..6bf7239 100644
--- a/internal/verifier/grafana/grafana_verifier_test.go
+++ b/internal/verifier/grafana/grafana_verifier_test.go
@@ -2,8 +2,10 @@ package grafana
import (
"context"
+ "fmt"
"net/http"
"net/http/httptest"
+ "strings"
"testing"
"github.com/stretchr/testify/assert"
@@ -15,8 +17,10 @@ import (
func TestVerify_ValidKey_ReturnsActive(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- assert.Equal(t, "/api/user", r.URL.Path)
- assert.Contains(t, r.Header.Get("Authorization"), "Bearer ")
+ assert.Equal(t, permissionsPath, r.URL.Path)
+ assert.Equal(t, http.MethodGet, r.Method)
+ assert.Equal(t, "application/json", r.Header.Get("Accept"))
+ assert.Equal(t, "Bearer glsa_ABCDEFGHIJKLMNOPQRSTUVWXYZabcdef12345678", r.Header.Get("Authorization"))
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
@@ -25,8 +29,8 @@ func TestVerify_ValidKey_ReturnsActive(t *testing.T) {
defer server.Close()
v := &Verifier{
- apiURL: server.URL,
- httpClient: server.Client(),
+ instanceURL: strings.TrimRight(server.URL, "/"),
+ httpClient: server.Client(),
}
raw := detector.RawFinding{
@@ -43,14 +47,15 @@ func TestVerify_ValidKey_ReturnsActive(t *testing.T) {
func TestVerify_InvalidKey_ReturnsInactive(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusUnauthorized)
_, _ = w.Write([]byte(`{"message":"Invalid API key"}`))
}))
defer server.Close()
v := &Verifier{
- apiURL: server.URL,
- httpClient: server.Client(),
+ instanceURL: server.URL,
+ httpClient: server.Client(),
}
raw := detector.RawFinding{
@@ -65,6 +70,25 @@ func TestVerify_InvalidKey_ReturnsInactive(t *testing.T) {
assert.Equal(t, "Grafana API key is invalid or revoked", result.Message)
}
+func TestVerify_InactiveResponseMustBeDefinitiveJSON(t *testing.T) {
+ tests := []struct{ contentType, body string }{
+ {contentType: "text/html", body: `login`},
+ {contentType: "application/json", body: `{"message":"WAF challenge"}`},
+ }
+ for _, tc := range tests {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", tc.contentType)
+ w.WriteHeader(http.StatusUnauthorized)
+ _, _ = w.Write([]byte(tc.body))
+ }))
+ result := (&Verifier{instanceURL: server.URL, httpClient: server.Client()}).Verify(
+ context.Background(), detector.RawFinding{Raw: []byte("glsa_synthetic123456789012345678901234")},
+ )
+ server.Close()
+ assert.Equal(t, finding.StatusVerifyError, result.Status)
+ }
+}
+
func TestVerify_ServerError_ReturnsError(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
@@ -73,8 +97,8 @@ func TestVerify_ServerError_ReturnsError(t *testing.T) {
defer server.Close()
v := &Verifier{
- apiURL: server.URL,
- httpClient: server.Client(),
+ instanceURL: server.URL,
+ httpClient: server.Client(),
}
raw := detector.RawFinding{
@@ -89,6 +113,24 @@ func TestVerify_ServerError_ReturnsError(t *testing.T) {
assert.Contains(t, result.Message, "500")
}
+func TestVerify_ForbiddenIsNotInactive(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusForbidden)
+ }))
+ defer server.Close()
+
+ v := &Verifier{instanceURL: server.URL, httpClient: server.Client()}
+ raw := detector.RawFinding{
+ DetectorID: detectorID,
+ Raw: []byte("glsa_permissionlimited12345678901234567890_12345678"),
+ Redacted: "glsa_****5678",
+ }
+
+ result := v.Verify(context.Background(), raw)
+ assert.Equal(t, finding.StatusVerifyError, result.Status)
+ assert.Contains(t, result.Message, "403")
+}
+
func TestVerify_Type_ReturnsCorrectID(t *testing.T) {
v := &Verifier{}
assert.Equal(t, "grafana-api-key", v.Type())
@@ -108,3 +150,160 @@ func TestVerify_EmptyToken_ReturnsUnverified(t *testing.T) {
assert.Equal(t, finding.StatusUnverified, result.Status)
assert.Equal(t, "empty token", result.Message)
}
+
+func TestVerify_WhitespaceToken_ReturnsUnverifiedWithoutRequest(t *testing.T) {
+ requests := 0
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ requests++
+ w.WriteHeader(http.StatusOK)
+ }))
+ defer server.Close()
+
+ v := &Verifier{instanceURL: server.URL, httpClient: server.Client()}
+ result := v.Verify(context.Background(), detector.RawFinding{Raw: []byte(" \t\r\n")})
+
+ assert.Equal(t, finding.StatusUnverified, result.Status)
+ assert.Equal(t, "empty token", result.Message)
+ assert.Zero(t, requests)
+}
+
+func TestVerify_NoTrustedInstance_ReturnsUnverifiedWithoutRequest(t *testing.T) {
+ requests := 0
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ requests++
+ w.WriteHeader(http.StatusUnauthorized)
+ }))
+ defer server.Close()
+
+ for _, field := range []string{"instance_url", "grafana_url", "url", "domain"} {
+ t.Run(field, func(t *testing.T) {
+ v := &Verifier{httpClient: server.Client()}
+ raw := detector.RawFinding{
+ DetectorID: detectorID,
+ Raw: []byte("glsa_repositorycontrolled123456789012345_12345678"),
+ Redacted: "glsa_****5678",
+ ExtraData: map[string]string{field: server.URL},
+ }
+
+ result := v.Verify(context.Background(), raw)
+ assert.Equal(t, finding.StatusUnverified, result.Status)
+ assert.Equal(t, "Grafana instance URL required", result.Message)
+ })
+ }
+ assert.Zero(t, requests, "repository-provided URLs must never become verifier targets")
+}
+
+func TestVerify_CentralPortalHintCannotProduceInactive(t *testing.T) {
+ v := &Verifier{}
+ raw := detector.RawFinding{
+ DetectorID: detectorID,
+ Raw: []byte("glsa_centralportalfixture123456789012345_12345678"),
+ Redacted: "glsa_****5678",
+ ExtraData: map[string]string{"instance_url": "https://grafana.com"},
+ }
+
+ result := v.Verify(context.Background(), raw)
+ assert.Equal(t, finding.StatusUnverified, result.Status)
+ assert.NotEqual(t, finding.StatusVerifiedInactive, result.Status)
+}
+
+func TestVerify_RedirectIsNotFollowedOrInactive(t *testing.T) {
+ targetRequests := 0
+ target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ targetRequests++
+ w.WriteHeader(http.StatusUnauthorized)
+ }))
+ defer target.Close()
+
+ issuer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ http.Redirect(w, r, target.URL, http.StatusFound)
+ }))
+ defer issuer.Close()
+ v := &Verifier{instanceURL: issuer.URL}
+ raw := detector.RawFinding{
+ DetectorID: detectorID,
+ Raw: []byte("glsa_redirectfixture123456789012345678901_12345678"),
+ Redacted: "glsa_****5678",
+ }
+ result := v.Verify(context.Background(), raw)
+
+ assert.Equal(t, finding.StatusVerifyError, result.Status)
+ assert.Zero(t, targetRequests)
+}
+
+func TestVerify_InvalidSuccessBodiesAreNotActive(t *testing.T) {
+ tests := []struct {
+ name string
+ contentType string
+ body string
+ }{
+ {name: "HTML login page", contentType: "text/html", body: "Sign in"},
+ {name: "HTML content type with JSON object", contentType: "text/html", body: `{}`},
+ {name: "plain content type with JSON object", contentType: "text/plain", body: `{}`},
+ {name: "missing content type with JSON object", body: `{}`},
+ {name: "malformed JSON", contentType: "application/json", body: `{not-json`},
+ {name: "null", contentType: "application/json", body: `null`},
+ {name: "array", contentType: "application/json", body: `[]`},
+ {name: "scalar", contentType: "application/json", body: `true`},
+ {name: "trailing value", contentType: "application/json", body: `{} {}`},
+ {name: "truncated oversized object", contentType: "application/json", body: `{"permission":"` + strings.Repeat("x", 1<<20) + `"}`},
+ {name: "oversized trailing whitespace", contentType: "application/json", body: `{}` + strings.Repeat(" ", (1<<20)+1)},
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", tc.contentType)
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write([]byte(tc.body))
+ }))
+ defer server.Close()
+
+ v := &Verifier{instanceURL: server.URL, httpClient: server.Client()}
+ result := v.Verify(context.Background(), detector.RawFinding{
+ Raw: []byte("glsa_invalidbodyfixture12345678901234567_12345678"),
+ Redacted: "glsa_****5678",
+ })
+
+ assert.Equal(t, finding.StatusVerifyError, result.Status)
+ assert.NotEqual(t, finding.StatusVerifiedActive, result.Status)
+ })
+ }
+}
+
+func TestNewForTrustedInstance_Validation(t *testing.T) {
+ tests := []struct {
+ name string
+ url string
+ want string
+ }{
+ {name: "Grafana Cloud stack", url: " https://example.grafana.net/ ", want: "https://example.grafana.net"},
+ {name: "self hosted origin", url: "https://grafana.example.test:8443", want: "https://grafana.example.test:8443"},
+ }
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ v, err := NewForTrustedInstance(tc.url)
+ require.NoError(t, err)
+ assert.Equal(t, tc.want, v.instanceURL)
+ })
+ }
+
+ invalid := []string{
+ "", "grafana.example.test", "http://grafana.example.test",
+ "https://user:pass@grafana.example.test", "https://grafana.example.test/path",
+ "https://grafana.example.test?next=x", "https://grafana.example.test#fragment",
+ "https://grafana.com", "https://www.grafana.com/", "https://localhost",
+ "https://grafana.localhost", "https://127.0.0.1", "https://10.0.0.1",
+ "https://169.254.169.254", "https://[::1]", "https://*.grafana.net",
+ "https://[fe80::1%25en0]", "https://[::ffff:127.0.0.1]",
+ "https://127.1", "https://2130706433", "https://017700000001", "https://0x7f000001",
+ "https://8.8.8.8", "https://[2001:4860:4860::8888]",
+ }
+ for i, rawURL := range invalid {
+ t.Run(fmt.Sprintf("invalid-%d", i), func(t *testing.T) {
+ v, err := NewForTrustedInstance(rawURL)
+ require.Error(t, err)
+ assert.Nil(t, v)
+ })
+ }
+}
diff --git a/internal/verifier/heroku/heroku_shared_test.go b/internal/verifier/heroku/heroku_shared_test.go
new file mode 100644
index 0000000..4b63a91
--- /dev/null
+++ b/internal/verifier/heroku/heroku_shared_test.go
@@ -0,0 +1,16 @@
+package heroku
+
+import (
+ "net/http"
+ "testing"
+
+ "github.com/HodeTech/leakwatch/internal/detector"
+ "github.com/HodeTech/leakwatch/internal/verifier"
+ "github.com/HodeTech/leakwatch/internal/verifier/internal/vtest"
+)
+
+func TestVerify_SharedSafetySuite(t *testing.T) {
+ vtest.Run(t, vtest.Case{Name: "heroku", New: func(url string, client *http.Client) verifier.Verifier {
+ return &Verifier{apiURL: url, httpClient: client}
+ }, Raw: detector.RawFinding{DetectorID: detectorID, Raw: []byte("12345678-1234-abcd-9876-1234567890ab")}})
+}
diff --git a/internal/verifier/http_contract_coverage_test.go b/internal/verifier/http_contract_coverage_test.go
new file mode 100644
index 0000000..3b02155
--- /dev/null
+++ b/internal/verifier/http_contract_coverage_test.go
@@ -0,0 +1,178 @@
+package verifier_test
+
+import (
+ "go/ast"
+ "go/build"
+ "go/parser"
+ "go/token"
+ "path/filepath"
+ "sort"
+ "strconv"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+var expectedHTTPVerifierPackages = []string{
+ "airtable", "anthropic", "auth0", "bitbucket", "circleci", "cloudflare",
+ "databricks", "datadog", "deepseek", "digitalocean", "discord", "dockerhub",
+ "doppler", "figma", "github", "gitlab", "grafana", "heroku", "huggingface",
+ "infura", "launchdarkly", "linear", "mailgun", "newrelic", "notion", "npm",
+ "okta", "openai", "pagerduty", "postmark", "pypi", "rubygems", "sendgrid",
+ "sentry", "shopify", "slack", "snyk", "sonarcloud", "stripe", "supabase",
+ "teams", "telegram", "terraform", "twilio", "vercel",
+}
+
+// TestHTTPVerifierPackages_UseSharedSafetySuite ties network discovery to exact
+// package identities and to a directly executed top-level vtest call. A raw
+// comment marker, an aliased import, a helper hidden in a non-verifier filename,
+// or a dead nested function cannot satisfy the guard.
+func TestHTTPVerifierPackages_UseSharedSafetySuite(t *testing.T) {
+ entries, err := filepath.Glob("*")
+ require.NoError(t, err)
+ discovered := make([]string, 0, len(expectedHTTPVerifierPackages))
+ for _, entry := range entries {
+ info, statErr := filepath.Glob(filepath.Join(entry, "*.go"))
+ require.NoError(t, statErr)
+ if len(info) == 0 || entry == "internal" || !packageUsesHTTP(t, entry) {
+ continue
+ }
+ discovered = append(discovered, entry)
+ assert.True(t, packageRunsVTestDirectly(t, entry),
+ "HTTP verifier package %q must execute vtest.Run as the first statement of a top-level Test function", entry)
+ }
+ sort.Strings(discovered)
+ assert.Equal(t, expectedHTTPVerifierPackages, discovered,
+ "HTTP verifier package identities changed; add/remove an executable safety contract deliberately")
+}
+
+func packageUsesHTTP(t *testing.T, directory string) bool {
+ t.Helper()
+ paths := activeGoFiles(t, directory, false)
+ for _, path := range paths {
+ file := parseGoFile(t, path)
+ httpxAliases := importAliases(file, "github.com/HodeTech/leakwatch/internal/verifier/internal/httpx", "httpx")
+ httpAliases := importAliases(file, "net/http", "http")
+ importsHTTP := len(httpAliases) > 0
+ found := false
+ ast.Inspect(file, func(node ast.Node) bool {
+ call, ok := node.(*ast.CallExpr)
+ if !ok {
+ return true
+ }
+ selector, ok := call.Fun.(*ast.SelectorExpr)
+ if !ok {
+ return true
+ }
+ receiver, _ := selector.X.(*ast.Ident)
+ if receiver != nil && httpxAliases[receiver.Name] && selector.Sel.Name == "VerifyToken" {
+ found = true
+ return false
+ }
+ if receiver != nil && httpAliases[receiver.Name] && (selector.Sel.Name == "NewRequestWithContext" || selector.Sel.Name == "NewRequest") {
+ found = true
+ return false
+ }
+ // Direct clients and transports may be stored behind a local field or
+ // interface, so their receiver is not the net/http import identifier.
+ // When the production file imports net/http, recognize the outbound
+ // method surface as network use as well.
+ if importsHTTP && (selector.Sel.Name == "Do" || selector.Sel.Name == "RoundTrip" ||
+ selector.Sel.Name == "Get" || selector.Sel.Name == "Post" || selector.Sel.Name == "PostForm" ||
+ selector.Sel.Name == "Head") {
+ found = true
+ return false
+ }
+ return true
+ })
+ if found {
+ return true
+ }
+ }
+ return false
+}
+
+func packageRunsVTestDirectly(t *testing.T, directory string) bool {
+ t.Helper()
+ paths := activeGoFiles(t, directory, true)
+ for _, path := range paths {
+ file := parseGoFile(t, path)
+ aliases := importAliases(file, "github.com/HodeTech/leakwatch/internal/verifier/internal/vtest", "vtest")
+ if len(aliases) == 0 {
+ continue
+ }
+ for _, declaration := range file.Decls {
+ function, ok := declaration.(*ast.FuncDecl)
+ if !ok || function.Recv != nil || !strings.HasPrefix(function.Name.Name, "Test") ||
+ function.Body == nil || len(function.Body.List) == 0 {
+ continue
+ }
+ expression, ok := function.Body.List[0].(*ast.ExprStmt)
+ if !ok {
+ continue
+ }
+ call, ok := expression.X.(*ast.CallExpr)
+ if !ok {
+ continue
+ }
+ selector, ok := call.Fun.(*ast.SelectorExpr)
+ if !ok || selector.Sel.Name != "Run" {
+ continue
+ }
+ receiver, ok := selector.X.(*ast.Ident)
+ if ok && aliases[receiver.Name] {
+ return true
+ }
+ }
+ }
+ return false
+}
+
+func activeGoFiles(t *testing.T, directory string, tests bool) []string {
+ t.Helper()
+ pattern := "*.go"
+ if tests {
+ pattern = "*_test.go"
+ }
+ paths, err := filepath.Glob(filepath.Join(directory, pattern))
+ require.NoError(t, err)
+ active := make([]string, 0, len(paths))
+ for _, path := range paths {
+ if !tests && strings.HasSuffix(path, "_test.go") {
+ continue
+ }
+ matched, matchErr := build.Default.MatchFile(directory, filepath.Base(path))
+ require.NoError(t, matchErr, path)
+ if matched {
+ active = append(active, path)
+ }
+ }
+ return active
+}
+
+func importAliases(file *ast.File, importPath, defaultName string) map[string]bool {
+ aliases := make(map[string]bool)
+ for _, spec := range file.Imports {
+ path, err := strconv.Unquote(spec.Path.Value)
+ if err != nil || path != importPath {
+ continue
+ }
+ name := defaultName
+ if spec.Name != nil {
+ name = spec.Name.Name
+ }
+ if name != "." && name != "_" {
+ aliases[name] = true
+ }
+ }
+ return aliases
+}
+
+func parseGoFile(t *testing.T, path string) *ast.File {
+ t.Helper()
+ file, err := parser.ParseFile(token.NewFileSet(), path, nil, 0)
+ require.NoError(t, err)
+ return file
+}
diff --git a/internal/verifier/huggingface/huggingface_shared_test.go b/internal/verifier/huggingface/huggingface_shared_test.go
new file mode 100644
index 0000000..66efe71
--- /dev/null
+++ b/internal/verifier/huggingface/huggingface_shared_test.go
@@ -0,0 +1,16 @@
+package huggingface
+
+import (
+ "net/http"
+ "testing"
+
+ "github.com/HodeTech/leakwatch/internal/detector"
+ "github.com/HodeTech/leakwatch/internal/verifier"
+ "github.com/HodeTech/leakwatch/internal/verifier/internal/vtest"
+)
+
+func TestVerify_SharedSafetySuite(t *testing.T) {
+ vtest.Run(t, vtest.Case{Name: "huggingface", New: func(url string, client *http.Client) verifier.Verifier {
+ return &Verifier{apiURL: url, httpClient: client}
+ }, Raw: detector.RawFinding{DetectorID: detectorID, Raw: []byte("hf_SyntheticHuggingFaceCredentialFixture1234")}})
+}
diff --git a/internal/verifier/infura/infura_shared_test.go b/internal/verifier/infura/infura_shared_test.go
new file mode 100644
index 0000000..a8afe0f
--- /dev/null
+++ b/internal/verifier/infura/infura_shared_test.go
@@ -0,0 +1,16 @@
+package infura
+
+import (
+ "net/http"
+ "testing"
+
+ "github.com/HodeTech/leakwatch/internal/detector"
+ "github.com/HodeTech/leakwatch/internal/verifier"
+ "github.com/HodeTech/leakwatch/internal/verifier/internal/vtest"
+)
+
+func TestVerify_SharedSafetySuite(t *testing.T) {
+ vtest.Run(t, vtest.Case{Name: "infura", New: func(url string, client *http.Client) verifier.Verifier {
+ return &Verifier{apiURL: url, httpClient: client}
+ }, Raw: detector.RawFinding{DetectorID: detectorID, Raw: []byte("abcdef0123456789abcdef0123456789")}})
+}
diff --git a/internal/verifier/internal/httpx/httpx.go b/internal/verifier/internal/httpx/httpx.go
index a8e703a..9dc1fb7 100644
--- a/internal/verifier/internal/httpx/httpx.go
+++ b/internal/verifier/internal/httpx/httpx.go
@@ -29,9 +29,12 @@
// relying on the crypto/tls default, as defense-in-depth and
// self-documentation.
//
-// This helper deliberately does NOT implement retry, backoff, or per-provider
-// rate limiting. Those concerns are handled (or deferred) elsewhere; keeping
-// this package focused on transport safety.
+// The shared token helper performs at most one Retry-After-aware HTTP 429 retry
+// for safe GET/HEAD probes. The wait is strictly bounded, context-aware, and
+// passes through an engine-owned admission gate immediately before every actual
+// send when verification runs through the engine. Format-only, empty-input, and
+// missing-context paths consume no limiter capacity. Unsafe methods and
+// missing/invalid Retry-After responses are never retried.
package httpx
import (
@@ -129,7 +132,10 @@ func RedactError(err error, secret string) string {
if err == nil {
return ""
}
- msg := err.Error()
+ return redactText(err.Error(), secret)
+}
+
+func redactText(msg, secret string) string {
if secret == "" {
return msg
}
diff --git a/internal/verifier/internal/httpx/verify.go b/internal/verifier/internal/httpx/verify.go
index 48001ce..b0d29a7 100644
--- a/internal/verifier/internal/httpx/verify.go
+++ b/internal/verifier/internal/httpx/verify.go
@@ -3,10 +3,17 @@ package httpx
import (
"bytes"
"context"
+ cryptorand "crypto/rand"
+ "encoding/base64"
+ "encoding/binary"
"fmt"
"io"
"log/slog"
+ "mime"
"net/http"
+ "strconv"
+ "strings"
+ "time"
"github.com/HodeTech/leakwatch/pkg/finding"
)
@@ -14,9 +21,49 @@ import (
// userAgent is the User-Agent every verifier request carries.
const userAgent = "leakwatch-verifier"
-// Request describes the single HTTP request a verifier sends to a provider.
+const (
+ max429Attempts = 2
+ max429RetryWait = 2 * time.Second
+ maxJitterRatio = 0.10
+)
+
+type retryGateKey struct{}
+
+type requestGateKey struct{}
+
+// RequestGate is installed by the verification engine for standard httpx
+// verifiers. It admits each actual send through both provider and global rate
+// limiters. Returning a non-nil result rejects the request before client.Do.
+type RequestGate func() *finding.VerificationResult
+
+// WithRequestGate returns a child context carrying the engine-owned admission
+// gate used immediately before every HTTP attempt, including a bounded retry.
+func WithRequestGate(ctx context.Context, gate RequestGate) context.Context {
+ if gate == nil {
+ return ctx
+ }
+ return context.WithValue(ctx, requestGateKey{}, gate)
+}
+
+// RetryGate is installed by the verification engine and admits one retry at
+// its actual send point through both the provider and global rate limiters.
+// Returning a non-nil result rejects the retry without sending it.
+type RetryGate func() *finding.VerificationResult
+
+// WithRetryGate returns a child context carrying an engine-owned admission gate
+// used only for an HTTP 429 retry. It supports manually gated multi-request
+// verifiers whose initial/fallback sends call their verifier.RequestGate
+// directly; standard httpx verifiers use WithRequestGate instead.
+func WithRetryGate(ctx context.Context, gate RetryGate) context.Context {
+ if gate == nil {
+ return ctx
+ }
+ return context.WithValue(ctx, retryGateKey{}, gate)
+}
+
+// Request describes the logical HTTP probe a verifier sends to a provider.
// VerifyToken builds and performs it through the shared, security-hardened
-// client, so callers only declare what differs between providers.
+// client; a safe GET/HEAD probe may be replayed once after a bounded HTTP 429.
type Request struct {
// Method defaults to GET when empty.
Method string
@@ -42,8 +89,21 @@ type Request struct {
// The reader passed to a DecodeFunc is already bounded by LimitReader.
type DecodeFunc func(body io.Reader) (extra map[string]string, downgradeMessage string, err error)
-// TokenSpec describes a standard single-request token verification: the request
-// to send and how each response status maps to a VerificationResult.
+// ResponseDecodeFunc is the header-aware counterpart to DecodeFunc. It is for
+// provider metadata that is available only in response headers (for example
+// GitHub OAuth scopes and token expiry). Header values are untrusted provider
+// input and must never be copied wholesale into ExtraData.
+type ResponseDecodeFunc func(header http.Header, body io.Reader) (extra map[string]string, downgradeMessage string, err error)
+
+// InactiveDecodeFunc validates that an inactive-status response is definitive
+// for the provider. A nil error permits verified-inactive; an error keeps the
+// outcome fail-conservative as StatusVerifyError. It is intended for providers
+// whose 401 class also includes challenges such as DPoP that do not prove a
+// credential is invalid.
+type InactiveDecodeFunc func(body io.Reader) error
+
+// TokenSpec describes a standard token verification probe: the request to send
+// and how each response status maps to a VerificationResult.
//
// The shared flow — User-Agent, no-redirect handling, bounded body, error
// redaction, and the canonical "unexpected status code" / "failed to decode"
@@ -57,9 +117,9 @@ type TokenSpec struct {
// Request is the provider request to send.
Request Request
- // Redact, when non-empty, is stripped from any error text before it is
- // logged or returned. Set it to the secret when the credential appears in
- // the request URL (token-in-path verifiers such as telegram and infura).
+ // Redact is an optional additional sensitive value stripped from error text.
+ // VerifyToken always redacts its token argument; use this field only when a
+ // request contains another credential-bearing representation.
Redact string
// ActiveStatuses are the HTTP status codes mapped to verified-active.
@@ -86,6 +146,26 @@ type TokenSpec struct {
// extract ExtraData (and optionally downgrade the result). When nil, an
// active-status response yields a bare active result without reading the body.
Decode DecodeFunc
+
+ // DecodeResponse is mutually exclusive with Decode and receives both the
+ // active response headers and bounded body. Use it only for explicitly
+ // validated, non-secret provider metadata.
+ DecodeResponse ResponseDecodeFunc
+
+ // DecodeInactive, when non-nil, must positively validate an inactive-status
+ // response body before the credential is classified as inactive. The body is
+ // read completely through the same strict size bound used for active bodies.
+ DecodeInactive InactiveDecodeFunc
+
+ // RequireCompleteBody reads the full active response through a strict
+ // MaxBodyBytes+1 bound before Decode runs. Responses over the bound are
+ // rejected instead of letting a truncated prefix appear valid.
+ RequireCompleteBody bool
+
+ // RequireJSONContentType rejects an active response unless its media type is
+ // application/json or an application/*+json subtype. The raw header is never
+ // reflected into logs or result messages.
+ RequireJSONContentType bool
}
// BaseURL returns override when it is non-empty, otherwise fallback. Verifiers
@@ -109,29 +189,71 @@ func VerifyToken(ctx context.Context, client *http.Client, token string, spec To
}
}
- resp, errResult := spec.send(ctx, client)
- if errResult != nil {
- return *errResult
+ for attempt := 1; attempt <= max429Attempts; attempt++ {
+ resp, errResult := spec.send(ctx, client, token)
+ if errResult != nil {
+ return *errResult
+ }
+
+ if resp.StatusCode == http.StatusTooManyRequests && attempt < max429Attempts && spec.retrySafe() {
+ rawRetryAfter := resp.Header.Get("Retry-After")
+ delay, ok := retryAfterDelay(rawRetryAfter, time.Now())
+ closeResponse(resp)
+ if ok && delay <= max429RetryWait {
+ delay = addRetryJitter(delay, max429RetryWait, retryJitterUnit())
+ if retryFitsContext(ctx, delay) {
+ slog.DebugContext(ctx, spec.Name+" verifier: scheduling bounded HTTP 429 retry",
+ slog.Int("next_attempt", attempt+1),
+ slog.Int("max_attempts", max429Attempts),
+ slog.Duration("retry_wait", delay),
+ slog.Duration("max_total_wait", max429RetryWait),
+ )
+ if result := waitForRetry(ctx, spec.Name, delay); result != nil {
+ return *result
+ }
+ if gate, _ := ctx.Value(retryGateKey{}).(RetryGate); gate != nil {
+ if rejection := gate(); rejection != nil {
+ return *rejection
+ }
+ }
+ continue
+ }
+ }
+ slog.DebugContext(ctx, spec.Name+" verifier: bounded HTTP 429 retry skipped",
+ slog.Int("attempt", attempt),
+ slog.Int("max_attempts", max429Attempts),
+ slog.Duration("max_total_wait", max429RetryWait),
+ )
+ return RateLimited(ctx, spec.Name, rawRetryAfter)
+ }
+
+ return func() finding.VerificationResult {
+ // Decode callbacks are provider-specific code. Preserve response-body
+ // cleanup even if one panics and the outer verification engine recovers.
+ defer closeResponse(resp)
+ return spec.handleResponse(ctx, resp)
+ }()
+ }
+
+ return finding.VerificationResult{
+ Status: finding.StatusVerifyError,
+ Message: "bounded verification attempts exhausted",
}
- defer func() {
- // Drain the remaining body (bounded by MaxBodyBytes so a misbehaving
- // endpoint cannot exhaust memory) before closing, so the underlying
- // connection can be returned to the shared client's keep-alive pool
- // instead of being discarded.
- _, _ = io.Copy(io.Discard, LimitReader(resp.Body))
- _ = resp.Body.Close()
- }()
+}
+func (spec TokenSpec) handleResponse(ctx context.Context, resp *http.Response) finding.VerificationResult {
code := resp.StatusCode
switch {
case containsStatus(spec.activeStatuses(), code):
- return spec.handleActive(ctx, resp.Body)
- case containsStatus(spec.inactiveStatuses(), code):
- slog.DebugContext(ctx, spec.Name+" verifier: secret is inactive")
- return finding.VerificationResult{
- Status: finding.StatusVerifiedInactive,
- Message: spec.InactiveMessage,
+ if spec.RequireJSONContentType && !isJSONContentType(resp.Header.Get("Content-Type")) {
+ return finding.VerificationResult{
+ Status: finding.StatusVerifyError,
+ Message: "200 OK but response Content-Type is not JSON",
+ }
}
+ return spec.handleActive(ctx, resp)
+ case containsStatus(spec.inactiveStatuses(), code):
+ return spec.handleInactive(ctx, resp)
case code == http.StatusTooManyRequests:
// A provider-side rate limit must be distinguishable from a genuine
// verification bug: report an actionable message rather than a generic
@@ -142,10 +264,114 @@ func VerifyToken(ctx context.Context, client *http.Client, token string, spec To
}
}
+func (spec TokenSpec) retrySafe() bool {
+ method := strings.ToUpper(strings.TrimSpace(spec.Request.Method))
+ if method == "" {
+ method = http.MethodGet
+ }
+ return method == http.MethodGet || method == http.MethodHead
+}
+
+func retryAfterDelay(raw string, now time.Time) (time.Duration, bool) {
+ value := strings.TrimSpace(raw)
+ if value == "" {
+ return 0, false
+ }
+ if seconds, err := strconv.ParseUint(value, 10, 64); err == nil {
+ if seconds > uint64(max429RetryWait/time.Second) {
+ return max429RetryWait + time.Nanosecond, true
+ }
+ return time.Duration(seconds) * time.Second, true
+ }
+ retryAt, err := http.ParseTime(value)
+ if err != nil {
+ return 0, false
+ }
+ delay := retryAt.Sub(now)
+ if delay < 0 {
+ delay = 0
+ }
+ return delay, true
+}
+
+func addRetryJitter(base, maxDelay time.Duration, unit float64) time.Duration {
+ if base <= 0 || maxDelay <= base {
+ return base
+ }
+ if unit < 0 {
+ unit = 0
+ }
+ if unit > 1 {
+ unit = 1
+ }
+ jitter := time.Duration(float64(base) * maxJitterRatio * unit)
+ if base+jitter > maxDelay {
+ return maxDelay
+ }
+ return base + jitter
+}
+
+func retryJitterUnit() float64 {
+ var random [8]byte
+ if _, err := cryptorand.Read(random[:]); err != nil {
+ // Jitter improves herd behavior but is not required for correctness. A
+ // secure-random failure therefore falls back to the provider's exact wait
+ // instead of introducing a weak pseudo-random source.
+ return 0
+ }
+ const precisionBits = 53
+ value := binary.LittleEndian.Uint64(random[:]) >> (64 - precisionBits)
+ return float64(value) / float64((uint64(1)< MaxBodyBytes {
+ return finding.VerificationResult{
+ Status: finding.StatusVerifyError,
+ Message: fmt.Sprintf("200 OK but response body exceeds %d bytes", MaxBodyBytes),
+ }
+ }
+ decodeBody = bytes.NewReader(contents)
+ }
+
+ var extra map[string]string
+ var downgrade string
+ var err error
+ if spec.DecodeResponse != nil {
+ extra, downgrade, err = spec.DecodeResponse(resp.Header.Clone(), decodeBody)
+ } else {
+ extra, downgrade, err = spec.Decode(decodeBody)
+ }
if err != nil {
slog.ErrorContext(ctx, spec.Name+" verifier: failed to decode response", slog.String("error", err.Error()))
return finding.VerificationResult{
@@ -238,12 +522,50 @@ func (spec TokenSpec) handleActive(ctx context.Context, body io.Reader) finding.
}
}
+func (spec TokenSpec) handleInactive(ctx context.Context, resp *http.Response) finding.VerificationResult {
+ if spec.RequireJSONContentType && !isJSONContentType(resp.Header.Get("Content-Type")) {
+ return finding.VerificationResult{
+ Status: finding.StatusVerifyError,
+ Message: fmt.Sprintf("HTTP %d inactive response Content-Type is not JSON", resp.StatusCode),
+ }
+ }
+ if spec.DecodeInactive != nil {
+ contents, err := io.ReadAll(io.LimitReader(resp.Body, MaxBodyBytes+1))
+ if err != nil {
+ return finding.VerificationResult{
+ Status: finding.StatusVerifyError,
+ Message: fmt.Sprintf("HTTP %d but failed to read response body: %v", resp.StatusCode, err),
+ }
+ }
+ if int64(len(contents)) > MaxBodyBytes {
+ return finding.VerificationResult{
+ Status: finding.StatusVerifyError,
+ Message: fmt.Sprintf("HTTP %d but response body exceeds %d bytes", resp.StatusCode, MaxBodyBytes),
+ }
+ }
+ if err := spec.DecodeInactive(bytes.NewReader(contents)); err != nil {
+ slog.DebugContext(ctx, spec.Name+" verifier: inactive response was not definitive")
+ return finding.VerificationResult{
+ Status: finding.StatusVerifyError,
+ Message: fmt.Sprintf("HTTP %d did not definitively prove the secret inactive", resp.StatusCode),
+ }
+ }
+ }
+
+ slog.DebugContext(ctx, spec.Name+" verifier: secret is inactive")
+ return finding.VerificationResult{
+ Status: finding.StatusVerifiedInactive,
+ Message: spec.InactiveMessage,
+ }
+}
+
// RateLimited returns a distinguished StatusVerifyError result for an HTTP 429
// (Too Many Requests) response, so a provider-side rate limit is never conflated
-// with a genuine verification bug or an inactive secret. retryAfter is the raw
-// Retry-After header value (may be empty); it is echoed into the message to make
-// the outcome actionable and never contains secret material.
+// with a genuine verification bug or an inactive secret. Only a syntactically
+// valid delta-seconds or HTTP-date Retry-After value is emitted; arbitrary
+// provider-controlled header text is never copied into logs or results.
func RateLimited(ctx context.Context, name, retryAfter string) finding.VerificationResult {
+ retryAfter = canonicalRetryAfter(retryAfter)
msg := "rate limited by provider (HTTP 429), retry later"
if retryAfter != "" {
msg = fmt.Sprintf("%s (Retry-After: %s)", msg, retryAfter)
@@ -256,6 +578,20 @@ func RateLimited(ctx context.Context, name, retryAfter string) finding.Verificat
}
}
+func canonicalRetryAfter(raw string) string {
+ value := strings.TrimSpace(raw)
+ if value == "" {
+ return ""
+ }
+ if seconds, err := strconv.ParseUint(value, 10, 64); err == nil {
+ return strconv.FormatUint(seconds, 10)
+ }
+ if retryAt, err := http.ParseTime(value); err == nil {
+ return retryAt.UTC().Format(http.TimeFormat)
+ }
+ return ""
+}
+
// UnexpectedStatus returns the canonical StatusVerifyError result for a response
// status code that a verifier does not recognize.
func UnexpectedStatus(ctx context.Context, name string, code int) finding.VerificationResult {
diff --git a/internal/verifier/internal/httpx/verify_test.go b/internal/verifier/internal/httpx/verify_test.go
index 5fcd966..a5d98d2 100644
--- a/internal/verifier/internal/httpx/verify_test.go
+++ b/internal/verifier/internal/httpx/verify_test.go
@@ -1,21 +1,45 @@
package httpx
import (
+ "bytes"
"context"
+ "encoding/base64"
"encoding/json"
"errors"
+ "fmt"
"io"
+ "log/slog"
"net/http"
"net/http/httptest"
+ "strings"
+ "sync/atomic"
"testing"
+ "time"
"github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
"github.com/HodeTech/leakwatch/pkg/finding"
)
const testToken = "test-token-1234567890"
+type roundTripFunc func(*http.Request) (*http.Response, error)
+
+func (f roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) {
+ return f(request)
+}
+
+type closeTrackingBody struct {
+ io.Reader
+ closed *atomic.Bool
+}
+
+func (b *closeTrackingBody) Close() error {
+ b.closed.Store(true)
+ return nil
+}
+
// jsonServer returns a test server that responds with the given status code and
// body, asserting nothing about the request.
func jsonServer(t *testing.T, status int, body string) *httptest.Server {
@@ -50,6 +74,40 @@ func TestVerifyToken_Active_NoDecode_NoExtra(t *testing.T) {
assert.Nil(t, res.ExtraData)
}
+func TestVerifyToken_RequiredJSONContentType(t *testing.T) {
+ tests := []struct {
+ contentType string
+ want finding.VerificationStatus
+ }{
+ {contentType: "application/json", want: finding.StatusVerifiedActive},
+ {contentType: "application/json; charset=utf-8", want: finding.StatusVerifiedActive},
+ {contentType: "application/problem+json", want: finding.StatusVerifiedActive},
+ {contentType: "text/html", want: finding.StatusVerifyError},
+ {contentType: "", want: finding.StatusVerifyError},
+ {contentType: "not a media type", want: finding.StatusVerifyError},
+ }
+ for _, tc := range tests {
+ t.Run(tc.contentType, func(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ if tc.contentType != "" {
+ w.Header().Set("Content-Type", tc.contentType)
+ }
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write([]byte(`{}`))
+ }))
+ defer server.Close()
+
+ result := VerifyToken(context.Background(), server.Client(), testToken, TokenSpec{
+ Name: "x",
+ Request: Request{URL: server.URL},
+ ActiveMessage: "active",
+ RequireJSONContentType: true,
+ })
+ assert.Equal(t, tc.want, result.Status)
+ })
+ }
+}
+
func TestVerifyToken_Active_NoDecode_WithActiveExtra(t *testing.T) {
server := jsonServer(t, http.StatusOK, `{}`)
defer server.Close()
@@ -86,6 +144,45 @@ func TestVerifyToken_Active_Decode_PopulatesExtra(t *testing.T) {
assert.Equal(t, "alice", res.ExtraData["name"])
}
+func TestVerifyToken_ResponseDecoderReceivesHeadersAndBoundedBody(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ w.Header().Set("X-Safe-Metadata", "scope-a")
+ _, _ = w.Write([]byte(`{"ok":true}`))
+ }))
+ defer server.Close()
+
+ result := VerifyToken(context.Background(), server.Client(), testToken, TokenSpec{
+ Name: "header-aware",
+ Request: Request{URL: server.URL},
+ DecodeResponse: func(header http.Header, body io.Reader) (map[string]string, string, error) {
+ contents, err := io.ReadAll(body)
+ require.NoError(t, err)
+ assert.JSONEq(t, `{"ok":true}`, string(contents))
+ return map[string]string{"scope": header.Get("X-Safe-Metadata")}, "", nil
+ },
+ RequireCompleteBody: true,
+ RequireJSONContentType: true,
+ })
+
+ require.Equal(t, finding.StatusVerifiedActive, result.Status)
+ assert.Equal(t, "scope-a", result.ExtraData["scope"])
+}
+
+func TestVerifyToken_RejectsMultipleActiveDecoders(t *testing.T) {
+ server := jsonServer(t, http.StatusOK, `{}`)
+ defer server.Close()
+
+ decode := func(io.Reader) (map[string]string, string, error) { return nil, "", nil }
+ result := VerifyToken(context.Background(), server.Client(), testToken, TokenSpec{
+ Name: "invalid-contract",
+ Request: Request{URL: server.URL},
+ Decode: decode,
+ DecodeResponse: func(http.Header, io.Reader) (map[string]string, string, error) { return nil, "", nil },
+ })
+ assert.Equal(t, finding.StatusVerifyError, result.Status)
+}
+
func TestVerifyToken_Decode_DowngradesToInactive(t *testing.T) {
server := jsonServer(t, http.StatusOK, `{"ok":false}`)
defer server.Close()
@@ -144,6 +241,75 @@ func TestVerifyToken_Inactive_CustomStatus403(t *testing.T) {
assert.Equal(t, finding.StatusVerifiedInactive, res.Status)
}
+func TestVerifyToken_InactiveRequiresDefinitiveBodyWhenConfigured(t *testing.T) {
+ tests := []struct {
+ name string
+ contentType string
+ body string
+ decodeErr error
+ want finding.VerificationStatus
+ }{
+ {name: "definitive json", contentType: "application/json", body: `{"message":"revoked"}`, want: finding.StatusVerifiedInactive},
+ {name: "ambiguous json", contentType: "application/json", body: `{"error":"challenge"}`, decodeErr: errors.New("not definitive"), want: finding.StatusVerifyError},
+ {name: "wrong content type", contentType: "text/html", body: `{"message":"revoked"}`, want: finding.StatusVerifyError},
+ }
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", tc.contentType)
+ w.WriteHeader(http.StatusUnauthorized)
+ _, _ = w.Write([]byte(tc.body))
+ }))
+ defer server.Close()
+
+ res := VerifyToken(context.Background(), server.Client(), testToken, TokenSpec{
+ Name: "x",
+ Request: Request{URL: server.URL},
+ InactiveMessage: "secret revoked",
+ RequireJSONContentType: true,
+ DecodeInactive: func(io.Reader) error {
+ return tc.decodeErr
+ },
+ })
+ assert.Equal(t, tc.want, res.Status)
+ })
+ }
+}
+
+func TestVerifyToken_InactiveRequiresJSONWithoutDecoder(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", "text/html")
+ w.WriteHeader(http.StatusUnauthorized)
+ _, _ = w.Write([]byte(`proxy login`))
+ }))
+ defer server.Close()
+
+ result := VerifyToken(context.Background(), server.Client(), testToken, TokenSpec{
+ Name: "x",
+ Request: Request{URL: server.URL},
+ InactiveMessage: "secret revoked",
+ RequireJSONContentType: true,
+ })
+ assert.Equal(t, finding.StatusVerifyError, result.Status)
+}
+
+func TestVerifyToken_InactiveBodyIsBounded(t *testing.T) {
+ server := jsonServer(t, http.StatusUnauthorized, strings.Repeat("x", int(MaxBodyBytes)+1))
+ defer server.Close()
+
+ result := VerifyToken(context.Background(), server.Client(), testToken, TokenSpec{
+ Name: "x",
+ Request: Request{URL: server.URL},
+ RequireJSONContentType: true,
+ DecodeInactive: func(io.Reader) error {
+ t.Fatal("oversized inactive body must not reach the decoder")
+ return nil
+ },
+ })
+ assert.Equal(t, finding.StatusVerifyError, result.Status)
+ assert.Contains(t, result.Message, "exceeds")
+}
+
func TestVerifyToken_CustomActiveStatus405(t *testing.T) {
server := jsonServer(t, http.StatusMethodNotAllowed, `{}`)
defer server.Close()
@@ -195,8 +361,10 @@ func TestVerifyToken_RateLimited_429IsDistinguished(t *testing.T) {
defer server.Close()
res := VerifyToken(context.Background(), server.Client(), testToken, TokenSpec{
- Name: "x",
- Request: Request{URL: server.URL},
+ Name: "x",
+ // POST is deliberately not retried: the shared helper only retries safe
+ // GET/HEAD probes.
+ Request: Request{Method: http.MethodPost, URL: server.URL},
})
assert.Equal(t, finding.StatusVerifyError, res.Status)
assert.Contains(t, res.Message, "rate limited by provider")
@@ -206,6 +374,282 @@ func TestVerifyToken_RateLimited_429IsDistinguished(t *testing.T) {
assert.NotContains(t, res.Message, "unexpected status code")
}
+func TestVerifyToken_RateLimitedGETRetriesOnceAndSucceeds(t *testing.T) {
+ var logs bytes.Buffer
+ previous := slog.Default()
+ slog.SetDefault(slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug})))
+ t.Cleanup(func() { slog.SetDefault(previous) })
+
+ var calls atomic.Int64
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ if calls.Add(1) == 1 {
+ w.Header().Set("Retry-After", "0")
+ w.WriteHeader(http.StatusTooManyRequests)
+ return
+ }
+ w.WriteHeader(http.StatusOK)
+ }))
+ defer server.Close()
+
+ result := VerifyToken(context.Background(), server.Client(), testToken, TokenSpec{
+ Name: "x",
+ Request: Request{URL: server.URL},
+ ActiveMessage: "active after retry",
+ })
+ assert.Equal(t, finding.StatusVerifiedActive, result.Status)
+ assert.Equal(t, int64(2), calls.Load())
+ assert.Contains(t, logs.String(), "max_attempts=2")
+ assert.Contains(t, logs.String(), "max_total_wait=2s")
+ assert.NotContains(t, logs.String(), testToken)
+}
+
+func TestVerifyToken_RateLimitedGETHasStrictAttemptBound(t *testing.T) {
+ var calls atomic.Int64
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ calls.Add(1)
+ w.Header().Set("Retry-After", "0")
+ w.WriteHeader(http.StatusTooManyRequests)
+ }))
+ defer server.Close()
+
+ result := VerifyToken(context.Background(), server.Client(), testToken, TokenSpec{
+ Name: "x",
+ Request: Request{URL: server.URL},
+ })
+ assert.Equal(t, finding.StatusVerifyError, result.Status)
+ assert.Contains(t, result.Message, "rate limited")
+ assert.Equal(t, int64(max429Attempts), calls.Load())
+}
+
+func TestVerifyToken_RateLimitedDoesNotRetryUnsafeOrUnscheduledRequests(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ method string
+ retryAfter string
+ }{
+ {name: "unsafe post", method: http.MethodPost, retryAfter: "0"},
+ {name: "missing Retry-After", method: http.MethodGet},
+ {name: "invalid Retry-After", method: http.MethodGet, retryAfter: "not-a-delay"},
+ {name: "wait exceeds bound", method: http.MethodGet, retryAfter: "3"},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ var calls atomic.Int64
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ calls.Add(1)
+ if tc.retryAfter != "" {
+ w.Header().Set("Retry-After", tc.retryAfter)
+ }
+ w.WriteHeader(http.StatusTooManyRequests)
+ }))
+ defer server.Close()
+
+ result := VerifyToken(context.Background(), server.Client(), testToken, TokenSpec{
+ Name: "x",
+ Request: Request{Method: tc.method, URL: server.URL},
+ })
+ assert.Equal(t, finding.StatusVerifyError, result.Status)
+ assert.Equal(t, int64(1), calls.Load())
+ })
+ }
+}
+
+func TestVerifyToken_RetryDeadlineAndAdmissionGate(t *testing.T) {
+ t.Run("request gate admits every actual send", func(t *testing.T) {
+ var calls atomic.Int64
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ if calls.Add(1) == 1 {
+ w.Header().Set("Retry-After", "0")
+ w.WriteHeader(http.StatusTooManyRequests)
+ return
+ }
+ w.WriteHeader(http.StatusOK)
+ }))
+ defer server.Close()
+
+ gateCalls := 0
+ ctx := WithRequestGate(context.Background(), func() *finding.VerificationResult {
+ gateCalls++
+ return nil
+ })
+ result := VerifyToken(ctx, server.Client(), testToken, TokenSpec{Name: "x", Request: Request{URL: server.URL}})
+ assert.Equal(t, finding.StatusVerifiedActive, result.Status)
+ assert.Equal(t, int64(2), calls.Load())
+ assert.Equal(t, 2, gateCalls)
+ })
+
+ t.Run("request gate rejection prevents initial transport", func(t *testing.T) {
+ var calls atomic.Int64
+ client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
+ calls.Add(1)
+ return nil, errors.New("transport must not run")
+ })}
+ ctx := WithRequestGate(context.Background(), func() *finding.VerificationResult {
+ return &finding.VerificationResult{Status: finding.StatusVerifyError, Message: "admission rejected"}
+ })
+
+ result := VerifyToken(ctx, client, testToken, TokenSpec{
+ Name: "x",
+ Request: Request{URL: "https://provider.example.test/probe"},
+ })
+ assert.Equal(t, finding.StatusVerifyError, result.Status)
+ assert.Equal(t, "admission rejected", result.Message)
+ assert.Zero(t, calls.Load())
+ })
+
+ t.Run("empty token consumes no request admission", func(t *testing.T) {
+ gateCalls := 0
+ ctx := WithRequestGate(context.Background(), func() *finding.VerificationResult {
+ gateCalls++
+ return nil
+ })
+ result := VerifyToken(ctx, nil, "", TokenSpec{Name: "x", Request: Request{URL: "https://provider.example.test"}})
+ assert.Equal(t, finding.StatusUnverified, result.Status)
+ assert.Zero(t, gateCalls)
+ })
+
+ t.Run("deadline too short prevents wait and request", func(t *testing.T) {
+ var calls atomic.Int64
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ calls.Add(1)
+ w.Header().Set("Retry-After", "1")
+ w.WriteHeader(http.StatusTooManyRequests)
+ }))
+ defer server.Close()
+
+ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
+ defer cancel()
+ started := time.Now()
+ result := VerifyToken(ctx, server.Client(), testToken, TokenSpec{Name: "x", Request: Request{URL: server.URL}})
+ assert.Equal(t, finding.StatusVerifyError, result.Status)
+ assert.Equal(t, int64(1), calls.Load())
+ assert.Less(t, time.Since(started), 200*time.Millisecond)
+ })
+
+ t.Run("gate rejection prevents retry request", func(t *testing.T) {
+ var calls atomic.Int64
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ calls.Add(1)
+ w.Header().Set("Retry-After", "0")
+ w.WriteHeader(http.StatusTooManyRequests)
+ }))
+ defer server.Close()
+
+ gateCalls := 0
+ ctx := WithRetryGate(context.Background(), func() *finding.VerificationResult {
+ gateCalls++
+ return &finding.VerificationResult{Status: finding.StatusVerifyError, Message: "admission rejected"}
+ })
+ result := VerifyToken(ctx, server.Client(), testToken, TokenSpec{Name: "x", Request: Request{URL: server.URL}})
+ assert.Equal(t, finding.StatusVerifyError, result.Status)
+ assert.Equal(t, "admission rejected", result.Message)
+ assert.Equal(t, 1, gateCalls)
+ assert.Equal(t, int64(1), calls.Load())
+ })
+}
+
+func TestAddRetryJitter_IsBoundedAndNeverShortensRetryAfter(t *testing.T) {
+ base := time.Second
+ assert.Equal(t, base, addRetryJitter(base, max429RetryWait, 0))
+ assert.Equal(t, 1100*time.Millisecond, addRetryJitter(base, max429RetryWait, 1))
+ assert.Equal(t, max429RetryWait, addRetryJitter(1950*time.Millisecond, max429RetryWait, 1))
+ for range 100 {
+ unit := retryJitterUnit()
+ assert.GreaterOrEqual(t, unit, float64(0))
+ assert.LessOrEqual(t, unit, float64(1))
+ }
+}
+
+func TestVerifyToken_RetryAndDecodePanicAlwaysCloseResponseBodies(t *testing.T) {
+ t.Run("redirect drains and closes response", func(t *testing.T) {
+ var closed atomic.Bool
+ client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
+ return &http.Response{
+ StatusCode: http.StatusTemporaryRedirect,
+ Header: make(http.Header),
+ Body: &closeTrackingBody{Reader: strings.NewReader("redirect body"), closed: &closed},
+ }, nil
+ })}
+
+ result := VerifyToken(context.Background(), client, testToken, TokenSpec{
+ Name: "x",
+ Request: Request{URL: "https://provider.example.test/probe"},
+ })
+ assert.Equal(t, finding.StatusVerifyError, result.Status)
+ assert.Contains(t, result.Message, "redirect")
+ assert.True(t, closed.Load())
+ })
+
+ t.Run("retry closes the 429 response before the next request", func(t *testing.T) {
+ var (
+ calls atomic.Int64
+ firstClosed atomic.Bool
+ lastClosed atomic.Bool
+ )
+ client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
+ if calls.Add(1) == 1 {
+ return &http.Response{
+ StatusCode: http.StatusTooManyRequests,
+ Header: http.Header{"Retry-After": []string{"0"}},
+ Body: &closeTrackingBody{Reader: strings.NewReader("rate limited"), closed: &firstClosed},
+ }, nil
+ }
+ return &http.Response{
+ StatusCode: http.StatusOK,
+ Header: make(http.Header),
+ Body: &closeTrackingBody{Reader: strings.NewReader("ok"), closed: &lastClosed},
+ }, nil
+ })}
+
+ result := VerifyToken(context.Background(), client, testToken, TokenSpec{
+ Name: "x",
+ Request: Request{URL: "https://provider.example.test/probe"},
+ ActiveMessage: "active",
+ })
+ assert.Equal(t, finding.StatusVerifiedActive, result.Status)
+ assert.True(t, firstClosed.Load())
+ assert.True(t, lastClosed.Load())
+ })
+
+ t.Run("provider decode panic still closes final response", func(t *testing.T) {
+ var closed atomic.Bool
+ client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
+ return &http.Response{
+ StatusCode: http.StatusOK,
+ Header: make(http.Header),
+ Body: &closeTrackingBody{Reader: strings.NewReader(`{}`), closed: &closed},
+ }, nil
+ })}
+ assert.Panics(t, func() {
+ VerifyToken(context.Background(), client, testToken, TokenSpec{
+ Name: "x",
+ Request: Request{URL: "https://provider.example.test/probe"},
+ Decode: func(io.Reader) (map[string]string, string, error) {
+ panic("synthetic decoder panic")
+ },
+ })
+ })
+ assert.True(t, closed.Load())
+ })
+}
+
+func TestRetryAfterDelay_ParsesBoundedly(t *testing.T) {
+ now := time.Date(2026, 8, 11, 12, 0, 0, 0, time.UTC)
+ delay, ok := retryAfterDelay("1", now)
+ require.True(t, ok)
+ assert.Equal(t, time.Second, delay)
+
+ delay, ok = retryAfterDelay(now.Add(time.Second).Format(http.TimeFormat), now)
+ require.True(t, ok)
+ assert.Equal(t, time.Second, delay)
+
+ delay, ok = retryAfterDelay("18446744073709551615", now)
+ require.True(t, ok)
+ assert.Greater(t, delay, max429RetryWait)
+
+ _, ok = retryAfterDelay("invalid", now)
+ assert.False(t, ok)
+}
+
func TestRateLimited(t *testing.T) {
withHeader := RateLimited(context.Background(), "x", "12")
assert.Equal(t, finding.StatusVerifyError, withHeader.Status)
@@ -217,6 +661,26 @@ func TestRateLimited(t *testing.T) {
assert.NotContains(t, noHeader.Message, "Retry-After")
}
+func TestRateLimited_DoesNotReflectArbitraryHeaderText(t *testing.T) {
+ var logs bytes.Buffer
+ previous := slog.Default()
+ slog.SetDefault(slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug})))
+ t.Cleanup(func() { slog.SetDefault(previous) })
+
+ result := RateLimited(context.Background(), "x", testToken)
+ assert.Equal(t, finding.StatusVerifyError, result.Status)
+ assert.NotContains(t, result.Message, testToken)
+ assert.NotContains(t, logs.String(), testToken)
+ assert.NotContains(t, result.Message, "Retry-After")
+
+ httpDate := "Wed, 21 Oct 2015 07:28:00 GMT"
+ result = RateLimited(context.Background(), "x", httpDate)
+ assert.Contains(t, result.Message, "Retry-After: "+httpDate)
+
+ result = RateLimited(context.Background(), "x", " 00030 ")
+ assert.Contains(t, result.Message, "Retry-After: 30")
+}
+
func TestVerifyToken_Redirect_IsVerifyError(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Location", "https://example.com/login")
@@ -249,6 +713,48 @@ func TestVerifyToken_TransportError_RedactsSecretInURL(t *testing.T) {
assert.Contains(t, res.Message, "[REDACTED]")
}
+func TestVerifyToken_TransportError_AlwaysRedactsTokenArgument(t *testing.T) {
+ var logs bytes.Buffer
+ previous := slog.Default()
+ slog.SetDefault(slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug})))
+ t.Cleanup(func() { slog.SetDefault(previous) })
+
+ client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
+ return nil, fmt.Errorf("proxy echoed header credential %s", testToken)
+ })}
+
+ res := VerifyToken(context.Background(), client, testToken, TokenSpec{
+ Name: "x",
+ Request: Request{URL: "https://api.example.invalid"},
+ })
+ require.Equal(t, finding.StatusVerifyError, res.Status)
+ assert.NotContains(t, res.Message, testToken)
+ assert.Contains(t, res.Message, "[REDACTED]")
+ assert.NotContains(t, logs.String(), testToken)
+}
+
+func TestVerifyToken_TransportError_RedactsDerivedBasicAuthorization(t *testing.T) {
+ const user = "synthetic-user"
+ const password = "synthetic-password-1234"
+ encoded := base64.StdEncoding.EncodeToString([]byte(user + ":" + password))
+ client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
+ return nil, fmt.Errorf("proxy echoed Authorization: Basic %s", encoded)
+ })}
+
+ res := VerifyToken(context.Background(), client, password, TokenSpec{
+ Name: "x",
+ Request: Request{
+ URL: "https://api.example.invalid",
+ BasicAuthUser: user,
+ BasicAuthPass: password,
+ },
+ })
+ require.Equal(t, finding.StatusVerifyError, res.Status)
+ assert.NotContains(t, res.Message, encoded)
+ assert.NotContains(t, res.Message, password)
+ assert.Contains(t, res.Message, "[REDACTED]")
+}
+
func TestVerifyToken_BasicAuthAndHeaders(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
user, pass, ok := r.BasicAuth()
diff --git a/internal/verifier/internal/vtest/vtest.go b/internal/verifier/internal/vtest/vtest.go
index 4d7186a..0eb7c3b 100644
--- a/internal/verifier/internal/vtest/vtest.go
+++ b/internal/verifier/internal/vtest/vtest.go
@@ -14,9 +14,16 @@
package vtest
import (
+ "bytes"
"context"
+ "fmt"
+ "log/slog"
"net/http"
"net/http/httptest"
+ "net/url"
+ "strings"
+ "sync"
+ "sync/atomic"
"testing"
"github.com/stretchr/testify/assert"
@@ -44,6 +51,15 @@ type Case struct {
// plausibly formatted secret so the verifier reaches the HTTP call.
Raw detector.RawFinding
+ // RawForURL optionally builds the finding after the hermetic endpoint is
+ // known. It is used by credentials whose request destination is the finding
+ // itself, such as provider webhook URLs.
+ RawForURL func(string) detector.RawFinding
+
+ // AdditionalSensitive lists credential-bearing representations that the
+ // verifier derives outside Raw/RawV2 and must redact from results and logs.
+ AdditionalSensitive [][]byte
+
// MalformedStatus is the status the verifier returns for a 200 response
// whose body is not valid JSON. Defaults to StatusVerifyError when zero
// (the project standard). Set explicitly for verifiers that only inspect
@@ -74,10 +90,35 @@ func Run(t *testing.T, c Case) {
server.Close() // Force a connection-refused transport error.
v := c.New(url, client)
- result := v.Verify(context.Background(), c.Raw)
+ testCase := c.withURL(url)
+ result := v.Verify(context.Background(), testCase.Raw)
assert.Equal(t, finding.StatusVerifyError, result.Status,
"a transport error must be a verify error")
+ assertSecretAbsent(t, result, "", testCase)
+ })
+
+ t.Run(c.Name+"/transport_error_cannot_echo_secret", func(t *testing.T) {
+ var calls atomic.Int32
+ const endpoint = "https://api.example.invalid"
+ testCase := c.withURL(endpoint)
+ client := &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
+ calls.Add(1)
+ echoed := []string{request.URL.String(), string(testCase.Raw.Raw), string(testCase.Raw.RawV2)}
+ for _, values := range request.Header {
+ echoed = append(echoed, values...)
+ }
+ return nil, fmt.Errorf("synthetic transport failure echoed request %q", echoed)
+ })}
+ v := c.New(endpoint, client)
+ result, logs := verifyWithCapturedLogs(t, func() finding.VerificationResult {
+ return v.Verify(context.Background(), testCase.Raw)
+ })
+
+ require.Equal(t, int32(1), calls.Load(), "injected transport must be exercised exactly once")
+ assert.Equal(t, finding.StatusVerifyError, result.Status,
+ "a transport error must be a verify error")
+ assertSecretAbsent(t, result, logs, testCase)
})
t.Run(c.Name+"/cancelled_context_is_not_inactive", func(t *testing.T) {
@@ -90,13 +131,15 @@ func Run(t *testing.T, c Case) {
ctx, cancel := context.WithCancel(context.Background())
cancel() // Cancel immediately.
+ testCase := c.withURL(server.URL)
v := c.New(server.URL, server.Client())
- result := v.Verify(ctx, c.Raw)
+ result := v.Verify(ctx, testCase.Raw)
require.NotEqual(t, finding.StatusVerifiedInactive, result.Status,
"a cancelled context must NOT be reported as verified-inactive")
assert.Equal(t, finding.StatusVerifyError, result.Status,
"a cancelled context must be a verify error")
+ assertSecretAbsent(t, result, "", testCase)
})
if c.SkipMalformed {
@@ -104,7 +147,9 @@ func Run(t *testing.T, c Case) {
}
t.Run(c.Name+"/malformed_body_has_defined_status", func(t *testing.T) {
+ var calls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ calls.Add(1)
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte(`{not valid json`))
@@ -119,12 +164,65 @@ func Run(t *testing.T, c Case) {
want = finding.StatusVerifyError
}
+ testCase := c.withURL(server.URL)
v := c.New(server.URL, server.Client())
- result := v.Verify(context.Background(), c.Raw)
+ result := v.Verify(context.Background(), testCase.Raw)
+ require.Equal(t, int32(1), calls.Load(), "malformed-response transport must be exercised exactly once")
assert.Equal(t, want, result.Status,
"a 200 with a malformed body must have a defined status")
assert.NotEqual(t, finding.StatusVerifiedInactive, result.Status,
"a malformed 200 body must never be reported as verified-inactive")
+ assertSecretAbsent(t, result, "", testCase)
})
}
+
+func (c Case) withURL(endpoint string) Case {
+ if c.RawForURL != nil {
+ c.Raw = c.RawForURL(endpoint)
+ }
+ return c
+}
+
+type roundTripFunc func(*http.Request) (*http.Response, error)
+
+func (f roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) {
+ return f(request)
+}
+
+func assertSecretAbsent(t *testing.T, result finding.VerificationResult, logs string, c Case) {
+ t.Helper()
+ secrets := append([][]byte{c.Raw.Raw, c.Raw.RawV2}, c.AdditionalSensitive...)
+ for _, secret := range secrets {
+ if len(secret) < 8 {
+ continue
+ }
+ for _, form := range sensitiveForms(string(secret)) {
+ assert.NotContains(t, result.Message, form, "verification message leaked credential")
+ assert.NotContains(t, logs, form, "verification log leaked credential")
+ }
+ for key, value := range result.ExtraData {
+ for _, form := range sensitiveForms(string(secret)) {
+ assert.False(t, strings.Contains(value, form),
+ "verification ExtraData[%q] leaked credential", key)
+ }
+ }
+ }
+}
+
+func sensitiveForms(secret string) []string {
+ return []string{secret, url.PathEscape(secret), url.QueryEscape(secret)}
+}
+
+var slogCaptureMu sync.Mutex
+
+func verifyWithCapturedLogs(t *testing.T, verify func() finding.VerificationResult) (finding.VerificationResult, string) {
+ t.Helper()
+ slogCaptureMu.Lock()
+ defer slogCaptureMu.Unlock()
+ var logs bytes.Buffer
+ previous := slog.Default()
+ slog.SetDefault(slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug})))
+ defer slog.SetDefault(previous)
+ return verify(), logs.String()
+}
diff --git a/internal/verifier/launchdarkly/launchdarkly_shared_test.go b/internal/verifier/launchdarkly/launchdarkly_shared_test.go
new file mode 100644
index 0000000..bf3276a
--- /dev/null
+++ b/internal/verifier/launchdarkly/launchdarkly_shared_test.go
@@ -0,0 +1,16 @@
+package launchdarkly
+
+import (
+ "net/http"
+ "testing"
+
+ "github.com/HodeTech/leakwatch/internal/detector"
+ "github.com/HodeTech/leakwatch/internal/verifier"
+ "github.com/HodeTech/leakwatch/internal/verifier/internal/vtest"
+)
+
+func TestVerify_SharedSafetySuite(t *testing.T) {
+ vtest.Run(t, vtest.Case{Name: "launchdarkly", New: func(url string, client *http.Client) verifier.Verifier {
+ return &Verifier{apiURL: url, httpClient: client}
+ }, Raw: detector.RawFinding{DetectorID: detectorID, Raw: []byte("sdk-abcdef01-2345-6789-abcd-0123456789ab")}, SkipMalformed: true})
+}
diff --git a/internal/verifier/mailgun/mailgun_verifier.go b/internal/verifier/mailgun/mailgun_verifier.go
index ab0c4d7..f4e11be 100644
--- a/internal/verifier/mailgun/mailgun_verifier.go
+++ b/internal/verifier/mailgun/mailgun_verifier.go
@@ -57,17 +57,60 @@ func (v *Verifier) Type() string {
// first probe is never treated as "inactive", so it is returned as-is
// without a second call.
func (v *Verifier) Verify(ctx context.Context, raw detector.RawFinding) finding.VerificationResult {
+ return v.verify(ctx, raw, nil)
+}
+
+// VerificationRequestBudget allows one initial request and one bounded HTTP
+// 429 replay in each of Mailgun's two fixed regions.
+func (v *Verifier) VerificationRequestBudget() int { return 4 }
+
+// VerifyWithRequestGate admits every actual regional send through the engine's
+// global and per-provider limiters. HTTP 429 replays receive the same gate via
+// the shared httpx context.
+func (v *Verifier) VerifyWithRequestGate(
+ ctx context.Context,
+ raw detector.RawFinding,
+ gate verifier.RequestGate,
+) finding.VerificationResult {
+ return v.verify(ctx, raw, gate)
+}
+
+func (v *Verifier) verify(
+ ctx context.Context,
+ raw detector.RawFinding,
+ gate verifier.RequestGate,
+) finding.VerificationResult {
token := string(raw.Raw)
+ if token == "" {
+ return finding.VerificationResult{
+ Status: finding.StatusUnverified,
+ Message: "empty token",
+ }
+ }
usURL := httpx.BaseURL(v.apiURL, defaultAPIURL)
- result := verifyDomains(ctx, v.httpClient, token, usURL)
+ result := v.verifyRegion(ctx, token, usURL, gate)
if result.Status != finding.StatusVerifiedInactive {
return result
}
slog.DebugContext(ctx, "mailgun verifier: US host reports inactive, retrying against EU host")
euURL := httpx.BaseURL(v.euAPIURL, defaultEUAPIURL)
- return verifyDomains(ctx, v.httpClient, token, euURL)
+ return v.verifyRegion(ctx, token, euURL, gate)
+}
+
+func (v *Verifier) verifyRegion(
+ ctx context.Context,
+ token, apiURL string,
+ gate verifier.RequestGate,
+) finding.VerificationResult {
+ if gate != nil {
+ if rejection := gate(); rejection != nil {
+ return *rejection
+ }
+ ctx = httpx.WithRetryGate(ctx, httpx.RetryGate(gate))
+ }
+ return verifyDomains(ctx, v.httpClient, token, apiURL)
}
// verifyDomains performs the Mailgun GET /v3/domains check against apiURL.
diff --git a/internal/verifier/mailgun/mailgun_verifier_test.go b/internal/verifier/mailgun/mailgun_verifier_test.go
index 4bfe9ed..1168dd8 100644
--- a/internal/verifier/mailgun/mailgun_verifier_test.go
+++ b/internal/verifier/mailgun/mailgun_verifier_test.go
@@ -4,15 +4,44 @@ import (
"context"
"net/http"
"net/http/httptest"
+ "sync"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/HodeTech/leakwatch/internal/detector"
+ "github.com/HodeTech/leakwatch/internal/verifier"
"github.com/HodeTech/leakwatch/pkg/finding"
)
+type statusSequence struct {
+ mu sync.Mutex
+ statuses []int
+ calls int
+}
+
+func (s *statusSequence) handler(w http.ResponseWriter, _ *http.Request) {
+ s.mu.Lock()
+ index := s.calls
+ s.calls++
+ status := s.statuses[len(s.statuses)-1]
+ if index < len(s.statuses) {
+ status = s.statuses[index]
+ }
+ s.mu.Unlock()
+ if status == http.StatusTooManyRequests {
+ w.Header().Set("Retry-After", "0")
+ }
+ w.WriteHeader(status)
+}
+
+func (s *statusSequence) callCount() int {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ return s.calls
+}
+
func TestVerify_ValidKey_ReturnsActive(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, "/v3/domains", r.URL.Path)
@@ -154,3 +183,87 @@ func TestVerify_EmptyToken_ReturnsUnverified(t *testing.T) {
assert.Equal(t, finding.StatusUnverified, result.Status)
assert.Equal(t, "empty token", result.Message)
}
+
+func TestVerifyWithRequestGate_AdmitsEveryFallbackAnd429Send(t *testing.T) {
+ tests := []struct {
+ name string
+ usStatuses []int
+ euStatuses []int
+ wantCalls int
+ }{
+ {name: "US inactive then EU active", usStatuses: []int{http.StatusUnauthorized}, euStatuses: []int{http.StatusOK}, wantCalls: 2},
+ {name: "US 429 retry inactive then EU active", usStatuses: []int{http.StatusTooManyRequests, http.StatusUnauthorized}, euStatuses: []int{http.StatusOK}, wantCalls: 3},
+ {name: "US inactive then EU 429 retry active", usStatuses: []int{http.StatusUnauthorized}, euStatuses: []int{http.StatusTooManyRequests, http.StatusOK}, wantCalls: 3},
+ }
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ us := &statusSequence{statuses: tc.usStatuses}
+ eu := &statusSequence{statuses: tc.euStatuses}
+ usServer := httptest.NewServer(http.HandlerFunc(us.handler))
+ defer usServer.Close()
+ euServer := httptest.NewServer(http.HandlerFunc(eu.handler))
+ defer euServer.Close()
+
+ v := &Verifier{apiURL: usServer.URL, euAPIURL: euServer.URL, httpClient: usServer.Client()}
+ gateCalls := 0
+ result := v.VerifyWithRequestGate(context.Background(), detector.RawFinding{
+ DetectorID: detectorID,
+ Raw: []byte("key-synthetic-region-sequence-123456"),
+ Redacted: "key-****3456",
+ }, func() *finding.VerificationResult {
+ gateCalls++
+ return nil
+ })
+
+ assert.Equal(t, finding.StatusVerifiedActive, result.Status)
+ assert.Equal(t, tc.wantCalls, us.callCount()+eu.callCount())
+ assert.Equal(t, tc.wantCalls, gateCalls, "each actual HTTP send needs one admission")
+ })
+ }
+}
+
+func TestVerifyWithRequestGate_RejectionPreventsEURequest(t *testing.T) {
+ us := &statusSequence{statuses: []int{http.StatusUnauthorized}}
+ eu := &statusSequence{statuses: []int{http.StatusOK}}
+ usServer := httptest.NewServer(http.HandlerFunc(us.handler))
+ defer usServer.Close()
+ euServer := httptest.NewServer(http.HandlerFunc(eu.handler))
+ defer euServer.Close()
+
+ v := &Verifier{apiURL: usServer.URL, euAPIURL: euServer.URL, httpClient: usServer.Client()}
+ gateCalls := 0
+ result := v.VerifyWithRequestGate(context.Background(), detector.RawFinding{
+ DetectorID: detectorID,
+ Raw: []byte("key-synthetic-gate-reject-123456789"),
+ Redacted: "key-****6789",
+ }, func() *finding.VerificationResult {
+ gateCalls++
+ if gateCalls == 2 {
+ return &finding.VerificationResult{Status: finding.StatusVerifyError, Message: "admission rejected"}
+ }
+ return nil
+ })
+
+ assert.Equal(t, finding.StatusVerifyError, result.Status)
+ assert.Equal(t, "admission rejected", result.Message)
+ assert.Equal(t, 1, us.callCount())
+ assert.Zero(t, eu.callCount())
+}
+
+func TestVerificationRequestBudget_CoversTwoRegionsAndRetries(t *testing.T) {
+ v := &Verifier{}
+ assert.Equal(t, 4, v.VerificationRequestBudget())
+ var _ verifier.RequestGatedVerifier = v
+}
+
+func TestVerifyWithRequestGate_EmptyTokenConsumesNoAdmission(t *testing.T) {
+ v := &Verifier{}
+ gateCalls := 0
+ result := v.VerifyWithRequestGate(context.Background(), detector.RawFinding{}, func() *finding.VerificationResult {
+ gateCalls++
+ return nil
+ })
+
+ assert.Equal(t, finding.StatusUnverified, result.Status)
+ assert.Zero(t, gateCalls)
+}
diff --git a/internal/verifier/newrelic/newrelic_shared_test.go b/internal/verifier/newrelic/newrelic_shared_test.go
new file mode 100644
index 0000000..bba1fbf
--- /dev/null
+++ b/internal/verifier/newrelic/newrelic_shared_test.go
@@ -0,0 +1,24 @@
+package newrelic
+
+import (
+ "net/http"
+ "testing"
+
+ "github.com/HodeTech/leakwatch/internal/detector"
+ "github.com/HodeTech/leakwatch/internal/verifier"
+ "github.com/HodeTech/leakwatch/internal/verifier/internal/vtest"
+)
+
+func TestVerify_SharedSafetySuite(t *testing.T) {
+ vtest.Run(t, vtest.Case{
+ Name: "newrelic",
+ New: func(apiURL string, client *http.Client) verifier.Verifier {
+ return &Verifier{endpoints: []endpoint{{region: "test", url: apiURL}}, httpClient: client}
+ },
+ Raw: detector.RawFinding{
+ DetectorID: detectorID,
+ Raw: []byte(testToken),
+ Redacted: "NRAK-****2MNO",
+ },
+ })
+}
diff --git a/internal/verifier/newrelic/newrelic_verifier.go b/internal/verifier/newrelic/newrelic_verifier.go
index d130348..e3ab5f3 100644
--- a/internal/verifier/newrelic/newrelic_verifier.go
+++ b/internal/verifier/newrelic/newrelic_verifier.go
@@ -1,10 +1,16 @@
-// Package newrelic provides a verifier for New Relic API keys.
-// It uses the New Relic API GET /v2/users.json endpoint to check token validity.
+// Package newrelic provides a verifier for New Relic user API keys.
+// It uses a read-only NerdGraph requestContext query against New Relic's
+// officially documented US and EU endpoints.
package newrelic
import (
"context"
+ "encoding/json"
+ "fmt"
+ "io"
"net/http"
+ "strconv"
+ "strings"
"github.com/HodeTech/leakwatch/internal/detector"
"github.com/HodeTech/leakwatch/internal/verifier"
@@ -12,17 +18,30 @@ import (
"github.com/HodeTech/leakwatch/pkg/finding"
)
-const detectorID = "newrelic-api-key"
+const (
+ detectorID = "newrelic-api-key"
-// defaultAPIURL is the base URL for the New Relic API.
-const defaultAPIURL = "https://api.newrelic.com"
+ usNerdGraphURL = "https://api.newrelic.com/graphql"
+ euNerdGraphURL = "https://api.eu.newrelic.com/graphql"
-// Verifier checks whether a New Relic API key is active by calling the
-// New Relic API. It NEVER logs or persists raw token values.
+ requestContextQuery = `{"query":"query LeakwatchVerifyKey { requestContext { userId } }"}`
+)
+
+type endpoint struct {
+ region string
+ url string
+}
+
+var officialEndpoints = []endpoint{
+ {region: "US", url: usNerdGraphURL},
+ {region: "EU", url: euNerdGraphURL},
+}
+
+// Verifier checks whether a New Relic user API key is active. It NEVER logs or
+// persists raw token values. endpoints and httpClient are hermetic test seams;
+// production uses the fixed officialEndpoints list and hardened shared client.
type Verifier struct {
- // apiURL overrides the New Relic API base URL (for testing).
- apiURL string
- // httpClient overrides the default HTTP client (for testing).
+ endpoints []endpoint
httpClient *http.Client
}
@@ -35,20 +54,176 @@ func (v *Verifier) Type() string {
return detectorID
}
-// Verify checks if the detected New Relic API key is valid/active.
-// Raw contains the token value.
+// VerificationRequestBudget returns the maximum number of HTTP probes this
+// verifier can issue for one finding. The verification engine enforces this
+// bound while admitting each actual request at its send point.
+func (v *Verifier) VerificationRequestBudget() int {
+ return len(v.resolvedEndpoints())
+}
+
+// Verify checks the key against New Relic's fixed US/EU NerdGraph endpoints.
+// A 401 from one region is not revocation evidence because the key may belong
+// to the other region. The key is inactive only when every supported region
+// returns 401; any successful region wins, while permission/network/provider
+// errors keep the outcome inconclusive.
func (v *Verifier) Verify(ctx context.Context, raw detector.RawFinding) finding.VerificationResult {
+ return v.verify(ctx, raw, nil)
+}
+
+// VerifyWithRequestGate checks every actual regional request through the
+// engine-owned rate limiter immediately before sending it. The fallback region
+// consumes no token unless it is genuinely attempted.
+func (v *Verifier) VerifyWithRequestGate(
+ ctx context.Context,
+ raw detector.RawFinding,
+ gate verifier.RequestGate,
+) finding.VerificationResult {
+ return v.verify(ctx, raw, gate)
+}
+
+func (v *Verifier) verify(
+ ctx context.Context,
+ raw detector.RawFinding,
+ gate verifier.RequestGate,
+) finding.VerificationResult {
token := string(raw.Raw)
- apiURL := httpx.BaseURL(v.apiURL, defaultAPIURL)
+ if strings.TrimSpace(token) == "" {
+ return finding.VerificationResult{Status: finding.StatusUnverified, Message: "empty token"}
+ }
+
+ endpoints := v.resolvedEndpoints()
+ unauthorized := 0
+ hadInconclusive := false
+ for _, target := range endpoints {
+ if ctx.Err() != nil {
+ return finding.VerificationResult{
+ Status: finding.StatusVerifyError,
+ Message: "New Relic verification cancelled before all regions were checked",
+ }
+ }
+ if gate != nil {
+ if rejection := gate(); rejection != nil {
+ return *rejection
+ }
+ }
+ result := v.verifyEndpoint(ctx, token, target)
+ switch result.Status {
+ case finding.StatusVerifiedActive:
+ return result
+ case finding.StatusVerifiedInactive:
+ unauthorized++
+ default:
+ hadInconclusive = true
+ }
+ }
+
+ if unauthorized == len(endpoints) && !hadInconclusive {
+ return finding.VerificationResult{
+ Status: finding.StatusVerifiedInactive,
+ Message: "New Relic API key is invalid or revoked in all supported regions",
+ }
+ }
+ return finding.VerificationResult{
+ Status: finding.StatusVerifyError,
+ Message: "New Relic verification was inconclusive across supported regions",
+ }
+}
+
+func (v *Verifier) resolvedEndpoints() []endpoint {
+ if len(v.endpoints) > 0 {
+ return v.endpoints
+ }
+ return officialEndpoints
+}
+
+func (v *Verifier) verifyEndpoint(ctx context.Context, token string, target endpoint) finding.VerificationResult {
return httpx.VerifyToken(ctx, v.httpClient, token, httpx.TokenSpec{
- Name: "newrelic",
+ Name: "newrelic-" + strings.ToLower(target.region),
Request: httpx.Request{
- URL: apiURL + "/v2/users.json",
- Header: map[string]string{"Api-Key": token},
+ Method: http.MethodPost,
+ URL: target.url,
+ Body: []byte(requestContextQuery),
+ Header: map[string]string{
+ "Accept": "application/json",
+ "Api-Key": token,
+ "Content-Type": "application/json",
+ },
},
- InactiveStatuses: []int{http.StatusUnauthorized, http.StatusForbidden},
- ActiveMessage: "New Relic API key is active",
- InactiveMessage: "New Relic API key is invalid or revoked",
+ ActiveMessage: "New Relic API key is active (" + target.region + " region)",
+ InactiveMessage: "New Relic API key was rejected by the " + target.region + " region",
+ Decode: decodeRequestContext,
+ DecodeInactive: decodeInactiveResponse,
+ RequireCompleteBody: true,
+ RequireJSONContentType: true,
})
}
+
+func decodeInactiveResponse(body io.Reader) error {
+ var response struct {
+ Errors []struct {
+ Message string `json:"message"`
+ } `json:"errors"`
+ }
+ decoder := json.NewDecoder(body)
+ if err := decoder.Decode(&response); err != nil {
+ return err
+ }
+ if len(response.Errors) == 0 {
+ return fmt.Errorf("missing New Relic authentication error")
+ }
+ for _, item := range response.Errors {
+ if strings.ToLower(strings.TrimSpace(item.Message)) != "authentication required" {
+ return fmt.Errorf("unexpected New Relic authentication error")
+ }
+ }
+ if decoder.Decode(&struct{}{}) != io.EOF {
+ return fmt.Errorf("unexpected trailing New Relic response data")
+ }
+ return nil
+}
+
+func decodeRequestContext(body io.Reader) (map[string]string, string, error) {
+ var response struct {
+ Data *struct {
+ RequestContext *struct {
+ UserID json.RawMessage `json:"userId"`
+ } `json:"requestContext"`
+ } `json:"data"`
+ Errors []json.RawMessage `json:"errors"`
+ }
+ decoder := json.NewDecoder(body)
+ if err := decoder.Decode(&response); err != nil {
+ return nil, "", fmt.Errorf("invalid NerdGraph response: %w", err)
+ }
+ if len(response.Errors) > 0 {
+ return nil, "", fmt.Errorf("NerdGraph returned GraphQL errors")
+ }
+ if response.Data == nil || response.Data.RequestContext == nil ||
+ !isValidUserID(response.Data.RequestContext.UserID) {
+ return nil, "", fmt.Errorf("NerdGraph response did not identify the API key owner")
+ }
+
+ var trailing json.RawMessage
+ if err := decoder.Decode(&trailing); err != io.EOF {
+ if err == nil {
+ return nil, "", fmt.Errorf("invalid NerdGraph response: trailing JSON value")
+ }
+ return nil, "", fmt.Errorf("invalid NerdGraph response: %w", err)
+ }
+ return nil, "", nil
+}
+
+func isValidUserID(value json.RawMessage) bool {
+ var text string
+ if err := json.Unmarshal(value, &text); err == nil {
+ return strings.TrimSpace(text) != ""
+ }
+
+ var number json.Number
+ if err := json.Unmarshal(value, &number); err != nil {
+ return false
+ }
+ id, err := strconv.ParseInt(number.String(), 10, 64)
+ return err == nil && id > 0
+}
diff --git a/internal/verifier/newrelic/newrelic_verifier_test.go b/internal/verifier/newrelic/newrelic_verifier_test.go
index 24eec75..3492733 100644
--- a/internal/verifier/newrelic/newrelic_verifier_test.go
+++ b/internal/verifier/newrelic/newrelic_verifier_test.go
@@ -2,8 +2,11 @@ package newrelic
import (
"context"
+ "io"
"net/http"
"net/http/httptest"
+ "strings"
+ "sync/atomic"
"testing"
"github.com/stretchr/testify/assert"
@@ -13,122 +16,286 @@ import (
"github.com/HodeTech/leakwatch/pkg/finding"
)
-func TestVerify_ValidToken_ReturnsActive(t *testing.T) {
+const testToken = "NRAK-ABC123DEF456GHI789JKL012MNO"
+
+func rawFinding(token string) detector.RawFinding {
+ return detector.RawFinding{DetectorID: detectorID, Raw: []byte(token), Redacted: "NRAK-****2MNO"}
+}
+
+func TestVerify_NerdGraphRequestAndResponseContract(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- assert.Equal(t, "/v2/users.json", r.URL.Path)
- assert.NotEmpty(t, r.Header.Get("Api-Key"))
+ assert.Equal(t, "/graphql", r.URL.Path)
+ assert.Equal(t, http.MethodPost, r.Method)
+ assert.Equal(t, testToken, r.Header.Get("Api-Key"))
+ assert.Equal(t, "application/json", r.Header.Get("Accept"))
+ assert.Equal(t, "application/json", r.Header.Get("Content-Type"))
+ body, err := io.ReadAll(r.Body)
+ require.NoError(t, err)
+ assert.Equal(t, requestContextQuery, string(body))
w.Header().Set("Content-Type", "application/json")
- w.WriteHeader(http.StatusOK)
- _, _ = w.Write([]byte(`{"users":[{"id":123,"email":"user@example.com"}]}`))
+ _, _ = w.Write([]byte(`{"data":{"requestContext":{"userId":12345}}}`))
}))
defer server.Close()
v := &Verifier{
- apiURL: server.URL,
+ endpoints: []endpoint{{region: "US", url: server.URL + "/graphql"}},
httpClient: server.Client(),
}
+ result := v.Verify(context.Background(), rawFinding(testToken))
+
+ require.Equal(t, finding.StatusVerifiedActive, result.Status)
+ assert.Equal(t, "New Relic API key is active (US region)", result.Message)
+ assert.Empty(t, result.ExtraData, "identity/PII must not be retained")
+}
- raw := detector.RawFinding{
- DetectorID: detectorID,
- Raw: []byte("NRAK-ABC123DEF456GHI789JKL012MNO345PQ"),
- Redacted: "NRAK****45PQ",
+func TestVerify_RegionFallbackDecisionMatrix(t *testing.T) {
+ tests := []struct {
+ name string
+ statuses []int
+ bodies []string
+ want finding.VerificationStatus
+ wantRequests int32
+ }{
+ {
+ name: "US unauthorized then EU active",
+ statuses: []int{http.StatusUnauthorized, http.StatusOK},
+ bodies: []string{`{"errors":[{"message":"authentication required"}]}`, `{"data":{"requestContext":{"userId":"eu-user"}}}`},
+ want: finding.StatusVerifiedActive, wantRequests: 2,
+ },
+ {
+ name: "US active stops fallback",
+ statuses: []int{http.StatusOK, http.StatusInternalServerError},
+ bodies: []string{`{"data":{"requestContext":{"userId":1}}}`, `{}`},
+ want: finding.StatusVerifiedActive, wantRequests: 1,
+ },
+ {
+ name: "all regions unauthorized",
+ statuses: []int{http.StatusUnauthorized, http.StatusUnauthorized},
+ bodies: []string{`{"errors":[{"message":"authentication required"}]}`, `{"errors":[{"message":"authentication required"}]}`},
+ want: finding.StatusVerifiedInactive, wantRequests: 2,
+ },
+ {
+ name: "403 is never inactive",
+ statuses: []int{http.StatusForbidden, http.StatusUnauthorized},
+ bodies: []string{`{}`, `{"errors":[{"message":"authentication required"}]}`},
+ want: finding.StatusVerifyError, wantRequests: 2,
+ },
+ {
+ name: "provider failure plus 401 is inconclusive",
+ statuses: []int{http.StatusInternalServerError, http.StatusUnauthorized},
+ bodies: []string{`{}`, `{"errors":[{"message":"authentication required"}]}`},
+ want: finding.StatusVerifyError, wantRequests: 2,
+ },
+ {
+ name: "GraphQL error plus 401 is inconclusive",
+ statuses: []int{http.StatusOK, http.StatusUnauthorized},
+ bodies: []string{`{"errors":[{"message":"forbidden"}]}`, `{"errors":[{"message":"authentication required"}]}`},
+ want: finding.StatusVerifyError, wantRequests: 2,
+ },
}
- result := v.Verify(context.Background(), raw)
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ var requests atomic.Int32
+ servers := make([]*httptest.Server, len(tc.statuses))
+ endpoints := make([]endpoint, len(tc.statuses))
+ for i := range tc.statuses {
+ status, body := tc.statuses[i], tc.bodies[i]
+ servers[i] = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ requests.Add(1)
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(status)
+ _, _ = w.Write([]byte(body))
+ }))
+ defer servers[i].Close()
+ endpoints[i] = endpoint{region: []string{"US", "EU"}[i], url: servers[i].URL}
+ }
- require.Equal(t, finding.StatusVerifiedActive, result.Status)
- assert.Equal(t, "New Relic API key is active", result.Message)
+ v := &Verifier{endpoints: endpoints, httpClient: servers[0].Client()}
+ result := v.Verify(context.Background(), rawFinding(testToken))
+ assert.Equal(t, tc.want, result.Status)
+ assert.Equal(t, tc.wantRequests, requests.Load())
+ })
+ }
}
-func TestVerify_Unauthorized_ReturnsInactive(t *testing.T) {
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
- w.WriteHeader(http.StatusUnauthorized)
- _, _ = w.Write([]byte(`{"error":{"title":"Invalid API key"}}`))
- }))
- defer server.Close()
-
- v := &Verifier{
- apiURL: server.URL,
- httpClient: server.Client(),
+func TestVerify_InactiveResponseMustBeDefinitiveJSON(t *testing.T) {
+ tests := []struct{ contentType, body string }{
+ {contentType: "text/html", body: `proxy login`},
+ {contentType: "application/json", body: `{"errors":[{"message":"challenge"}]}`},
}
+ for _, tc := range tests {
+ servers := make([]*httptest.Server, 2)
+ endpoints := make([]endpoint, 2)
+ for i := range servers {
+ servers[i] = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", tc.contentType)
+ w.WriteHeader(http.StatusUnauthorized)
+ _, _ = w.Write([]byte(tc.body))
+ }))
+ endpoints[i] = endpoint{region: []string{"US", "EU"}[i], url: servers[i].URL}
+ }
+ result := (&Verifier{endpoints: endpoints, httpClient: servers[0].Client()}).Verify(
+ context.Background(), rawFinding(testToken),
+ )
+ for _, server := range servers {
+ server.Close()
+ }
+ assert.Equal(t, finding.StatusVerifyError, result.Status)
+ }
+}
- raw := detector.RawFinding{
- DetectorID: detectorID,
- Raw: []byte("NRAK-invalidkey123456789012345678901234"),
- Redacted: "NRAK****1234",
+func TestVerifyWithRequestGate_AdmitsOnlyActualRegionalRequests(t *testing.T) {
+ tests := []struct {
+ name string
+ statuses []int
+ bodies []string
+ wantGates int32
+ wantCalls int32
+ }{
+ {
+ name: "US active does not admit unused EU fallback",
+ statuses: []int{http.StatusOK, http.StatusInternalServerError},
+ bodies: []string{`{"data":{"requestContext":{"userId":1}}}`, `{}`},
+ wantGates: 1,
+ wantCalls: 1,
+ },
+ {
+ name: "US rejection admits EU immediately before fallback",
+ statuses: []int{http.StatusUnauthorized, http.StatusOK},
+ bodies: []string{`{"errors":[{"message":"authentication required"}]}`, `{"data":{"requestContext":{"userId":2}}}`},
+ wantGates: 2,
+ wantCalls: 2,
+ },
}
- result := v.Verify(context.Background(), raw)
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ var gates atomic.Int32
+ var calls atomic.Int32
+ servers := make([]*httptest.Server, len(tc.statuses))
+ endpoints := make([]endpoint, len(tc.statuses))
+ for i := range tc.statuses {
+ status, body := tc.statuses[i], tc.bodies[i]
+ servers[i] = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ calls.Add(1)
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(status)
+ _, _ = w.Write([]byte(body))
+ }))
+ defer servers[i].Close()
+ endpoints[i] = endpoint{region: []string{"US", "EU"}[i], url: servers[i].URL}
+ }
- assert.Equal(t, finding.StatusVerifiedInactive, result.Status)
- assert.Equal(t, "New Relic API key is invalid or revoked", result.Message)
+ v := &Verifier{endpoints: endpoints, httpClient: servers[0].Client()}
+ result := v.VerifyWithRequestGate(context.Background(), rawFinding(testToken), func() *finding.VerificationResult {
+ gates.Add(1)
+ return nil
+ })
+ assert.Equal(t, finding.StatusVerifiedActive, result.Status)
+ assert.Equal(t, tc.wantGates, gates.Load())
+ assert.Equal(t, tc.wantCalls, calls.Load())
+ })
+ }
}
-func TestVerify_Forbidden_ReturnsInactive(t *testing.T) {
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
- w.WriteHeader(http.StatusForbidden)
- _, _ = w.Write([]byte(`{"error":{"title":"Forbidden"}}`))
- }))
- defer server.Close()
-
+func TestVerifyWithRequestGate_RejectionPreventsFallbackRequest(t *testing.T) {
+ var requests atomic.Int32
+ servers := make([]*httptest.Server, 2)
+ for i, status := range []int{http.StatusUnauthorized, http.StatusOK} {
+ status := status
+ servers[i] = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ requests.Add(1)
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(status)
+ _, _ = w.Write([]byte(`{"data":{"requestContext":{"userId":2}}}`))
+ }))
+ defer servers[i].Close()
+ }
v := &Verifier{
- apiURL: server.URL,
- httpClient: server.Client(),
+ endpoints: []endpoint{{region: "US", url: servers[0].URL}, {region: "EU", url: servers[1].URL}},
+ httpClient: servers[0].Client(),
}
+ var gates atomic.Int32
+ result := v.VerifyWithRequestGate(context.Background(), rawFinding(testToken), func() *finding.VerificationResult {
+ if gates.Add(1) == 2 {
+ return &finding.VerificationResult{Status: finding.StatusVerifyError, Message: "rate limiter cancelled"}
+ }
+ return nil
+ })
+
+ assert.Equal(t, finding.StatusVerifyError, result.Status)
+ assert.Equal(t, int32(2), gates.Load())
+ assert.Equal(t, int32(1), requests.Load(), "rejected admission must happen before the EU request is sent")
+}
- raw := detector.RawFinding{
- DetectorID: detectorID,
- Raw: []byte("NRAK-forbiddenkey12345678901234567890123"),
- Redacted: "NRAK****0123",
+func TestVerify_InvalidSuccessBodiesAreInconclusive(t *testing.T) {
+ tests := []struct {
+ name string
+ contentType string
+ body string
+ }{
+ {name: "HTML", contentType: "text/html", body: `login`},
+ {name: "JSON with HTML media type", contentType: "text/html", body: `{"data":{"requestContext":{"userId":1}}}`},
+ {name: "malformed", contentType: "application/json", body: `{not-json`},
+ {name: "missing data", contentType: "application/json", body: `{}`},
+ {name: "null data", contentType: "application/json", body: `{"data":null}`},
+ {name: "null user", contentType: "application/json", body: `{"data":{"requestContext":{"userId":null}}}`},
+ {name: "empty user", contentType: "application/json", body: `{"data":{"requestContext":{"userId":""}}}`},
+ {name: "object user", contentType: "application/json", body: `{"data":{"requestContext":{"userId":{}}}}`},
+ {name: "boolean user", contentType: "application/json", body: `{"data":{"requestContext":{"userId":true}}}`},
+ {name: "zero user", contentType: "application/json", body: `{"data":{"requestContext":{"userId":0}}}`},
+ {name: "fractional user", contentType: "application/json", body: `{"data":{"requestContext":{"userId":1.5}}}`},
+ {name: "trailing JSON", contentType: "application/json", body: `{"data":{"requestContext":{"userId":1}}} {}`},
+ {name: "oversized", contentType: "application/json", body: `{"data":{"requestContext":{"userId":1}}}` + strings.Repeat(" ", (1<<20)+1)},
}
- result := v.Verify(context.Background(), raw)
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ if tc.contentType != "" {
+ w.Header().Set("Content-Type", tc.contentType)
+ }
+ _, _ = w.Write([]byte(tc.body))
+ }))
+ defer server.Close()
- assert.Equal(t, finding.StatusVerifiedInactive, result.Status)
- assert.Equal(t, "New Relic API key is invalid or revoked", result.Message)
+ v := &Verifier{endpoints: []endpoint{{region: "US", url: server.URL}}, httpClient: server.Client()}
+ result := v.Verify(context.Background(), rawFinding(testToken))
+ assert.Equal(t, finding.StatusVerifyError, result.Status)
+ assert.NotEqual(t, finding.StatusVerifiedActive, result.Status)
+ assert.NotEqual(t, finding.StatusVerifiedInactive, result.Status)
+ })
+ }
}
-func TestVerify_ServerError_ReturnsError(t *testing.T) {
+func TestVerify_EmptyOrCancelledIsSafe(t *testing.T) {
+ var requests atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
- w.WriteHeader(http.StatusInternalServerError)
- _, _ = w.Write([]byte(`{"error":{"title":"Internal server error"}}`))
+ requests.Add(1)
+ w.WriteHeader(http.StatusUnauthorized)
}))
defer server.Close()
+ v := &Verifier{endpoints: []endpoint{{region: "US", url: server.URL}}, httpClient: server.Client()}
- v := &Verifier{
- apiURL: server.URL,
- httpClient: server.Client(),
- }
-
- raw := detector.RawFinding{
- DetectorID: detectorID,
- Raw: []byte("NRAK-somekey12345678901234567890123456789"),
- Redacted: "NRAK****6789",
+ for _, token := range []string{"", " \t\r\n"} {
+ result := v.Verify(context.Background(), rawFinding(token))
+ assert.Equal(t, finding.StatusUnverified, result.Status)
}
- result := v.Verify(context.Background(), raw)
-
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+ result := v.Verify(ctx, rawFinding(testToken))
assert.Equal(t, finding.StatusVerifyError, result.Status)
- assert.Contains(t, result.Message, "500")
+ assert.Zero(t, requests.Load())
}
-func TestVerify_Type_ReturnsCorrectID(t *testing.T) {
- v := &Verifier{}
- assert.Equal(t, "newrelic-api-key", v.Type())
+func TestVerificationRequestBudget(t *testing.T) {
+ assert.Equal(t, len(officialEndpoints), (&Verifier{}).VerificationRequestBudget())
+ assert.Equal(t, 1, (&Verifier{endpoints: []endpoint{{region: "test", url: "https://example.test"}}}).VerificationRequestBudget())
}
-func TestVerify_EmptyToken_ReturnsUnverified(t *testing.T) {
- v := &Verifier{}
-
- raw := detector.RawFinding{
- DetectorID: detectorID,
- Raw: []byte(""),
- Redacted: "",
- }
-
- result := v.Verify(context.Background(), raw)
-
- assert.Equal(t, finding.StatusUnverified, result.Status)
- assert.Equal(t, "empty token", result.Message)
+func TestVerify_Type_ReturnsCorrectID(t *testing.T) {
+ assert.Equal(t, detectorID, (&Verifier{}).Type())
}
diff --git a/internal/verifier/notion/notion_shared_test.go b/internal/verifier/notion/notion_shared_test.go
new file mode 100644
index 0000000..7333c5b
--- /dev/null
+++ b/internal/verifier/notion/notion_shared_test.go
@@ -0,0 +1,16 @@
+package notion
+
+import (
+ "net/http"
+ "testing"
+
+ "github.com/HodeTech/leakwatch/internal/detector"
+ "github.com/HodeTech/leakwatch/internal/verifier"
+ "github.com/HodeTech/leakwatch/internal/verifier/internal/vtest"
+)
+
+func TestVerify_SharedSafetySuite(t *testing.T) {
+ vtest.Run(t, vtest.Case{Name: "notion", New: func(url string, client *http.Client) verifier.Verifier {
+ return &Verifier{apiURL: url, httpClient: client}
+ }, Raw: detector.RawFinding{DetectorID: detectorID, Raw: []byte("ntn_SyntheticNotionCredentialFixture123456789")}})
+}
diff --git a/internal/verifier/okta/okta_shared_test.go b/internal/verifier/okta/okta_shared_test.go
new file mode 100644
index 0000000..337c170
--- /dev/null
+++ b/internal/verifier/okta/okta_shared_test.go
@@ -0,0 +1,16 @@
+package okta
+
+import (
+ "net/http"
+ "testing"
+
+ "github.com/HodeTech/leakwatch/internal/detector"
+ "github.com/HodeTech/leakwatch/internal/verifier"
+ "github.com/HodeTech/leakwatch/internal/verifier/internal/vtest"
+)
+
+func TestVerify_SharedSafetySuite(t *testing.T) {
+ vtest.Run(t, vtest.Case{Name: "okta", New: func(url string, client *http.Client) verifier.Verifier {
+ return &Verifier{apiURL: url, httpClient: client}
+ }, Raw: detector.RawFinding{DetectorID: detectorID, Raw: []byte("00SyntheticOktaCredentialFixture1234567890")}})
+}
diff --git a/internal/verifier/okta/okta_verifier_test.go b/internal/verifier/okta/okta_verifier_test.go
index 97734b7..b9a72da 100644
--- a/internal/verifier/okta/okta_verifier_test.go
+++ b/internal/verifier/okta/okta_verifier_test.go
@@ -2,17 +2,45 @@ package okta
import (
"context"
+ "io"
"net/http"
"net/http/httptest"
+ "strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/HodeTech/leakwatch/internal/detector"
+ oktadetector "github.com/HodeTech/leakwatch/internal/detector/okta"
+ "github.com/HodeTech/leakwatch/internal/detector/testutil"
"github.com/HodeTech/leakwatch/pkg/finding"
)
+func TestVerify_RealDetectorOutput_UsesOktaDomain(t *testing.T) {
+ fixture := testutil.RegisteredDetectorFixtures()[detectorID]
+ findings := testutil.ScanViaMatcher(&oktadetector.Detector{}, fixture.Input)
+ require.Len(t, findings, 1)
+ wantDomain := findings[0].ExtraData["domain"]
+ require.NotEmpty(t, wantDomain)
+
+ var gotURL string
+ client := &http.Client{Transport: oktaRoundTrip(func(r *http.Request) (*http.Response, error) {
+ gotURL = r.URL.String()
+ return &http.Response{
+ StatusCode: http.StatusOK, Header: http.Header{"Content-Type": []string{"application/json"}},
+ Body: io.NopCloser(strings.NewReader(`{"profile":{"login":"fixture@example.com"}}`)), Request: r,
+ }, nil
+ })}
+ result := (&Verifier{httpClient: client}).Verify(t.Context(), findings[0])
+ require.Equal(t, finding.StatusVerifiedActive, result.Status)
+ assert.Equal(t, "https://"+wantDomain+"/api/v1/users/me", gotURL)
+}
+
+type oktaRoundTrip func(*http.Request) (*http.Response, error)
+
+func (f oktaRoundTrip) RoundTrip(request *http.Request) (*http.Response, error) { return f(request) }
+
func TestVerify_ValidToken_ReturnsActive(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, "/api/v1/users/me", r.URL.Path)
diff --git a/internal/verifier/postmark/postmark_shared_test.go b/internal/verifier/postmark/postmark_shared_test.go
new file mode 100644
index 0000000..a5a8e60
--- /dev/null
+++ b/internal/verifier/postmark/postmark_shared_test.go
@@ -0,0 +1,16 @@
+package postmark
+
+import (
+ "net/http"
+ "testing"
+
+ "github.com/HodeTech/leakwatch/internal/detector"
+ "github.com/HodeTech/leakwatch/internal/verifier"
+ "github.com/HodeTech/leakwatch/internal/verifier/internal/vtest"
+)
+
+func TestVerify_SharedSafetySuite(t *testing.T) {
+ vtest.Run(t, vtest.Case{Name: "postmark", New: func(url string, client *http.Client) verifier.Verifier {
+ return &Verifier{apiURL: url, httpClient: client}
+ }, Raw: detector.RawFinding{DetectorID: detectorID, Raw: []byte("abcdef01-2345-6789-abcd-0123456789ab")}})
+}
diff --git a/internal/verifier/pypi/pypi_shared_test.go b/internal/verifier/pypi/pypi_shared_test.go
new file mode 100644
index 0000000..9c018a4
--- /dev/null
+++ b/internal/verifier/pypi/pypi_shared_test.go
@@ -0,0 +1,16 @@
+package pypi
+
+import (
+ "net/http"
+ "testing"
+
+ "github.com/HodeTech/leakwatch/internal/detector"
+ "github.com/HodeTech/leakwatch/internal/verifier"
+ "github.com/HodeTech/leakwatch/internal/verifier/internal/vtest"
+)
+
+func TestVerify_SharedSafetySuite(t *testing.T) {
+ vtest.Run(t, vtest.Case{Name: "pypi", New: func(url string, client *http.Client) verifier.Verifier {
+ return &Verifier{apiURL: url, httpClient: client}
+ }, Raw: detector.RawFinding{DetectorID: detectorID, Raw: []byte("pypi-SyntheticCredentialFixture1234567890")}, SkipMalformed: true})
+}
diff --git a/internal/verifier/registry.go b/internal/verifier/registry.go
index dfcc876..6997ce8 100644
--- a/internal/verifier/registry.go
+++ b/internal/verifier/registry.go
@@ -14,12 +14,19 @@ var (
// Each verifier package calls this function in its init() function.
// Panics if a verifier with the same Type() is already registered.
func Register(v Verifier) {
+ if v == nil || isNilVerifier(v) {
+ panic("cannot register a nil verifier")
+ }
+ typeID := v.Type()
+ if typeID == "" {
+ panic("cannot register a verifier with an empty type")
+ }
mu.Lock()
defer mu.Unlock()
- if _, exists := verifiers[v.Type()]; exists {
- panic("duplicate verifier type: " + v.Type())
+ if _, exists := verifiers[typeID]; exists {
+ panic("duplicate verifier type: " + typeID)
}
- verifiers[v.Type()] = v
+ verifiers[typeID] = v
}
// Get returns the verifier registered for the given detector ID.
diff --git a/internal/verifier/registry_test.go b/internal/verifier/registry_test.go
index e80ea2e..6037f70 100644
--- a/internal/verifier/registry_test.go
+++ b/internal/verifier/registry_test.go
@@ -51,6 +51,15 @@ func TestRegister_DuplicateType_Panics(t *testing.T) {
})
}
+func TestRegister_NilOrEmptyType_PanicsWithoutMutatingRegistry(t *testing.T) {
+ Reset()
+ var typedNil *mockVerifier
+ assert.Panics(t, func() { Register(nil) })
+ assert.Panics(t, func() { Register(typedNil) })
+ assert.Panics(t, func() { Register(newMock("")) })
+ assert.Empty(t, All())
+}
+
func TestGet_UnregisteredType_ReturnsFalse(t *testing.T) {
t.Cleanup(func() { Reset() })
diff --git a/internal/verifier/rubygems/rubygems_shared_test.go b/internal/verifier/rubygems/rubygems_shared_test.go
new file mode 100644
index 0000000..7fd5bff
--- /dev/null
+++ b/internal/verifier/rubygems/rubygems_shared_test.go
@@ -0,0 +1,16 @@
+package rubygems
+
+import (
+ "net/http"
+ "testing"
+
+ "github.com/HodeTech/leakwatch/internal/detector"
+ "github.com/HodeTech/leakwatch/internal/verifier"
+ "github.com/HodeTech/leakwatch/internal/verifier/internal/vtest"
+)
+
+func TestVerify_SharedSafetySuite(t *testing.T) {
+ vtest.Run(t, vtest.Case{Name: "rubygems", New: func(url string, client *http.Client) verifier.Verifier {
+ return &Verifier{apiURL: url, httpClient: client}
+ }, Raw: detector.RawFinding{DetectorID: detectorID, Raw: []byte("rubygems_abcdef0123456789abcdef0123456789")}, SkipMalformed: true})
+}
diff --git a/internal/verifier/shopify/shopify_shared_test.go b/internal/verifier/shopify/shopify_shared_test.go
new file mode 100644
index 0000000..96af353
--- /dev/null
+++ b/internal/verifier/shopify/shopify_shared_test.go
@@ -0,0 +1,16 @@
+package shopify
+
+import (
+ "net/http"
+ "testing"
+
+ "github.com/HodeTech/leakwatch/internal/detector"
+ "github.com/HodeTech/leakwatch/internal/verifier"
+ "github.com/HodeTech/leakwatch/internal/verifier/internal/vtest"
+)
+
+func TestVerify_SharedSafetySuite(t *testing.T) {
+ vtest.Run(t, vtest.Case{Name: "shopify", New: func(url string, client *http.Client) verifier.Verifier {
+ return &Verifier{apiURL: url, httpClient: client}
+ }, Raw: detector.RawFinding{DetectorID: detectorID, Raw: []byte("shpat_abcdef0123456789abcdef0123456789")}})
+}
diff --git a/internal/verifier/shopify/shopify_verifier.go b/internal/verifier/shopify/shopify_verifier.go
index c368175..bc8e069 100644
--- a/internal/verifier/shopify/shopify_verifier.go
+++ b/internal/verifier/shopify/shopify_verifier.go
@@ -1,13 +1,15 @@
-// Package shopify provides a verifier for Shopify access tokens.
-// It uses the Shopify Admin API GET /admin/api/2024-01/shop.json endpoint
-// to check token validity. A store domain is required via ExtraData.
+// Package shopify provides fail-closed verification for Shopify Admin API
+// access tokens against an operator-trusted store origin.
package shopify
import (
"context"
"encoding/json"
+ "errors"
"io"
"net/http"
+ "net/url"
+ "strings"
"github.com/HodeTech/leakwatch/internal/detector"
"github.com/HodeTech/leakwatch/internal/verifier"
@@ -15,79 +17,105 @@ import (
"github.com/HodeTech/leakwatch/pkg/finding"
)
-const detectorID = "shopify-access-token"
-
-// shopAPIPath is the Shopify Admin API endpoint for shop details.
-const shopAPIPath = "/admin/api/2024-01/shop.json"
+const (
+ detectorID = "shopify-access-token"
+ // Shopify versions are supported for at least 12 months. This pin is
+ // reviewed alongside LastContractReviewedAt in the capability manifest.
+ shopifyAPIVersion = "2026-07"
+ shopAPIPath = "/admin/api/" + shopifyAPIVersion + "/graphql.json"
+ shopIdentityQuery = `{"query":"query LeakwatchVerify { shop { name } }"}`
+)
-// Verifier checks whether a Shopify access token is active by calling the
-// Shopify Admin API. It NEVER logs or persists raw token values.
+// Verifier checks a token only when apiURL has been set from trusted operator
+// configuration. Finding-controlled store_domain metadata is never used for
+// routing, preventing wrong-issuer false-inactive results and credential
+// forwarding to an untrusted host.
type Verifier struct {
- // apiURL overrides the full base URL (for testing). When set, store_domain
- // from ExtraData is ignored and apiURL is used directly.
- apiURL string
- // httpClient overrides the default HTTP client (for testing).
+ apiURL string
httpClient *http.Client
}
+// NewForTrustedInstance accepts only an operator-selected canonical
+// myshopify.com store origin. Custom domains and repository metadata are not
+// routing authority for Admin API credentials.
+func NewForTrustedInstance(instanceURL string) (*Verifier, error) {
+ normalized, err := verifier.NormalizeTrustedHTTPSOrigin(instanceURL)
+ if err != nil {
+ return nil, err
+ }
+ u, err := url.Parse(normalized)
+ if err != nil || u.Port() != "" || !strings.HasSuffix(u.Hostname(), ".myshopify.com") ||
+ strings.TrimSuffix(u.Hostname(), ".myshopify.com") == "" {
+ return nil, errors.New("invalid Shopify store origin: a concrete myshopify.com store is required")
+ }
+ return &Verifier{apiURL: normalized}, nil
+}
+
+// WithTrustedInstance implements verifier.TrustedInstanceConfigurer.
+func (*Verifier) WithTrustedInstance(instanceURL string) (verifier.Verifier, error) {
+ return NewForTrustedInstance(instanceURL)
+}
+
func init() {
verifier.Register(&Verifier{})
}
// Type returns the detector ID this verifier handles.
-func (v *Verifier) Type() string {
- return detectorID
-}
+func (v *Verifier) Type() string { return detectorID }
-// Verify checks if the detected Shopify access token is valid/active.
-// Raw contains the token value. ExtraData["store_domain"] must contain the
-// Shopify store domain (e.g., "mystore.myshopify.com").
+// Verify performs a read-only GraphQL shop identity query. Only HTTP 401 on
+// the trusted store is definitive inactivity; authorization and GraphQL errors
+// remain verify_error.
func (v *Verifier) Verify(ctx context.Context, raw detector.RawFinding) finding.VerificationResult {
token := string(raw.Raw)
if token == "" {
+ return finding.VerificationResult{Status: finding.StatusUnverified, Message: "empty token"}
+ }
+ if v.apiURL == "" {
return finding.VerificationResult{
Status: finding.StatusUnverified,
- Message: "empty token",
+ Message: "trusted Shopify store origin is not configured",
}
}
- baseURL := v.apiURL
- if baseURL == "" {
- domain := raw.ExtraData["store_domain"]
- if domain == "" {
- return finding.VerificationResult{
- Status: finding.StatusUnverified,
- Message: "store domain required for verification",
- }
- }
- baseURL = "https://" + domain
- }
-
return httpx.VerifyToken(ctx, v.httpClient, token, httpx.TokenSpec{
Name: "shopify",
Request: httpx.Request{
- URL: baseURL + shopAPIPath,
- Header: map[string]string{"X-Shopify-Access-Token": token},
+ Method: http.MethodPost,
+ URL: v.apiURL + shopAPIPath,
+ Body: []byte(shopIdentityQuery),
+ Header: map[string]string{
+ "Accept": "application/json",
+ "Content-Type": "application/json",
+ "X-Shopify-Access-Token": token,
+ },
},
- ActiveMessage: "Shopify access token is active",
- InactiveMessage: "Shopify access token is invalid or revoked",
- Decode: decodeShop,
+ InactiveStatuses: []int{http.StatusUnauthorized},
+ ActiveMessage: "Shopify access token is active on the trusted store",
+ InactiveMessage: "Shopify access token is invalid or revoked on the trusted store",
+ Decode: decodeShopIdentity,
+ RequireCompleteBody: true,
+ RequireJSONContentType: true,
})
}
-// decodeShop reports the shop name as shop_name when present.
-func decodeShop(body io.Reader) (map[string]string, string, error) {
- var shopResp struct {
- Shop struct {
- Name string `json:"name"`
- } `json:"shop"`
+func decodeShopIdentity(body io.Reader) (map[string]string, string, error) {
+ var response struct {
+ Data *struct {
+ Shop *struct {
+ Name string `json:"name"`
+ } `json:"shop"`
+ } `json:"data"`
+ Errors []json.RawMessage `json:"errors"`
}
- if err := json.NewDecoder(body).Decode(&shopResp); err != nil {
+ if err := json.NewDecoder(body).Decode(&response); err != nil {
return nil, "", err
}
- extra := map[string]string{}
- if shopResp.Shop.Name != "" {
- extra["shop_name"] = shopResp.Shop.Name
+ if len(response.Errors) > 0 {
+ return nil, "", errors.New("shopify GraphQL response contains errors")
+ }
+ if response.Data == nil || response.Data.Shop == nil || response.Data.Shop.Name == "" {
+ return nil, "", errors.New("shopify GraphQL response is missing shop identity")
}
- return extra, "", nil
+ return map[string]string{"shop_name": response.Data.Shop.Name}, "", nil
}
diff --git a/internal/verifier/shopify/shopify_verifier_test.go b/internal/verifier/shopify/shopify_verifier_test.go
index cf6ccfd..5ffb1f8 100644
--- a/internal/verifier/shopify/shopify_verifier_test.go
+++ b/internal/verifier/shopify/shopify_verifier_test.go
@@ -2,159 +2,157 @@ package shopify
import (
"context"
+ "io"
"net/http"
"net/http/httptest"
+ "strings"
"testing"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-
"github.com/HodeTech/leakwatch/internal/detector"
+ shopifydetector "github.com/HodeTech/leakwatch/internal/detector/shopify"
"github.com/HodeTech/leakwatch/pkg/finding"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
)
-func TestVerify_ValidToken_ReturnsActive(t *testing.T) {
+func shopifyFinding() detector.RawFinding {
+ return detector.RawFinding{
+ DetectorID: detectorID,
+ Raw: []byte("shpat_" + strings.Repeat("ab12cd34", 4)),
+ Redacted: "shpat_****cd34",
+ }
+}
+
+func TestVerify_TrustedStoreUsesCurrentGraphQLContract(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- assert.Equal(t, shopAPIPath, r.URL.Path)
- assert.NotEmpty(t, r.Header.Get("X-Shopify-Access-Token"))
+ assert.Equal(t, http.MethodPost, r.Method)
+ assert.Equal(t, "/admin/api/2026-07/graphql.json", r.URL.Path)
+ assert.Equal(t, "application/json", r.Header.Get("Content-Type"))
+ assert.Equal(t, "application/json", r.Header.Get("Accept"))
+ assert.Equal(t, string(shopifyFinding().Raw), r.Header.Get("X-Shopify-Access-Token"))
+ body, err := io.ReadAll(r.Body)
+ require.NoError(t, err)
+ assert.JSONEq(t, shopIdentityQuery, string(body))
w.Header().Set("Content-Type", "application/json")
- w.WriteHeader(http.StatusOK)
- _, _ = w.Write([]byte(`{"shop":{"name":"My Test Store"}}`))
+ _, _ = w.Write([]byte(`{"data":{"shop":{"name":"My Test Store"}}}`))
}))
defer server.Close()
- v := &Verifier{
- apiURL: server.URL,
- httpClient: server.Client(),
- }
-
- raw := detector.RawFinding{
- DetectorID: detectorID,
- Raw: []byte("shpat_abcdef1234567890abcdef1234567890"),
- Redacted: "shpat_****7890",
- }
-
- result := v.Verify(context.Background(), raw)
+ result := (&Verifier{apiURL: server.URL, httpClient: server.Client()}).Verify(
+ context.Background(), shopifyFinding(),
+ )
require.Equal(t, finding.StatusVerifiedActive, result.Status)
- assert.Equal(t, "Shopify access token is active", result.Message)
+ assert.Equal(t, "Shopify access token is active on the trusted store", result.Message)
assert.Equal(t, "My Test Store", result.ExtraData["shop_name"])
}
-func TestVerify_InvalidToken_ReturnsInactive(t *testing.T) {
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
- w.WriteHeader(http.StatusUnauthorized)
- _, _ = w.Write([]byte(`{"errors":"[API] Invalid API key or access token"}`))
+func TestVerify_WithoutTrustedStoreMakesNoRequest(t *testing.T) {
+ requests := 0
+ server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
+ requests++
}))
defer server.Close()
- v := &Verifier{
- apiURL: server.URL,
- httpClient: server.Client(),
- }
+ raw := shopifyFinding()
+ raw.ExtraData = map[string]string{"store_domain": strings.TrimPrefix(server.URL, "http://")}
+ result := (&Verifier{httpClient: server.Client()}).Verify(context.Background(), raw)
- raw := detector.RawFinding{
- DetectorID: detectorID,
- Raw: []byte("shpat_invalidtoken1234567890abcdef1234"),
- Redacted: "shpat_****1234",
- }
-
- result := v.Verify(context.Background(), raw)
-
- assert.Equal(t, finding.StatusVerifiedInactive, result.Status)
- assert.Equal(t, "Shopify access token is invalid or revoked", result.Message)
+ assert.Equal(t, finding.StatusUnverified, result.Status)
+ assert.Equal(t, "trusted Shopify store origin is not configured", result.Message)
+ assert.Zero(t, requests, "finding-controlled store_domain must never select a request target")
}
-func TestVerify_ServerError_ReturnsError(t *testing.T) {
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
- w.WriteHeader(http.StatusInternalServerError)
- _, _ = w.Write([]byte(`{"errors":"Internal server error"}`))
- }))
- defer server.Close()
-
- v := &Verifier{
- apiURL: server.URL,
- httpClient: server.Client(),
+func TestNewForTrustedInstance_AcceptsOnlyCanonicalShopifyStore(t *testing.T) {
+ configured, err := NewForTrustedInstance("https://fixture-store.myshopify.com/")
+ require.NoError(t, err)
+ assert.Equal(t, "https://fixture-store.myshopify.com", configured.apiURL)
+ for _, origin := range []string{
+ "https://myshopify.com", "https://fixture-store.myshopify.com.attacker.example",
+ "https://fixture-store.myshopify.com:443", "https://custom-store.example",
+ } {
+ _, err := NewForTrustedInstance(origin)
+ assert.Error(t, err, origin)
}
-
- raw := detector.RawFinding{
- DetectorID: detectorID,
- Raw: []byte("shpat_sometoken12345678901234567890abcd"),
- Redacted: "shpat_****abcd",
- }
-
- result := v.Verify(context.Background(), raw)
-
- assert.Equal(t, finding.StatusVerifyError, result.Status)
- assert.Contains(t, result.Message, "500")
-}
-
-func TestVerify_Type_ReturnsCorrectID(t *testing.T) {
- v := &Verifier{}
- assert.Equal(t, "shopify-access-token", v.Type())
}
-func TestVerify_EmptyToken_ReturnsUnverified(t *testing.T) {
- v := &Verifier{}
-
- raw := detector.RawFinding{
- DetectorID: detectorID,
- Raw: []byte(""),
- Redacted: "",
- }
-
- result := v.Verify(context.Background(), raw)
+func TestVerify_RealDetectorFindingMatchesRequiresContextCapability(t *testing.T) {
+ findings := (&shopifydetector.Detector{}).Scan(context.Background(), shopifyFinding().Raw)
+ require.Len(t, findings, 1)
+ result := (&Verifier{}).Verify(context.Background(), findings[0])
assert.Equal(t, finding.StatusUnverified, result.Status)
- assert.Equal(t, "empty token", result.Message)
+ assert.Equal(t, "trusted Shopify store origin is not configured", result.Message)
}
-func TestVerify_NoDomain_ReturnsUnverified(t *testing.T) {
- v := &Verifier{}
-
- raw := detector.RawFinding{
- DetectorID: detectorID,
- Raw: []byte("shpat_abcdef1234567890abcdef1234567890"),
- Redacted: "shpat_****7890",
- ExtraData: map[string]string{},
+func TestVerify_Only401IsInactive(t *testing.T) {
+ tests := []struct {
+ name string
+ status int
+ want finding.VerificationStatus
+ }{
+ {name: "401 authentication rejection", status: http.StatusUnauthorized, want: finding.StatusVerifiedInactive},
+ {name: "403 authorization rejection", status: http.StatusForbidden, want: finding.StatusVerifyError},
+ }
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(tc.status)
+ }))
+ defer server.Close()
+
+ result := (&Verifier{apiURL: server.URL, httpClient: server.Client()}).Verify(
+ context.Background(), shopifyFinding(),
+ )
+ assert.Equal(t, tc.want, result.Status)
+ })
}
-
- result := v.Verify(context.Background(), raw)
-
- assert.Equal(t, finding.StatusUnverified, result.Status)
- assert.Equal(t, "store domain required for verification", result.Message)
}
-func TestVerify_WithDomain_UsesStoreDomain(t *testing.T) {
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- assert.Equal(t, shopAPIPath, r.URL.Path)
- assert.NotEmpty(t, r.Header.Get("X-Shopify-Access-Token"))
+func TestVerify_MalformedOrGraphQLErrorResponseIsVerifyError(t *testing.T) {
+ tests := []struct {
+ name string
+ contentType string
+ body string
+ }{
+ {name: "missing data", contentType: "application/json", body: `{}`},
+ {name: "null shop", contentType: "application/json", body: `{"data":{"shop":null}}`},
+ {name: "GraphQL errors", contentType: "application/json", body: `{"errors":[{"message":"denied"}]}`},
+ {name: "wrong content type", contentType: "text/html", body: `{"data":{"shop":{"name":"Store"}}}`},
+ }
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", tc.contentType)
+ _, _ = w.Write([]byte(tc.body))
+ }))
+ defer server.Close()
+
+ result := (&Verifier{apiURL: server.URL, httpClient: server.Client()}).Verify(
+ context.Background(), shopifyFinding(),
+ )
+ assert.Equal(t, finding.StatusVerifyError, result.Status)
+ })
+ }
+}
- w.Header().Set("Content-Type", "application/json")
- w.WriteHeader(http.StatusOK)
- _, _ = w.Write([]byte(`{"shop":{"name":"Domain Store"}}`))
+func TestVerify_ServerErrorReturnsError(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusInternalServerError)
}))
defer server.Close()
- // Use apiURL to simulate domain-based URL construction in tests.
- v := &Verifier{
- apiURL: server.URL,
- httpClient: server.Client(),
- }
-
- raw := detector.RawFinding{
- DetectorID: detectorID,
- Raw: []byte("shpat_abcdef1234567890abcdef1234567890"),
- Redacted: "shpat_****7890",
- ExtraData: map[string]string{
- "store_domain": "mystore.myshopify.com",
- },
- }
-
- result := v.Verify(context.Background(), raw)
+ result := (&Verifier{apiURL: server.URL, httpClient: server.Client()}).Verify(
+ context.Background(), shopifyFinding(),
+ )
+ assert.Equal(t, finding.StatusVerifyError, result.Status)
+}
- require.Equal(t, finding.StatusVerifiedActive, result.Status)
- assert.Equal(t, "Shopify access token is active", result.Message)
- assert.Equal(t, "Domain Store", result.ExtraData["shop_name"])
+func TestVerify_TypeAndEmptyToken(t *testing.T) {
+ assert.Equal(t, detectorID, (&Verifier{}).Type())
+ result := (&Verifier{}).Verify(context.Background(), detector.RawFinding{})
+ assert.Equal(t, finding.StatusUnverified, result.Status)
+ assert.Equal(t, "empty token", result.Message)
}
diff --git a/internal/verifier/snowflake/snowflake_verifier_test.go b/internal/verifier/snowflake/snowflake_verifier_test.go
index 5e09974..ca04fdb 100644
--- a/internal/verifier/snowflake/snowflake_verifier_test.go
+++ b/internal/verifier/snowflake/snowflake_verifier_test.go
@@ -7,9 +7,22 @@ import (
"github.com/stretchr/testify/assert"
"github.com/HodeTech/leakwatch/internal/detector"
+ snowflakedetector "github.com/HodeTech/leakwatch/internal/detector/snowflake"
+ "github.com/HodeTech/leakwatch/internal/detector/testutil"
"github.com/HodeTech/leakwatch/pkg/finding"
+ "github.com/stretchr/testify/require"
)
+func TestVerify_RealDetectorOutput_ValidatesPairedConnection(t *testing.T) {
+ fixture := testutil.RegisteredDetectorFixtures()[detectorID]
+ findings := testutil.ScanViaMatcher(&snowflakedetector.Detector{}, fixture.Input)
+ require.Len(t, findings, 1)
+ require.NotEmpty(t, findings[0].RawV2)
+ result := (&Verifier{}).Verify(t.Context(), findings[0])
+ assert.Equal(t, finding.StatusUnverified, result.Status)
+ assert.Contains(t, result.Message, "format valid")
+}
+
func TestVerifier_Type_ReturnsCorrectID(t *testing.T) {
v := &Verifier{}
assert.Equal(t, "snowflake-credentials", v.Type())
diff --git a/internal/verifier/snyk/snyk_verifier.go b/internal/verifier/snyk/snyk_verifier.go
index 04d2598..21440b1 100644
--- a/internal/verifier/snyk/snyk_verifier.go
+++ b/internal/verifier/snyk/snyk_verifier.go
@@ -4,8 +4,12 @@ package snyk
import (
"context"
+ "encoding/json"
+ "fmt"
+ "io"
"net/http"
"net/url"
+ "strings"
"github.com/HodeTech/leakwatch/internal/detector"
"github.com/HodeTech/leakwatch/internal/verifier"
@@ -15,22 +19,36 @@ import (
const detectorID = "snyk-api-key"
-// defaultAPIURL is the base URL for the Snyk API.
-const defaultAPIURL = "https://api.snyk.io"
-
// apiVersion is the Snyk REST API version. The REST API mandates a
// ?version=YYYY-MM-DD query parameter; omitting it makes the API respond 400.
-const apiVersion = "2024-04-29"
+const apiVersion = "2024-10-15"
// Verifier checks whether a Snyk API key is active by calling the
// Snyk REST API. It NEVER logs or persists raw key values.
type Verifier struct {
- // apiURL overrides the Snyk API base URL (for testing).
+ // apiURL is reserved for a validated, trusted Snyk API origin. The
+ // production registration leaves it empty until operator wiring exists;
+ // tests inject a local server.
apiURL string
// httpClient overrides the default HTTP client (for testing).
httpClient *http.Client
}
+// NewForTrustedInstance constructs a Snyk verifier for an explicit regional,
+// government, or operator-controlled private Snyk API origin.
+func NewForTrustedInstance(instanceURL string) (*Verifier, error) {
+ normalized, err := verifier.NormalizeTrustedHTTPSOrigin(instanceURL)
+ if err != nil {
+ return nil, err
+ }
+ return &Verifier{apiURL: normalized}, nil
+}
+
+// WithTrustedInstance implements verifier.TrustedInstanceConfigurer.
+func (*Verifier) WithTrustedInstance(instanceURL string) (verifier.Verifier, error) {
+ return NewForTrustedInstance(instanceURL)
+}
+
func init() {
verifier.Register(&Verifier{})
}
@@ -44,7 +62,16 @@ func (v *Verifier) Type() string {
// Raw contains the key value.
func (v *Verifier) Verify(ctx context.Context, raw detector.RawFinding) finding.VerificationResult {
token := string(raw.Raw)
- apiURL := httpx.BaseURL(v.apiURL, defaultAPIURL)
+ if token == "" {
+ return finding.VerificationResult{Status: finding.StatusUnverified, Message: "empty token"}
+ }
+ if v.apiURL == "" {
+ return finding.VerificationResult{
+ Status: finding.StatusUnverified,
+ Message: "trusted Snyk API origin is not configured",
+ }
+ }
+ apiURL := v.apiURL
// The Snyk REST API requires the version as a query parameter; without it
// the live API returns 400. The Version header is kept for compatibility.
@@ -59,8 +86,44 @@ func (v *Verifier) Verify(ctx context.Context, raw detector.RawFinding) finding.
"Version": apiVersion,
},
},
- InactiveStatuses: []int{http.StatusUnauthorized, http.StatusForbidden},
- ActiveMessage: "Snyk API key is active",
- InactiveMessage: "Snyk API key is invalid or revoked",
+ InactiveStatuses: []int{http.StatusUnauthorized},
+ ActiveMessage: "Snyk API key is active",
+ InactiveMessage: "Snyk API key is invalid or revoked",
+ Decode: decodeSelf,
+ DecodeInactive: decodeInactiveResponse,
+ RequireCompleteBody: true,
+ RequireJSONContentType: true,
})
}
+
+func decodeInactiveResponse(body io.Reader) error {
+ var response struct {
+ Code int `json:"code"`
+ Message string `json:"message"`
+ }
+ decoder := json.NewDecoder(body)
+ if err := decoder.Decode(&response); err != nil {
+ return err
+ }
+ if response.Code != http.StatusUnauthorized ||
+ strings.ToLower(strings.TrimSpace(response.Message)) != "invalid auth token provided" {
+ return fmt.Errorf("unexpected Snyk authentication error")
+ }
+ if decoder.Decode(&struct{}{}) != io.EOF {
+ return fmt.Errorf("unexpected trailing Snyk response data")
+ }
+ return nil
+}
+
+func decodeSelf(body io.Reader) (map[string]string, string, error) {
+ var response struct {
+ Data json.RawMessage `json:"data"`
+ }
+ if err := json.NewDecoder(body).Decode(&response); err != nil {
+ return nil, "", err
+ }
+ if len(response.Data) == 0 || string(response.Data) == "null" {
+ return nil, "", fmt.Errorf("missing Snyk self data")
+ }
+ return nil, "", nil
+}
diff --git a/internal/verifier/snyk/snyk_verifier_test.go b/internal/verifier/snyk/snyk_verifier_test.go
index 904289c..52d6edf 100644
--- a/internal/verifier/snyk/snyk_verifier_test.go
+++ b/internal/verifier/snyk/snyk_verifier_test.go
@@ -17,9 +17,9 @@ func TestVerify_ValidKey_ReturnsActive(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, "/rest/self", r.URL.Path)
// The Snyk REST API requires the version as a query parameter.
- assert.Equal(t, "2024-04-29", r.URL.Query().Get("version"))
+ assert.Equal(t, "2024-10-15", r.URL.Query().Get("version"))
assert.Equal(t, "token test-snyk-api-key-abcdef1234567890", r.Header.Get("Authorization"))
- assert.Equal(t, "2024-04-29", r.Header.Get("Version"))
+ assert.Equal(t, "2024-10-15", r.Header.Get("Version"))
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
@@ -46,6 +46,7 @@ func TestVerify_ValidKey_ReturnsActive(t *testing.T) {
func TestVerify_InvalidKey_ReturnsInactive(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusUnauthorized)
_, _ = w.Write([]byte(`{"code":401,"message":"Invalid auth token provided"}`))
}))
@@ -68,7 +69,26 @@ func TestVerify_InvalidKey_ReturnsInactive(t *testing.T) {
assert.Equal(t, "Snyk API key is invalid or revoked", result.Message)
}
-func TestVerify_ForbiddenKey_ReturnsInactive(t *testing.T) {
+func TestVerify_InactiveResponseMustBeDefinitiveJSON(t *testing.T) {
+ tests := []struct{ contentType, body string }{
+ {contentType: "text/html", body: `login`},
+ {contentType: "application/json", body: `{"code":401,"message":"challenge"}`},
+ }
+ for _, tc := range tests {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", tc.contentType)
+ w.WriteHeader(http.StatusUnauthorized)
+ _, _ = w.Write([]byte(tc.body))
+ }))
+ result := (&Verifier{apiURL: server.URL, httpClient: server.Client()}).Verify(
+ context.Background(), detector.RawFinding{Raw: []byte("synthetic-snyk-key")},
+ )
+ server.Close()
+ assert.Equal(t, finding.StatusVerifyError, result.Status)
+ }
+}
+
+func TestVerify_ForbiddenKey_ReturnsVerifyError(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusForbidden)
_, _ = w.Write([]byte(`{"code":403,"message":"Forbidden"}`))
@@ -88,8 +108,35 @@ func TestVerify_ForbiddenKey_ReturnsInactive(t *testing.T) {
result := v.Verify(context.Background(), raw)
- assert.Equal(t, finding.StatusVerifiedInactive, result.Status)
- assert.Equal(t, "Snyk API key is invalid or revoked", result.Message)
+ assert.Equal(t, finding.StatusVerifyError, result.Status)
+ assert.Contains(t, result.Message, "403")
+}
+
+func TestVerify_MalformedSuccessResponse_ReturnsVerifyError(t *testing.T) {
+ tests := []struct {
+ name string
+ contentType string
+ body string
+ }{
+ {name: "missing data", contentType: "application/vnd.api+json", body: `{}`},
+ {name: "null data", contentType: "application/json", body: `{"data":null}`},
+ {name: "wrong content type", contentType: "text/plain", body: `{"data":{}}`},
+ }
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", tc.contentType)
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write([]byte(tc.body))
+ }))
+ defer server.Close()
+
+ result := (&Verifier{apiURL: server.URL, httpClient: server.Client()}).Verify(
+ context.Background(), detector.RawFinding{Raw: []byte("synthetic-snyk-key")},
+ )
+ assert.Equal(t, finding.StatusVerifyError, result.Status)
+ })
+ }
}
func TestVerify_ServerError_ReturnsError(t *testing.T) {
@@ -135,3 +182,21 @@ func TestVerify_EmptyToken_ReturnsUnverified(t *testing.T) {
assert.Equal(t, finding.StatusUnverified, result.Status)
assert.Equal(t, "empty token", result.Message)
}
+
+func TestVerify_WithoutTrustedOrigin_MakesNoRequest(t *testing.T) {
+ v := &Verifier{}
+ result := v.Verify(context.Background(), detector.RawFinding{Raw: []byte("synthetic-snyk-key")})
+
+ assert.Equal(t, finding.StatusUnverified, result.Status)
+ assert.Equal(t, "trusted Snyk API origin is not configured", result.Message)
+}
+
+func TestNewForTrustedInstance_ValidatesOrigin(t *testing.T) {
+ configured, err := NewForTrustedInstance("https://api.eu.snyk.io/")
+ require.NoError(t, err)
+ assert.Equal(t, "https://api.eu.snyk.io", configured.apiURL)
+ for _, origin := range []string{"http://api.snyk.io", "https://127.0.0.1", "https://localhost", "https://api.snyk.io/rest"} {
+ _, err := NewForTrustedInstance(origin)
+ assert.Error(t, err, origin)
+ }
+}
diff --git a/internal/verifier/sonarcloud/sonarcloud_shared_test.go b/internal/verifier/sonarcloud/sonarcloud_shared_test.go
new file mode 100644
index 0000000..3ab6b4c
--- /dev/null
+++ b/internal/verifier/sonarcloud/sonarcloud_shared_test.go
@@ -0,0 +1,16 @@
+package sonarcloud
+
+import (
+ "net/http"
+ "testing"
+
+ "github.com/HodeTech/leakwatch/internal/detector"
+ "github.com/HodeTech/leakwatch/internal/verifier"
+ "github.com/HodeTech/leakwatch/internal/verifier/internal/vtest"
+)
+
+func TestVerify_SharedSafetySuite(t *testing.T) {
+ vtest.Run(t, vtest.Case{Name: "sonarcloud", New: func(url string, client *http.Client) verifier.Verifier {
+ return &Verifier{apiURL: url, httpClient: client}
+ }, Raw: detector.RawFinding{DetectorID: detectorID, Raw: []byte("sqp_abcdef0123456789abcdef0123456789abcdef01")}})
+}
diff --git a/internal/verifier/supabase/supabase_shared_test.go b/internal/verifier/supabase/supabase_shared_test.go
new file mode 100644
index 0000000..4c40f31
--- /dev/null
+++ b/internal/verifier/supabase/supabase_shared_test.go
@@ -0,0 +1,16 @@
+package supabase
+
+import (
+ "net/http"
+ "testing"
+
+ "github.com/HodeTech/leakwatch/internal/detector"
+ "github.com/HodeTech/leakwatch/internal/verifier"
+ "github.com/HodeTech/leakwatch/internal/verifier/internal/vtest"
+)
+
+func TestVerify_SharedSafetySuite(t *testing.T) {
+ vtest.Run(t, vtest.Case{Name: "supabase", New: func(url string, client *http.Client) verifier.Verifier {
+ return &Verifier{apiURL: url, httpClient: client}
+ }, Raw: detector.RawFinding{DetectorID: detectorID, Raw: []byte("sbp_abcdef0123456789abcdef0123456789abcdef01")}, SkipMalformed: true})
+}
diff --git a/internal/verifier/supabase/supabase_verifier.go b/internal/verifier/supabase/supabase_verifier.go
index a669598..57c522b 100644
--- a/internal/verifier/supabase/supabase_verifier.go
+++ b/internal/verifier/supabase/supabase_verifier.go
@@ -1,5 +1,5 @@
-// Package supabase provides a verifier for Supabase service keys.
-// It uses the Supabase Management API GET /v1/projects endpoint to check key validity.
+// Package supabase provides a verifier for Supabase personal access tokens. It
+// uses the Supabase Management API GET /v1/projects endpoint to check validity.
package supabase
import (
@@ -17,8 +17,8 @@ const detectorID = "supabase-service-key"
// defaultAPIURL is the base URL for the Supabase Management API.
const defaultAPIURL = "https://api.supabase.com"
-// Verifier checks whether a Supabase service key is active by calling the
-// Supabase Management API. It NEVER logs or persists raw key values.
+// Verifier checks whether a Supabase personal access token is active by calling
+// the Supabase Management API. It NEVER logs or persists raw token values.
type Verifier struct {
// apiURL overrides the Supabase API base URL (for testing).
apiURL string
@@ -35,7 +35,7 @@ func (v *Verifier) Type() string {
return detectorID
}
-// Verify checks if the detected Supabase service key is valid/active.
+// Verify checks if the detected Supabase personal access token is valid/active.
// Raw contains the key value.
func (v *Verifier) Verify(ctx context.Context, raw detector.RawFinding) finding.VerificationResult {
token := string(raw.Raw)
@@ -47,7 +47,7 @@ func (v *Verifier) Verify(ctx context.Context, raw detector.RawFinding) finding.
URL: apiURL + "/v1/projects",
Header: map[string]string{"Authorization": "Bearer " + token},
},
- ActiveMessage: "Supabase service key is active",
- InactiveMessage: "Supabase service key is invalid or revoked",
+ ActiveMessage: "Supabase personal access token is active",
+ InactiveMessage: "Supabase personal access token is invalid or revoked",
})
}
diff --git a/internal/verifier/supabase/supabase_verifier_test.go b/internal/verifier/supabase/supabase_verifier_test.go
index c9bd28e..d1e1a76 100644
--- a/internal/verifier/supabase/supabase_verifier_test.go
+++ b/internal/verifier/supabase/supabase_verifier_test.go
@@ -38,7 +38,7 @@ func TestVerify_ValidKey_ReturnsActive(t *testing.T) {
result := v.Verify(context.Background(), raw)
require.Equal(t, finding.StatusVerifiedActive, result.Status)
- assert.Equal(t, "Supabase service key is active", result.Message)
+ assert.Equal(t, "Supabase personal access token is active", result.Message)
}
func TestVerify_InvalidKey_ReturnsInactive(t *testing.T) {
@@ -62,7 +62,21 @@ func TestVerify_InvalidKey_ReturnsInactive(t *testing.T) {
result := v.Verify(context.Background(), raw)
assert.Equal(t, finding.StatusVerifiedInactive, result.Status)
- assert.Equal(t, "Supabase service key is invalid or revoked", result.Message)
+ assert.Equal(t, "Supabase personal access token is invalid or revoked", result.Message)
+}
+
+func TestVerify_Forbidden_RemainsInconclusive(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusForbidden)
+ _, _ = w.Write([]byte(`{"message":"Forbidden"}`))
+ }))
+ defer server.Close()
+
+ v := &Verifier{apiURL: server.URL, httpClient: server.Client()}
+ result := v.Verify(context.Background(), detector.RawFinding{Raw: []byte("synthetic-sbp-token")})
+
+ assert.Equal(t, finding.StatusVerifyError, result.Status)
+ assert.NotEqual(t, finding.StatusVerifiedInactive, result.Status)
}
func TestVerify_ServerError_ReturnsError(t *testing.T) {
diff --git a/internal/verifier/teams/teams_shared_test.go b/internal/verifier/teams/teams_shared_test.go
new file mode 100644
index 0000000..bb3f08c
--- /dev/null
+++ b/internal/verifier/teams/teams_shared_test.go
@@ -0,0 +1,23 @@
+package teams
+
+import (
+ "net/http"
+ "testing"
+
+ "github.com/HodeTech/leakwatch/internal/detector"
+ "github.com/HodeTech/leakwatch/internal/verifier"
+ "github.com/HodeTech/leakwatch/internal/verifier/internal/vtest"
+)
+
+func TestVerify_SharedSafetySuite(t *testing.T) {
+ vtest.Run(t, vtest.Case{
+ Name: "teams",
+ New: func(_ string, client *http.Client) verifier.Verifier {
+ return &Verifier{httpClient: client}
+ },
+ RawForURL: func(endpoint string) detector.RawFinding {
+ return detector.RawFinding{DetectorID: detectorID, Raw: []byte(endpoint), Redacted: "https://teams.invalid/****"}
+ },
+ SkipMalformed: true,
+ })
+}
diff --git a/internal/verifier/teams/teams_verifier.go b/internal/verifier/teams/teams_verifier.go
index 3c6439d..91d9607 100644
--- a/internal/verifier/teams/teams_verifier.go
+++ b/internal/verifier/teams/teams_verifier.go
@@ -36,6 +36,8 @@ type Verifier struct {
httpClient *http.Client
}
+var _ verifier.RequestGatedVerifier = (*Verifier)(nil)
+
func init() {
verifier.Register(&Verifier{})
}
@@ -54,6 +56,26 @@ func (v *Verifier) Type() string {
// - 404 Not Found -> inactive (webhook deleted or disabled)
// - anything else -> unverified (cannot decide non-destructively)
func (v *Verifier) Verify(ctx context.Context, raw detector.RawFinding) finding.VerificationResult {
+ return v.verify(ctx, raw, nil)
+}
+
+// VerificationRequestBudget declares the single non-destructive Teams probe.
+func (v *Verifier) VerificationRequestBudget() int { return 1 }
+
+// VerifyWithRequestGate admits the probe at its actual HTTP send point.
+func (v *Verifier) VerifyWithRequestGate(
+ ctx context.Context,
+ raw detector.RawFinding,
+ gate verifier.RequestGate,
+) finding.VerificationResult {
+ return v.verify(ctx, raw, gate)
+}
+
+func (v *Verifier) verify(
+ ctx context.Context,
+ raw detector.RawFinding,
+ gate verifier.RequestGate,
+) finding.VerificationResult {
webhookURL := string(raw.Raw)
if webhookURL == "" {
return finding.VerificationResult{
@@ -85,6 +107,11 @@ func (v *Verifier) Verify(ctx context.Context, raw detector.RawFinding) finding.
if client == nil {
client = httpx.Client()
}
+ if gate != nil {
+ if rejection := gate(); rejection != nil {
+ return *rejection
+ }
+ }
resp, err := client.Do(req)
if err != nil {
diff --git a/internal/verifier/teams/teams_verifier_test.go b/internal/verifier/teams/teams_verifier_test.go
index 32145f4..4c7d47e 100644
--- a/internal/verifier/teams/teams_verifier_test.go
+++ b/internal/verifier/teams/teams_verifier_test.go
@@ -7,6 +7,7 @@ import (
"net/http"
"net/http/httptest"
"net/url"
+ "sync/atomic"
"testing"
"github.com/stretchr/testify/assert"
@@ -215,3 +216,31 @@ func TestVerify_EmptyURL_ReturnsUnverified(t *testing.T) {
assert.Equal(t, finding.StatusUnverified, result.Status)
assert.Equal(t, "empty webhook URL", result.Message)
}
+
+func TestVerifyWithRequestGate_AdmitsOnlyActualTeamsSend(t *testing.T) {
+ t.Run("empty URL consumes no admission", func(t *testing.T) {
+ gateCalls := 0
+ result := (&Verifier{}).VerifyWithRequestGate(context.Background(), detector.RawFinding{}, func() *finding.VerificationResult {
+ gateCalls++
+ return nil
+ })
+ assert.Equal(t, finding.StatusUnverified, result.Status)
+ assert.Zero(t, gateCalls)
+ })
+
+ t.Run("rejection prevents transport", func(t *testing.T) {
+ var calls atomic.Int64
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ calls.Add(1)
+ w.WriteHeader(http.StatusBadRequest)
+ }))
+ defer server.Close()
+ v := &Verifier{httpClient: server.Client()}
+
+ result := v.VerifyWithRequestGate(context.Background(), detector.RawFinding{Raw: []byte(server.URL)}, func() *finding.VerificationResult {
+ return &finding.VerificationResult{Status: finding.StatusVerifyError, Message: "admission rejected"}
+ })
+ assert.Equal(t, "admission rejected", result.Message)
+ assert.Zero(t, calls.Load())
+ })
+}
diff --git a/internal/verifier/terraform/terraform_shared_test.go b/internal/verifier/terraform/terraform_shared_test.go
new file mode 100644
index 0000000..c1c4595
--- /dev/null
+++ b/internal/verifier/terraform/terraform_shared_test.go
@@ -0,0 +1,16 @@
+package terraform
+
+import (
+ "net/http"
+ "testing"
+
+ "github.com/HodeTech/leakwatch/internal/detector"
+ "github.com/HodeTech/leakwatch/internal/verifier"
+ "github.com/HodeTech/leakwatch/internal/verifier/internal/vtest"
+)
+
+func TestVerify_SharedSafetySuite(t *testing.T) {
+ vtest.Run(t, vtest.Case{Name: "terraform", New: func(url string, client *http.Client) verifier.Verifier {
+ return &Verifier{apiURL: url, httpClient: client}
+ }, Raw: detector.RawFinding{DetectorID: detectorID, Raw: []byte("Ab12Cd34Ef56Gh.atlasv1.SyntheticTerraformCredentialFixture123456789")}})
+}
diff --git a/internal/verifier/trusted_origin.go b/internal/verifier/trusted_origin.go
new file mode 100644
index 0000000..0eee5e7
--- /dev/null
+++ b/internal/verifier/trusted_origin.go
@@ -0,0 +1,114 @@
+package verifier
+
+import (
+ "fmt"
+ "net/netip"
+ "net/url"
+ "strconv"
+ "strings"
+)
+
+// NormalizeTrustedHTTPSOrigin validates an origin explicitly supplied by the
+// operator. Scanned repository content must never be passed to this function:
+// accepting a finding-derived URL would turn verification into a credential
+// exfiltration primitive.
+func NormalizeTrustedHTTPSOrigin(raw string) (string, error) {
+ u, err := parseTrustedHTTPSOrigin(raw)
+ if err != nil {
+ return "", err
+ }
+ hostname, err := trustedOriginHostname(u)
+ if err != nil {
+ return "", err
+ }
+ u.Host = hostnameWithOptionalPort(hostname, u.Port())
+ u.Scheme = "https"
+ u.Path = ""
+ u.RawPath = ""
+ return strings.TrimRight(u.String(), "/"), nil
+}
+
+func parseTrustedHTTPSOrigin(raw string) (*url.URL, error) {
+ u, err := url.ParseRequestURI(strings.TrimSpace(raw))
+ if err != nil {
+ return nil, fmt.Errorf("invalid trusted origin: %w", err)
+ }
+ if !strings.EqualFold(u.Scheme, "https") || u.Host == "" || u.Opaque != "" {
+ return nil, fmt.Errorf("invalid trusted origin: an absolute HTTPS origin is required")
+ }
+ if u.User != nil || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" {
+ return nil, fmt.Errorf("invalid trusted origin: userinfo, query, and fragment are not allowed")
+ }
+ if u.EscapedPath() != "" && u.EscapedPath() != "/" {
+ return nil, fmt.Errorf("invalid trusted origin: a base origin without a path is required")
+ }
+ return u, nil
+}
+
+func trustedOriginHostname(u *url.URL) (string, error) {
+ hostname := strings.ToLower(strings.TrimSuffix(u.Hostname(), "."))
+ if hostname == "" || strings.ContainsAny(hostname, "* \t\r\n") {
+ return "", fmt.Errorf("invalid trusted origin: a concrete hostname is required")
+ }
+ if hostname == "localhost" || strings.HasSuffix(hostname, ".localhost") {
+ return "", fmt.Errorf("invalid trusted origin: local targets are not allowed")
+ }
+ if _, parseErr := netip.ParseAddr(hostname); parseErr == nil {
+ return "", fmt.Errorf("invalid trusted origin: IP-literal targets are not allowed")
+ }
+ if looksLikeNumericHost(hostname) {
+ return "", fmt.Errorf("invalid trusted origin: non-canonical numeric targets are not allowed")
+ }
+ if err := validateTrustedOriginPort(u.Port()); err != nil {
+ return "", err
+ }
+ return hostname, nil
+}
+
+func validateTrustedOriginPort(port string) error {
+ if port != "" {
+ value, portErr := strconv.ParseUint(port, 10, 16)
+ if portErr != nil || value == 0 {
+ return fmt.Errorf("invalid trusted origin: port must be between 1 and 65535")
+ }
+ }
+ return nil
+}
+
+func hostnameWithOptionalPort(hostname, port string) string {
+ if port != "" {
+ return hostname + ":" + port
+ }
+ return hostname
+}
+
+func looksLikeNumericHost(hostname string) bool {
+ parts := strings.Split(hostname, ".")
+ if len(parts) == 0 {
+ return false
+ }
+ for _, part := range parts {
+ if part == "" {
+ continue
+ }
+ digits := part
+ base := byte(10)
+ if strings.HasPrefix(part, "0x") {
+ digits = part[2:]
+ base = 16
+ }
+ if digits == "" {
+ return false
+ }
+ for _, r := range digits {
+ if r >= '0' && r <= '9' {
+ continue
+ }
+ if base == 16 && r >= 'a' && r <= 'f' {
+ continue
+ }
+ return false
+ }
+ }
+ return true
+}
diff --git a/internal/verifier/trusted_origin_test.go b/internal/verifier/trusted_origin_test.go
new file mode 100644
index 0000000..5d7e4a3
--- /dev/null
+++ b/internal/verifier/trusted_origin_test.go
@@ -0,0 +1,42 @@
+package verifier
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestNormalizeTrustedHTTPSOrigin(t *testing.T) {
+ tests := []struct {
+ name string
+ raw string
+ want string
+ }{
+ {name: "canonical", raw: "https://provider.example", want: "https://provider.example"},
+ {name: "trim and normalize host", raw: " https://PROVIDER.Example./ ", want: "https://provider.example"},
+ {name: "self managed port", raw: "https://gitlab.example:8443/", want: "https://gitlab.example:8443"},
+ }
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ got, err := NormalizeTrustedHTTPSOrigin(tc.raw)
+ require.NoError(t, err)
+ assert.Equal(t, tc.want, got)
+ })
+ }
+
+ for _, raw := range []string{
+ "", "provider.example", "http://provider.example", "https://provider.example/path",
+ "https://provider.example?next=x", "https://provider.example/#fragment",
+ "https://user:pass@provider.example", "https://*.provider.example",
+ "https://localhost", "https://api.localhost", "https://127.0.0.1",
+ "https://127.1", "https://0x7f000001", "https://0x7f.0x0.0x0.0x1",
+ "https://provider.example:0",
+ "https://provider.example:65536", "https://[::1]", "https://[fe80::1%25en0]",
+ } {
+ t.Run("reject_"+raw, func(t *testing.T) {
+ _, err := NormalizeTrustedHTTPSOrigin(raw)
+ assert.Error(t, err)
+ })
+ }
+}
diff --git a/internal/verifier/twilio/twilio_shared_test.go b/internal/verifier/twilio/twilio_shared_test.go
new file mode 100644
index 0000000..bb7f130
--- /dev/null
+++ b/internal/verifier/twilio/twilio_shared_test.go
@@ -0,0 +1,20 @@
+package twilio
+
+import (
+ "net/http"
+ "testing"
+
+ "github.com/HodeTech/leakwatch/internal/detector"
+ "github.com/HodeTech/leakwatch/internal/verifier"
+ "github.com/HodeTech/leakwatch/internal/verifier/internal/vtest"
+)
+
+func TestVerify_SharedSafetySuite(t *testing.T) {
+ vtest.Run(t, vtest.Case{Name: "twilio", New: func(url string, client *http.Client) verifier.Verifier {
+ return &Verifier{apiURL: url, httpClient: client}
+ }, Raw: detector.RawFinding{
+ DetectorID: detectorID,
+ Raw: []byte("SyntheticTwilioAPIKeySecretFixture"),
+ ExtraData: map[string]string{"api_key_sid": "SKabcdef0123456789abcdef0123456789"},
+ }})
+}
diff --git a/internal/verifier/twilio/twilio_verifier.go b/internal/verifier/twilio/twilio_verifier.go
index dea0820..0d35e79 100644
--- a/internal/verifier/twilio/twilio_verifier.go
+++ b/internal/verifier/twilio/twilio_verifier.go
@@ -1,11 +1,17 @@
-// Package twilio provides a verifier for Twilio API keys.
-// It uses the Twilio Accounts API GET /2010-04-01/Accounts.json endpoint
-// with Basic auth to check key validity.
+// Package twilio provides conservative verification for paired Twilio API Key
+// credentials. Production makes no request until a trusted regional API origin
+// is supplied by operator-controlled configuration.
package twilio
import (
"context"
+ "encoding/json"
+ "errors"
+ "io"
"net/http"
+ "net/url"
+ "regexp"
+ "strings"
"github.com/HodeTech/leakwatch/internal/detector"
"github.com/HodeTech/leakwatch/internal/verifier"
@@ -15,80 +21,120 @@ import (
const detectorID = "twilio-api-key"
-// defaultAPIURL is the base URL for the Twilio API.
-const defaultAPIURL = "https://api.twilio.com"
+var (
+ apiKeySIDPattern = regexp.MustCompile(`^SK[0-9a-fA-F]{32}$`)
+ accountSIDPattern = regexp.MustCompile(`^AC[0-9a-fA-F]{32}$`)
+)
-// Verifier checks whether a Twilio API key is active by calling the
-// Twilio Accounts API. It NEVER logs or persists raw key values.
+// Verifier checks a Twilio API Key Secret with its paired API Key SID. apiURL
+// is an unexported trusted-origin seam; the production registration leaves it
+// empty because Twilio credentials are region-specific.
type Verifier struct {
- // apiURL overrides the Twilio API base URL (for testing).
- apiURL string
- // httpClient overrides the default HTTP client (for testing).
+ apiURL string
httpClient *http.Client
}
+// NewForTrustedInstance accepts an operator-selected official Twilio regional
+// API origin. Region selection is never inferred from scanned content.
+func NewForTrustedInstance(instanceURL string) (*Verifier, error) {
+ normalized, err := verifier.NormalizeTrustedHTTPSOrigin(instanceURL)
+ if err != nil {
+ return nil, err
+ }
+ u, err := url.Parse(normalized)
+ hostname := u.Hostname()
+ if err != nil || u.Port() != "" ||
+ (hostname != "api.twilio.com" &&
+ (!strings.HasPrefix(hostname, "api.") || !strings.HasSuffix(hostname, ".twilio.com"))) {
+ return nil, errors.New("invalid Twilio API origin: an official twilio.com host is required")
+ }
+ return &Verifier{apiURL: normalized}, nil
+}
+
+// WithTrustedInstance implements verifier.TrustedInstanceConfigurer.
+func (*Verifier) WithTrustedInstance(instanceURL string) (verifier.Verifier, error) {
+ return NewForTrustedInstance(instanceURL)
+}
+
func init() {
verifier.Register(&Verifier{})
}
// Type returns the detector ID this verifier handles.
-func (v *Verifier) Type() string {
- return detectorID
-}
+func (v *Verifier) Type() string { return detectorID }
-// Verify checks if the detected Twilio API key is valid/active.
-//
-// The detector emits the API Key SID ("SK...") as raw.Raw and the Account SID
-// ("AC...") as raw.ExtraData["account_sid"]. Twilio's REST API accepts exactly
-// two Basic-Auth pairs: (Account SID, Auth Token) or (API Key SID, API Key
-// Secret). Since raw.Raw is an API Key SID, the only valid pairing is
-// (API Key SID, API Key Secret): the SID is the Basic-Auth username and the
-// paired API Key Secret is the password. The API Key SID alone is a non-secret
-// identifier and cannot authenticate on its own, so when the paired secret is
-// not available (raw.ExtraData["api_key_secret"] is empty) the key is reported
-// as Unverified — never as inactive, which would misread a live but unpaired
-// credential as revoked.
+// Verify authenticates with API Key SID as the Basic username and API Key
+// Secret as the password. Main, Standard, and Restricted keys have different
+// permissions: only 401 proves inactive; a permission-denied 403 is
+// deliberately a verify_error.
func (v *Verifier) Verify(ctx context.Context, raw detector.RawFinding) finding.VerificationResult {
- apiKeySID := string(raw.Raw)
- if apiKeySID == "" {
- return finding.VerificationResult{
- Status: finding.StatusUnverified,
- Message: "empty token",
- }
+ apiKeySecret := string(raw.Raw)
+ if apiKeySecret == "" {
+ return finding.VerificationResult{Status: finding.StatusUnverified, Message: "empty token"}
}
- var accountSID, apiKeySecret string
+ apiKeySID := ""
+ accountSID := ""
if raw.ExtraData != nil {
+ apiKeySID = raw.ExtraData["api_key_sid"]
accountSID = raw.ExtraData["account_sid"]
- apiKeySecret = raw.ExtraData["api_key_secret"]
}
-
- if apiKeySecret == "" {
- // Without the paired API Key Secret the credential cannot be
- // authenticated. Report Unverified rather than guessing at liveness.
+ if !apiKeySIDPattern.MatchString(apiKeySID) {
return finding.VerificationResult{
Status: finding.StatusUnverified,
- Message: "API Key Secret unavailable; cannot verify",
+ Message: "paired Twilio API Key SID is required for verification",
+ }
+ }
+ if accountSID != "" && !accountSIDPattern.MatchString(accountSID) {
+ return finding.VerificationResult{
+ Status: finding.StatusUnverified,
+ Message: "invalid Twilio Account SID context",
+ }
+ }
+ if v.apiURL == "" {
+ return finding.VerificationResult{
+ Status: finding.StatusUnverified,
+ Message: "trusted Twilio regional API origin is not configured",
}
}
- apiURL := httpx.BaseURL(v.apiURL, defaultAPIURL)
- // When the Account SID is known, scope the request to that account;
- // otherwise fall back to the accounts collection, which an API Key can also
- // reach.
path := "/2010-04-01/Accounts.json"
if accountSID != "" {
path = "/2010-04-01/Accounts/" + accountSID + ".json"
}
- return httpx.VerifyToken(ctx, v.httpClient, apiKeySID, httpx.TokenSpec{
+ return httpx.VerifyToken(ctx, v.httpClient, apiKeySecret, httpx.TokenSpec{
Name: "twilio",
Request: httpx.Request{
- URL: apiURL + path,
+ URL: v.apiURL + path,
BasicAuthUser: apiKeySID,
BasicAuthPass: apiKeySecret,
},
- ActiveMessage: "Twilio API key is active",
- InactiveMessage: "Twilio API key is invalid or revoked",
+ InactiveStatuses: []int{http.StatusUnauthorized},
+ ActiveMessage: "Twilio API key is active for the trusted region and probe",
+ InactiveMessage: "Twilio API key is invalid or revoked on the trusted region",
+ Decode: func(body io.Reader) (map[string]string, string, error) { return decodeAccountProbe(body, accountSID) },
+ RequireCompleteBody: true,
+ RequireJSONContentType: true,
})
}
+
+func decodeAccountProbe(body io.Reader, expectedAccountSID string) (map[string]string, string, error) {
+ var response struct {
+ SID string `json:"sid"`
+ Accounts json.RawMessage `json:"accounts"`
+ }
+ if err := json.NewDecoder(body).Decode(&response); err != nil {
+ return nil, "", err
+ }
+ if expectedAccountSID != "" {
+ if response.SID != expectedAccountSID {
+ return nil, "", errors.New("twilio account probe returned an unexpected account")
+ }
+ return map[string]string{"account_sid": expectedAccountSID}, "", nil
+ }
+ if len(response.Accounts) == 0 || string(response.Accounts) == "null" {
+ return nil, "", errors.New("twilio account collection is missing")
+ }
+ return nil, "", nil
+}
diff --git a/internal/verifier/twilio/twilio_verifier_test.go b/internal/verifier/twilio/twilio_verifier_test.go
index a0d29a2..7f67716 100644
--- a/internal/verifier/twilio/twilio_verifier_test.go
+++ b/internal/verifier/twilio/twilio_verifier_test.go
@@ -8,180 +8,172 @@ import (
"strings"
"testing"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-
"github.com/HodeTech/leakwatch/internal/detector"
+ twiliodetector "github.com/HodeTech/leakwatch/internal/detector/twilio"
"github.com/HodeTech/leakwatch/pkg/finding"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
)
-// Fixtures mirror the real Twilio detector output: raw.Raw is the API Key SID
-// ("SK" + 32 hex) and ExtraData["account_sid"] is the Account SID ("AC" + 32
-// hex). The API Key Secret is the paired secret required to authenticate.
const (
fixtureAPIKeySID = "SKabcdef0123456789abcdef0123456789"
- fixtureAccount = "AC1234567890abcdef1234567890abcd"
- fixtureSecret = "api_key_secret_0123456789abcdef0"
+ fixtureAccount = "AC1234567890abcdef1234567890ABCDEF"
)
-func TestVerify_ValidKey_ReturnsActive(t *testing.T) {
+func fixtureAPIKeySecret() string { return strings.Repeat("Ab12Cd34", 4) }
+
+func detectorFixture(t *testing.T, includeAccount bool) detector.RawFinding {
+ t.Helper()
+ input := "TWILIO_API_KEY_SID=" + fixtureAPIKeySID + "\n"
+ if includeAccount {
+ input = "TWILIO_ACCOUNT_SID=" + fixtureAccount + "\n" + input
+ }
+ input += "TWILIO_API_KEY_" + "SECRET=" + fixtureAPIKeySecret()
+ findings := (&twiliodetector.Detector{}).Scan(context.Background(), []byte(input))
+ require.Len(t, findings, 1)
+ return findings[0]
+}
+
+func TestVerify_RealDetectorFinding_ReachesTrustedProbe(t *testing.T) {
+ secret := fixtureAPIKeySecret()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, "/2010-04-01/Accounts/"+fixtureAccount+".json", r.URL.Path)
-
- // Verify Basic auth: API Key SID as username, API Key Secret as password.
auth := r.Header.Get("Authorization")
require.True(t, strings.HasPrefix(auth, "Basic "))
decoded, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(auth, "Basic "))
require.NoError(t, err)
- parts := strings.SplitN(string(decoded), ":", 2)
- assert.Equal(t, fixtureAPIKeySID, parts[0])
- assert.Equal(t, fixtureSecret, parts[1])
+ assert.Equal(t, fixtureAPIKeySID+":"+secret, string(decoded))
w.Header().Set("Content-Type", "application/json")
- w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte(`{"sid":"` + fixtureAccount + `"}`))
}))
defer server.Close()
- v := &Verifier{
- apiURL: server.URL,
- httpClient: server.Client(),
- }
-
- raw := detector.RawFinding{
- DetectorID: detectorID,
- Raw: []byte(fixtureAPIKeySID),
- Redacted: "SK****6789",
- ExtraData: map[string]string{
- "account_sid": fixtureAccount,
- "api_key_secret": fixtureSecret,
- },
- }
-
- result := v.Verify(context.Background(), raw)
+ result := (&Verifier{apiURL: server.URL, httpClient: server.Client()}).Verify(
+ context.Background(), detectorFixture(t, true),
+ )
require.Equal(t, finding.StatusVerifiedActive, result.Status)
- assert.Equal(t, "Twilio API key is active", result.Message)
-}
-
-func TestVerify_InvalidKey_ReturnsInactive(t *testing.T) {
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
- w.WriteHeader(http.StatusUnauthorized)
- _, _ = w.Write([]byte(`{"message":"Authenticate"}`))
- }))
- defer server.Close()
-
- v := &Verifier{
- apiURL: server.URL,
- httpClient: server.Client(),
- }
-
- raw := detector.RawFinding{
- DetectorID: detectorID,
- Raw: []byte(fixtureAPIKeySID),
- Redacted: "SK****6789",
- ExtraData: map[string]string{
- "account_sid": fixtureAccount,
- "api_key_secret": fixtureSecret,
- },
+ assert.Equal(t, fixtureAccount, result.ExtraData["account_sid"])
+ assert.NotContains(t, result.Message, secret)
+ for _, value := range result.ExtraData {
+ assert.NotEqual(t, secret, value)
}
+}
- result := v.Verify(context.Background(), raw)
+func TestVerify_ProductionFindingWithoutTrustedOrigin_IsUnverified(t *testing.T) {
+ result := (&Verifier{}).Verify(context.Background(), detectorFixture(t, true))
- assert.Equal(t, finding.StatusVerifiedInactive, result.Status)
- assert.Equal(t, "Twilio API key is invalid or revoked", result.Message)
+ assert.Equal(t, finding.StatusUnverified, result.Status)
+ assert.Equal(t, "trusted Twilio regional API origin is not configured", result.Message)
}
-func TestVerify_ServerError_ReturnsError(t *testing.T) {
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
- w.WriteHeader(http.StatusInternalServerError)
- }))
- defer server.Close()
-
- v := &Verifier{
- apiURL: server.URL,
- httpClient: server.Client(),
+func TestNewForTrustedInstance_AcceptsOnlyTwilioAPIOrigins(t *testing.T) {
+ for _, origin := range []string{
+ "https://api.twilio.com", "https://api.ie1.twilio.com", "https://api.sydney.au1.twilio.com",
+ } {
+ configured, err := NewForTrustedInstance(origin)
+ require.NoError(t, err, origin)
+ assert.Equal(t, origin, configured.apiURL)
}
-
- raw := detector.RawFinding{
- DetectorID: detectorID,
- Raw: []byte(fixtureAPIKeySID),
- Redacted: "SK****6789",
- ExtraData: map[string]string{
- "account_sid": fixtureAccount,
- "api_key_secret": fixtureSecret,
- },
+ for _, origin := range []string{
+ "https://twilio.com", "https://events.twilio.com", "https://api.twilio.com.attacker.example",
+ "https://api.twilio.com:443", "http://api.twilio.com",
+ } {
+ _, err := NewForTrustedInstance(origin)
+ assert.Error(t, err, origin)
}
-
- result := v.Verify(context.Background(), raw)
-
- assert.Equal(t, finding.StatusVerifyError, result.Status)
- assert.Contains(t, result.Message, "500")
}
-// TestVerify_NoAPIKeySecret_ReturnsUnverified verifies that the common real
-// detector output (API Key SID + Account SID, but no paired secret) is reported
-// as Unverified — never falsely inactive.
-func TestVerify_NoAPIKeySecret_ReturnsUnverified(t *testing.T) {
- v := &Verifier{}
-
- raw := detector.RawFinding{
- DetectorID: detectorID,
- Raw: []byte(fixtureAPIKeySID),
- Redacted: "SK****6789",
- ExtraData: map[string]string{"account_sid": fixtureAccount},
+func TestVerify_Only401IsInactive_403IsPermissionError(t *testing.T) {
+ tests := []struct {
+ name string
+ status int
+ want finding.VerificationStatus
+ }{
+ {name: "authentication rejection", status: http.StatusUnauthorized, want: finding.StatusVerifiedInactive},
+ {name: "permission rejection", status: http.StatusForbidden, want: finding.StatusVerifyError},
+ }
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(tc.status)
+ }))
+ defer server.Close()
+
+ result := (&Verifier{apiURL: server.URL, httpClient: server.Client()}).Verify(
+ context.Background(), detectorFixture(t, true),
+ )
+ assert.Equal(t, tc.want, result.Status)
+ })
}
-
- result := v.Verify(context.Background(), raw)
-
- assert.Equal(t, finding.StatusUnverified, result.Status)
- assert.Equal(t, "API Key Secret unavailable; cannot verify", result.Message)
}
-// TestVerify_NoAccountSID_UsesCollectionEndpoint verifies that the request
-// falls back to the accounts collection when no Account SID is known.
-func TestVerify_NoAccountSID_UsesCollectionEndpoint(t *testing.T) {
+func TestVerify_NoAccountSID_UsesCollectionProbe(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, "/2010-04-01/Accounts.json", r.URL.Path)
w.Header().Set("Content-Type", "application/json")
- w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte(`{"accounts":[]}`))
}))
defer server.Close()
- v := &Verifier{
- apiURL: server.URL,
- httpClient: server.Client(),
- }
-
- raw := detector.RawFinding{
- DetectorID: detectorID,
- Raw: []byte(fixtureAPIKeySID),
- Redacted: "SK****6789",
- ExtraData: map[string]string{"api_key_secret": fixtureSecret},
- }
-
- result := v.Verify(context.Background(), raw)
-
+ result := (&Verifier{apiURL: server.URL, httpClient: server.Client()}).Verify(
+ context.Background(), detectorFixture(t, false),
+ )
assert.Equal(t, finding.StatusVerifiedActive, result.Status)
}
-func TestVerify_Type_ReturnsCorrectID(t *testing.T) {
- v := &Verifier{}
- assert.Equal(t, "twilio-api-key", v.Type())
+func TestVerify_MissingOrInvalidCompanionContext_IsUnverified(t *testing.T) {
+ secret := fixtureAPIKeySecret()
+ tests := []detector.RawFinding{
+ {Raw: []byte(secret)},
+ {Raw: []byte(secret), ExtraData: map[string]string{"api_key_sid": "not-a-sid"}},
+ {Raw: []byte(secret), ExtraData: map[string]string{"api_key_sid": fixtureAPIKeySID, "account_sid": "not-an-account"}},
+ }
+ for _, raw := range tests {
+ result := (&Verifier{apiURL: "http://127.0.0.1:1"}).Verify(context.Background(), raw)
+ assert.Equal(t, finding.StatusUnverified, result.Status)
+ }
}
-func TestVerify_EmptyToken_ReturnsUnverified(t *testing.T) {
- v := &Verifier{}
-
- raw := detector.RawFinding{
- DetectorID: detectorID,
- Raw: []byte(""),
- Redacted: "",
+func TestVerify_MalformedSuccessResponse_IsVerifyError(t *testing.T) {
+ tests := []struct {
+ contentType string
+ body string
+ }{
+ {contentType: "application/json", body: `{}`},
+ {contentType: "text/html", body: `{"sid":"` + fixtureAccount + `"}`},
+ {contentType: "application/json", body: `{"sid":"ACaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}`},
}
+ for _, tc := range tests {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", tc.contentType)
+ _, _ = w.Write([]byte(tc.body))
+ }))
+ result := (&Verifier{apiURL: server.URL, httpClient: server.Client()}).Verify(
+ context.Background(), detectorFixture(t, true),
+ )
+ server.Close()
+ assert.Equal(t, finding.StatusVerifyError, result.Status)
+ }
+}
+
+func TestVerify_ServerError_ReturnsError(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusInternalServerError)
+ }))
+ defer server.Close()
- result := v.Verify(context.Background(), raw)
+ result := (&Verifier{apiURL: server.URL, httpClient: server.Client()}).Verify(
+ context.Background(), detectorFixture(t, true),
+ )
+ assert.Equal(t, finding.StatusVerifyError, result.Status)
+}
+func TestVerify_TypeAndEmptyToken(t *testing.T) {
+ assert.Equal(t, detectorID, (&Verifier{}).Type())
+ result := (&Verifier{}).Verify(context.Background(), detector.RawFinding{})
assert.Equal(t, finding.StatusUnverified, result.Status)
assert.Equal(t, "empty token", result.Message)
}
diff --git a/internal/verifier/vercel/vercel_shared_test.go b/internal/verifier/vercel/vercel_shared_test.go
new file mode 100644
index 0000000..cc54fee
--- /dev/null
+++ b/internal/verifier/vercel/vercel_shared_test.go
@@ -0,0 +1,16 @@
+package vercel
+
+import (
+ "net/http"
+ "testing"
+
+ "github.com/HodeTech/leakwatch/internal/detector"
+ "github.com/HodeTech/leakwatch/internal/verifier"
+ "github.com/HodeTech/leakwatch/internal/verifier/internal/vtest"
+)
+
+func TestVerify_SharedSafetySuite(t *testing.T) {
+ vtest.Run(t, vtest.Case{Name: "vercel", New: func(url string, client *http.Client) verifier.Verifier {
+ return &Verifier{apiURL: url, httpClient: client}
+ }, Raw: detector.RawFinding{DetectorID: detectorID, Raw: []byte("vercel_SyntheticVercelCredentialFixture123")}})
+}
diff --git a/internal/verifier/verifier.go b/internal/verifier/verifier.go
index 4160f2b..e1daece 100644
--- a/internal/verifier/verifier.go
+++ b/internal/verifier/verifier.go
@@ -23,6 +23,33 @@ type Verifier interface {
Verify(ctx context.Context, raw detector.RawFinding) finding.VerificationResult
}
+// RequestGate is an engine-owned admission gate for a single outbound
+// verification request. A RequestGatedVerifier must call it immediately before
+// every provider request and must return a non-nil result without sending the
+// request when admission fails.
+type RequestGate func() *finding.VerificationResult
+
+// RequestGatedVerifier is implemented by verifiers that need explicit request
+// admission because they use an SDK/hand-rolled client or may issue more than
+// one provider request (for example a bounded regional fallback). The engine
+// uses the declared maximum to fail closed on runaway attempts while pacing
+// each request at its actual send point.
+type RequestGatedVerifier interface {
+ Verifier
+
+ // VerificationRequestBudget is the maximum number of provider requests that
+ // VerifyWithRequestGate may attempt for one finding.
+ VerificationRequestBudget() int
+
+ // VerifyWithRequestGate performs verification and calls gate immediately
+ // before every outbound provider request.
+ VerifyWithRequestGate(
+ ctx context.Context,
+ raw detector.RawFinding,
+ gate RequestGate,
+ ) finding.VerificationResult
+}
+
// VerifyPair associates a Finding with its corresponding RawFinding
// so the verification engine can pass raw secret data to verifiers
// while returning enriched Finding objects.
diff --git a/pkg/finding/finding.go b/pkg/finding/finding.go
index 1e3a89b..c38b1a7 100644
--- a/pkg/finding/finding.go
+++ b/pkg/finding/finding.go
@@ -226,15 +226,72 @@ type Finding struct {
SourceMetadata SourceMetadata `json:"source"`
Verification VerificationResult `json:"verification"`
Remediation *Remediation `json:"remediation,omitempty"`
- DetectedAt time.Time `json:"detected_at"`
- // Entropy is the Shannon entropy of the raw match, populated only when
- // entropy analysis is enabled. It is omitted (via omitempty) when it holds
- // the Go zero value, so a serialized finding cannot distinguish "not
- // computed" from "computed as exactly 0.0"; consumers should treat an
- // absent entropy as "not computed" rather than zero.
- Entropy float64 `json:"entropy,omitempty"`
+ DetectedAt time.Time `json:"detected_at,omitempty"`
+ // Entropy is the Shannon entropy of the raw match. EntropyCalculated records
+ // whether the value was actually computed, so a legitimate 0.0 is distinct
+ // from "not computed" on the JSON wire without changing the long-standing
+ // public Entropy float64 field. Call SetEntropy when constructing findings
+ // outside the engine. For source compatibility, a non-zero Entropy value is
+ // also serialized even when an older caller did not set EntropyCalculated.
+ Entropy float64 `json:"entropy,omitempty"`
+ EntropyCalculated bool `json:"-"`
// ExtraData carries non-secret contextual metadata. It is json:"-" as a
// defense-in-depth measure (see the type doc); it must never hold secret
// material.
ExtraData map[string]string `json:"-"`
}
+
+// SetEntropy records a computed Shannon entropy value, including a legitimate
+// zero. The explicit presence bit preserves the public float64 API while making
+// the JSON representation nullable.
+func (f *Finding) SetEntropy(value float64) {
+ f.Entropy = value
+ f.EntropyCalculated = true
+}
+
+// MarshalJSON omits zero DetectedAt and emits entropy only when it was
+// calculated. A non-zero legacy value remains visible for source compatibility
+// with callers written before EntropyCalculated existed.
+func (f Finding) MarshalJSON() ([]byte, error) {
+ type alias Finding
+ shadow := struct {
+ alias
+ DetectedAt *time.Time `json:"detected_at,omitempty"`
+ Entropy *float64 `json:"entropy,omitempty"`
+ }{alias: alias(f)}
+ if !f.DetectedAt.IsZero() {
+ shadow.DetectedAt = &f.DetectedAt
+ }
+ if f.EntropyCalculated || f.Entropy != 0 {
+ shadow.Entropy = &f.Entropy
+ }
+ out, err := json.Marshal(shadow)
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal Finding JSON: %w", err)
+ }
+ return out, nil
+}
+
+// UnmarshalJSON restores the entropy presence bit so a computed zero survives a
+// JSON round trip. Missing timestamps and entropy values retain their Go zero
+// values.
+func (f *Finding) UnmarshalJSON(data []byte) error {
+ type alias Finding
+ var decoded alias
+ shadow := struct {
+ *alias
+ DetectedAt *time.Time `json:"detected_at,omitempty"`
+ Entropy *float64 `json:"entropy,omitempty"`
+ }{alias: &decoded}
+ if err := json.Unmarshal(data, &shadow); err != nil {
+ return fmt.Errorf("failed to unmarshal Finding JSON: %w", err)
+ }
+ *f = Finding(decoded)
+ if shadow.DetectedAt != nil {
+ f.DetectedAt = *shadow.DetectedAt
+ }
+ if shadow.Entropy != nil {
+ f.SetEntropy(*shadow.Entropy)
+ }
+ return nil
+}
diff --git a/pkg/finding/finding_test.go b/pkg/finding/finding_test.go
index d96acad..b02ddd0 100644
--- a/pkg/finding/finding_test.go
+++ b/pkg/finding/finding_test.go
@@ -215,8 +215,8 @@ func TestFinding_JSONMarshalUnmarshal_SeverityAsString(t *testing.T) {
Status: StatusUnverified,
},
DetectedAt: now,
- Entropy: 4.5,
}
+ f.SetEntropy(4.5)
data, err := json.Marshal(f)
require.NoError(t, err)
@@ -243,9 +243,58 @@ func TestFinding_JSONMarshalUnmarshal_SeverityAsString(t *testing.T) {
assert.Equal(t, f.SourceMetadata.FilePath, decoded.SourceMetadata.FilePath)
assert.Equal(t, f.SourceMetadata.Line, decoded.SourceMetadata.Line)
assert.Equal(t, f.Verification.Status, decoded.Verification.Status)
+ assert.True(t, decoded.EntropyCalculated)
+ assert.Equal(t, 4.5, decoded.Entropy)
assert.Empty(t, decoded.Raw) // Raw is never serialized via json:"-"
}
+func TestFinding_JSONNullableEntropyAndDetectedAt(t *testing.T) {
+ t.Run("missing values are omitted", func(t *testing.T) {
+ data, err := json.Marshal(Finding{ID: "test-1"})
+ require.NoError(t, err)
+
+ var object map[string]any
+ require.NoError(t, json.Unmarshal(data, &object))
+ assert.NotContains(t, object, "detected_at")
+ assert.NotContains(t, object, "entropy")
+ assert.NotContains(t, string(data), "0001-01-01")
+ })
+
+ t.Run("computed zero is present and round trips", func(t *testing.T) {
+ original := Finding{ID: "test-2"}
+ original.SetEntropy(0)
+
+ data, err := json.Marshal(original)
+ require.NoError(t, err)
+ var object map[string]any
+ require.NoError(t, json.Unmarshal(data, &object))
+ value, exists := object["entropy"]
+ require.True(t, exists)
+ assert.Equal(t, float64(0), value)
+
+ var decoded Finding
+ require.NoError(t, json.Unmarshal(data, &decoded))
+ assert.True(t, decoded.EntropyCalculated)
+ assert.Zero(t, decoded.Entropy)
+ })
+
+ t.Run("legacy non-zero entropy remains source compatible", func(t *testing.T) {
+ data, err := json.Marshal(Finding{ID: "test-3", Entropy: 3.5})
+ require.NoError(t, err)
+ assert.Contains(t, string(data), `"entropy":3.5`)
+ })
+
+ t.Run("missing values clear a reused destination", func(t *testing.T) {
+ destination := Finding{DetectedAt: time.Now().UTC()}
+ destination.SetEntropy(4.2)
+ require.NoError(t, json.Unmarshal([]byte(`{"id":"replacement"}`), &destination))
+ assert.Equal(t, "replacement", destination.ID)
+ assert.True(t, destination.DetectedAt.IsZero())
+ assert.Zero(t, destination.Entropy)
+ assert.False(t, destination.EntropyCalculated)
+ })
+}
+
func TestFinding_JSONOmitsEmptyRaw(t *testing.T) {
f := Finding{
ID: "test-1",
diff --git a/site/README.md b/site/README.md
index f5d6b88..045ae31 100644
--- a/site/README.md
+++ b/site/README.md
@@ -72,16 +72,21 @@ POST without JavaScript. It includes a honeypot field and never asks for secrets
## Editing the documentation
-Manual pages are authored as Markdown under
+Current product behavior is authored in the English Markdown manual under
[`../docs/user-manuals/`](../docs/user-manuals/):
```
docs/user-manuals/
├── _meta.yaml navigation: sections, page order, titles
-├── en//.md English source
-└── tr//.md Turkish source
+├── en//.md authoritative English source
+└── tr//.md reviewed Turkish translation
```
+Update the English page first, then its Turkish counterpart in the same change.
+Files under `js/manuals/` are generated publication artifacts and must never be
+edited by hand. Supplemental long-form guides under `docs/guides/` may add
+operational depth, but they defer product contracts to the user manual.
+
Each page has YAML front matter (`title`, `description`), GFM Markdown, fenced
code blocks, optional `:::tip` / `:::note` / `:::warn` / `:::danger` callouts,
and ` ```mermaid ` diagrams. Cross-links use the hash-router form
diff --git a/site/assets/og.svg b/site/assets/og.svg
index 95593f7..48d9931 100644
--- a/site/assets/og.svg
+++ b/site/assets/og.svg
@@ -21,10 +21,10 @@
- 64 detectors
- 54 live verifiers
- 6 sources
- SARIF · JSON · CSV
+ 65 detectors
+ 39 direct-live checks
+ 6 sources
+ SARIF · JSON · CSV
github.com/HodeTech/Leakwatch · MIT
diff --git a/site/contact.html b/site/contact.html
index 3490a57..dd547f6 100644
--- a/site/contact.html
+++ b/site/contact.html
@@ -141,7 +141,7 @@