[INS-228] Add ignorePattern configuration support to Postgres and Sqlserver detectors (#4612)

* add ignorePattern configuration support to postgres and sqlserver detectors

* add test for sqlserver detector that doesn't use New

Author: mustansir14

pkg/detectors/postgres/postgres.go

@@ -55,6 +55,33 @@ var (
 type Scanner struct {
 	detectors.DefaultMultiPartCredentialProvider
 	detectLoopback bool // Automated tests run against localhost, but we want to ignore those results in the wild
+	ignorePatterns []*regexp.Regexp
+}
+
+func New(opts ...func(*Scanner)) *Scanner {
+	scanner := &Scanner{
+		ignorePatterns: []*regexp.Regexp{},
+	}
+	for _, opt := range opts {
+		opt(scanner)
+	}
+
+	return scanner
+}
+
+func WithIgnorePattern(ignoreStrings []string) func(*Scanner) {
+	return func(s *Scanner) {
+		var ignorePatterns []*regexp.Regexp
+		for _, ignoreString := range ignoreStrings {
+			ignorePattern, err := regexp.Compile(ignoreString)
+			if err != nil {
+				panic(fmt.Sprintf("%s is not a valid regex, error received: %v", ignoreString, err))
+			}
+			ignorePatterns = append(ignorePatterns, ignorePattern)
+		}
+
+		s.ignorePatterns = ignorePatterns
+	}
 }
 
 var _ detectors.Detector = (*Scanner)(nil)
@@ -66,7 +93,7 @@ func (s Scanner) Keywords() []string {
 
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) ([]detectors.Result, error) {
 	var results []detectors.Result
-	candidateParamSets := findUriMatches(data)
+	candidateParamSets := findUriMatches(data, s.ignorePatterns)
 
 	for _, params := range candidateParamSets {
 		if common.IsDone(ctx) {
@@ -161,9 +188,12 @@ func (s Scanner) IsFalsePositive(_ detectors.Result) (bool, string) {
 	return false, ""
 }
 
-func findUriMatches(data []byte) []map[string]string {
+func findUriMatches(data []byte, ignorePatterns []*regexp.Regexp) []map[string]string {
 	var matches []map[string]string
 	for _, uri := range uriPattern.FindAll(data, -1) {
+		if shouldIgnore(uri, ignorePatterns) {
+			continue
+		}
 		// Capture the database type (e.g., "postgres" or "postgresql")
 		dbTypeMatch := uriPattern.FindSubmatch(uri)
 		if len(dbTypeMatch) < 2 {
@@ -188,6 +218,15 @@ func findUriMatches(data []byte) []map[string]string {
 	return matches
 }
 
+func shouldIgnore(uri []byte, ignorePatterns []*regexp.Regexp) bool {
+	for _, ignore := range ignorePatterns {
+		if ignore.Match(uri) {
+			return true
+		}
+	}
+	return false
+}
+
 // getDeadlineInSeconds gets the deadline from the context in seconds. If there
 // is no deadline, false is returned. If the deadline is already exceeded, a
 // negative or 0 value will be returned.

pkg/detectors/postgres/postgres_test.go

@@ -81,3 +81,18 @@ func TestPostgres_Pattern(t *testing.T) {
 		})
 	}
 }
+
+func TestPostgres_FromDataWithIgnorePattern(t *testing.T) {
+	s := New(
+		WithIgnorePattern([]string{
+			`1\.2\.3\.4`,
+		}))
+	got, err := s.FromData(context.Background(), false, []byte(validUriPattern))
+	if err != nil {
+		t.Errorf("FromData() error = %v", err)
+		return
+	}
+	if len(got) != 0 {
+		t.Errorf("expected no results, but got %d", len(got))
+	}
+}

pkg/detectors/sqlserver/sqlserver.go

@@ -18,7 +18,35 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 )
 
-type Scanner struct{}
+type Scanner struct {
+	ignorePatterns []*regexp.Regexp
+}
+
+func New(opts ...func(*Scanner)) *Scanner {
+	scanner := &Scanner{
+		ignorePatterns: []*regexp.Regexp{},
+	}
+	for _, opt := range opts {
+		opt(scanner)
+	}
+
+	return scanner
+}
+
+func WithIgnorePattern(ignoreStrings []string) func(*Scanner) {
+	return func(s *Scanner) {
+		var ignorePatterns []*regexp.Regexp
+		for _, ignoreString := range ignoreStrings {
+			ignorePattern, err := regexp.Compile(ignoreString)
+			if err != nil {
+				panic(fmt.Sprintf("%s is not a valid regex, error received: %v", ignoreString, err))
+			}
+			ignorePatterns = append(ignorePatterns, ignorePattern)
+		}
+
+		s.ignorePatterns = ignorePatterns
+	}
+}
 
 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
@@ -38,6 +66,9 @@ func (s Scanner) Keywords() []string {
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
 	matches := pattern.FindAllStringSubmatch(string(data), -1)
 	for _, match := range matches {
+		if shouldIgnore(match[1], s.ignorePatterns) {
+			continue
+		}
 		paramsUnsafe, err := msdsn.Parse(match[1])
 		if err != nil {
 			continue
@@ -80,6 +111,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 	return results, nil
 }
 
+func shouldIgnore(uri string, ignorePatterns []*regexp.Regexp) bool {
+	for _, ignore := range ignorePatterns {
+		if ignore.MatchString(uri) {
+			return true
+		}
+	}
+	return false
+}
+
 var ping = func(ctx context.Context, config msdsn.Config) (bool, error) {
 	// TCP connectivity check to prevent indefinite hangs
 	address := net.JoinHostPort(config.Host, strconv.Itoa(int(config.Port)))

pkg/detectors/sqlserver/sqlserver_test.go

@@ -1,7 +1,13 @@
 package sqlserver
 
 import (
+	"context"
+	"fmt"
 	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
 )
 
 func TestSQLServer_Pattern(t *testing.T) {
@@ -18,3 +24,87 @@ func TestSQLServer_Pattern(t *testing.T) {
 		t.Errorf("SQLServer.pattern: did not find connection string in xml format")
 	}
 }
+
+const (
+	validConnStr   = `builder.Services.AddDbContext<Database>(optionsBuilder => optionsBuilder.UseSqlServer("Server=localhost;Initial Catalog=master;User ID=sa;Password=P@ssw0rd!;Persist Security Info=true;MultipleActiveResultSets=true;"));`
+	validParsedStr = `sqlserver://sa:P%40ssw0rd%21@localhost?database=master&dial+timeout=15&disableretry=false`
+	invalidConnStr = `some random text without connection string`
+)
+
+func TestSqlServer_Pattern(t *testing.T) {
+	d := Scanner{}
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
+	tests := []struct {
+		name  string
+		input string
+		want  []string
+	}{
+		{
+			name:  "valid pattern - with keyword sql",
+			input: fmt.Sprintf(`sql - %s`, validConnStr),
+			want:  []string{validParsedStr},
+		},
+		{
+			name:  "invalid pattern",
+			input: fmt.Sprintf("sql=%s", invalidConnStr),
+			want:  []string{},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+			if len(matchedDetectors) == 0 {
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
+				return
+			}
+
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
+			if err != nil {
+				t.Errorf("error = %v", err)
+				return
+			}
+
+			if len(results) != len(test.want) {
+				if len(results) == 0 {
+					t.Errorf("did not receive result")
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+			}
+		})
+	}
+}
+
+func TestSqlServer_FromDataWithIgnorePattern(t *testing.T) {
+	s := New(
+		WithIgnorePattern([]string{
+			`^Server=localhost`,
+		}))
+	got, err := s.FromData(context.Background(), false, []byte("Server=localhost;Initial Catalog=master;User ID=sa;Password=P@ssw0rd!;Persist Security Info=true;MultipleActiveResultSets=true"))
+	if err != nil {
+		t.Errorf("FromData() error = %v", err)
+		return
+	}
+	if len(got) != 0 {
+		t.Errorf("expected no results, but got %d", len(got))
+	}
+}