Skip to content

Commit 254a96e

Browse files
knqyf263VaismanLior
knqyf263
and
VaismanLior
authored
feat: add secret scanning (fanal#431)
Co-authored-by: VaismanLior <[email protected]>
1 parent dff5dce commit 254a96e

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

51 files changed

+2115
-140
lines changed

‎analyzer/all/import.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,4 +31,5 @@ import (
3131
_ "github.com/aquasecurity/fanal/analyzer/pkg/dpkg"
3232
_ "github.com/aquasecurity/fanal/analyzer/pkg/rpm"
3333
_ "github.com/aquasecurity/fanal/analyzer/repo/apk"
34+
_ "github.com/aquasecurity/fanal/analyzer/secret"
3435
)

‎analyzer/analyzer.go‎

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"strings"
1010
"sync"
1111

12+
"golang.org/x/exp/slices"
1213
"golang.org/x/sync/semaphore"
1314
"golang.org/x/xerrors"
1415

@@ -84,6 +85,7 @@ type AnalysisResult struct {
8485
PackageInfos []types.PackageInfo
8586
Applications []types.Application
8687
Configs []types.Config
88+
Secrets []types.Secret
8789
SystemInstalledFiles []string // A list of files installed by OS package manager
8890

8991
// For Red Hat
@@ -96,7 +98,8 @@ type AnalysisResult struct {
9698

9799
func (r *AnalysisResult) isEmpty() bool {
98100
return r.OS == nil && r.Repository == nil && len(r.PackageInfos) == 0 && len(r.Applications) == 0 &&
99-
len(r.Configs) == 0 && len(r.SystemInstalledFiles) == 0 && r.BuildInfo == nil && len(r.CustomResources) == 0
101+
len(r.Configs) == 0 && len(r.Secrets) == 0 && len(r.SystemInstalledFiles) == 0 && r.BuildInfo == nil &&
102+
len(r.CustomResources) == 0
100103
}
101104

102105
func (r *AnalysisResult) Sort() {
@@ -122,6 +125,19 @@ func (r *AnalysisResult) Sort() {
122125
return app.Libraries[i].Version < app.Libraries[j].Version
123126
})
124127
}
128+
129+
// Secrets
130+
sort.Slice(r.Secrets, func(i, j int) bool {
131+
return r.Secrets[i].FilePath < r.Secrets[j].FilePath
132+
})
133+
for _, sec := range r.Secrets {
134+
sort.Slice(sec.Findings, func(i, j int) bool {
135+
if sec.Findings[i].RuleID != sec.Findings[j].RuleID {
136+
return sec.Findings[i].RuleID < sec.Findings[j].RuleID
137+
}
138+
return sec.Findings[i].StartLine < sec.Findings[j].StartLine
139+
})
140+
}
125141
}
126142

127143
func (r *AnalysisResult) Merge(new *AnalysisResult) {
@@ -155,7 +171,7 @@ func (r *AnalysisResult) Merge(new *AnalysisResult) {
155171
}
156172

157173
r.Configs = append(r.Configs, new.Configs...)
158-
174+
r.Secrets = append(r.Secrets, new.Secrets...)
159175
r.SystemInstalledFiles = append(r.SystemInstalledFiles, new.SystemInstalledFiles...)
160176

161177
if new.BuildInfo != nil {
@@ -233,11 +249,16 @@ func (ag AnalyzerGroup) ImageConfigAnalyzerVersions() map[string]int {
233249
}
234250

235251
func (ag AnalyzerGroup) AnalyzeFile(ctx context.Context, wg *sync.WaitGroup, limit *semaphore.Weighted, result *AnalysisResult,
236-
dir, filePath string, info os.FileInfo, opener Opener, opts AnalysisOptions) error {
252+
dir, filePath string, info os.FileInfo, opener Opener, disabled []Type, opts AnalysisOptions) error {
237253
if info.IsDir() {
238254
return nil
239255
}
240256
for _, a := range ag.analyzers {
257+
// Skip disabled analyzers
258+
if slices.Contains(disabled, a.Type()) {
259+
continue
260+
}
261+
241262
// filepath extracted from tar file doesn't have the prefix "/"
242263
if !a.Required(strings.TrimLeft(filePath, "/"), info) {
243264
continue

‎analyzer/analyzer_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -418,7 +418,7 @@ func TestAnalyzeFile(t *testing.T) {
418418
}
419419
return os.Open(tt.args.testFilePath)
420420
},
421-
analyzer.AnalysisOptions{},
421+
nil, analyzer.AnalysisOptions{},
422422
)
423423

424424
wg.Wait()

‎analyzer/const.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,11 @@ const (
8080
TypeTerraform Type = "terraform"
8181
TypeCloudFormation Type = "cloudFormation"
8282

83+
// ========
84+
// Secrets
85+
// ========
86+
TypeSecret Type = "secret"
87+
8388
// =======
8489
// Red Hat
8590
// =======

‎analyzer/secret/secret.go‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
package secret
2+
3+
import (
4+
"context"
5+
"io"
6+
"math"
7+
"os"
8+
"path/filepath"
9+
"strings"
10+
11+
"golang.org/x/exp/slices"
12+
"golang.org/x/xerrors"
13+
14+
"github.com/aquasecurity/fanal/analyzer"
15+
"github.com/aquasecurity/fanal/secret"
16+
"github.com/aquasecurity/fanal/types"
17+
dio "github.com/aquasecurity/go-dep-parser/pkg/io"
18+
)
19+
20+
const version = 1
21+
22+
var (
23+
skipFiles = []string{
24+
"go.mod",
25+
"go.sum",
26+
"package-lock.json",
27+
"yarn.lock",
28+
"Pipfile.lock",
29+
"Gemfile.lock",
30+
}
31+
skipDirs = []string{".git", "node_modules"}
32+
skipExts = []string{".jpg", ".png", ".gif", ".doc", ".pdf", ".bin", ".svg", ".socket", ".deb", ".rpm",
33+
".zip", ".gz", ".gzip", ".tar", ".pyc"}
34+
)
35+
36+
type ScannerOption struct {
37+
ConfigPath string
38+
}
39+
40+
// SecretAnalyzer is an analyzer for secrets
41+
type SecretAnalyzer struct {
42+
scanner secret.Scanner
43+
}
44+
45+
func RegisterSecretAnalyzer(opt ScannerOption) error {
46+
a, err := newSecretAnalyzer(opt.ConfigPath)
47+
if err != nil {
48+
return xerrors.Errorf("secret scanner init error: %w", err)
49+
}
50+
analyzer.RegisterAnalyzer(a)
51+
return nil
52+
}
53+
54+
func newSecretAnalyzer(configPath string) (SecretAnalyzer, error) {
55+
s, err := secret.NewScanner(configPath)
56+
if err != nil {
57+
return SecretAnalyzer{}, xerrors.Errorf("secret scanner error: %w", err)
58+
}
59+
return SecretAnalyzer{scanner: s}, nil
60+
}
61+
62+
func (a SecretAnalyzer) Analyze(_ context.Context, input analyzer.AnalysisInput) (*analyzer.AnalysisResult, error) {
63+
// Do not scan binaries
64+
binary, err := isBinary(input.Content, input.Info.Size())
65+
if binary || err != nil {
66+
return nil, nil
67+
}
68+
69+
content, err := io.ReadAll(input.Content)
70+
if err != nil {
71+
return nil, xerrors.Errorf("read error %s: %w", input.FilePath, err)
72+
}
73+
74+
result := a.scanner.Scan(secret.ScanArgs{
75+
FilePath: input.FilePath,
76+
Content: content,
77+
})
78+
79+
if len(result.Findings) == 0 {
80+
return nil, nil
81+
}
82+
83+
return &analyzer.AnalysisResult{
84+
Secrets: []types.Secret{result},
85+
}, nil
86+
}
87+
88+
func isBinary(content dio.ReadSeekerAt, fileSize int64) (bool, error) {
89+
headSize := int(math.Min(float64(fileSize), 300))
90+
head := make([]byte, headSize)
91+
if _, err := content.Read(head); err != nil {
92+
return false, err
93+
}
94+
if _, err := content.Seek(0, io.SeekStart); err != nil {
95+
return false, err
96+
}
97+
98+
// cf. https://github.com/file/file/blob/f2a6e7cb7db9b5fd86100403df6b2f830c7f22ba/src/encoding.c#L151-L228
99+
for _, b := range head {
100+
if b < 7 || b == 11 || (13 < b && b < 27) || (27 < b && b < 0x20) || b == 0x7f {
101+
return true, nil
102+
}
103+
}
104+
105+
return false, nil
106+
}
107+
108+
func (a SecretAnalyzer) Required(filePath string, fi os.FileInfo) bool {
109+
// Skip small files
110+
if fi.Size() < 10 {
111+
return false
112+
}
113+
114+
dir, fileName := filepath.Split(filePath)
115+
dir = filepath.ToSlash(dir)
116+
dirs := strings.Split(dir, "/")
117+
118+
// Check if the directory should be skipped
119+
for _, skipDir := range skipDirs {
120+
if slices.Contains(dirs, skipDir) {
121+
return false
122+
}
123+
}
124+
125+
// Check if the file should be skipped
126+
if slices.Contains(skipFiles, fileName) {
127+
return false
128+
}
129+
130+
// Check if the file extension should be skipped
131+
ext := filepath.Ext(fileName)
132+
if slices.Contains(skipExts, ext) {
133+
return false
134+
}
135+
136+
if a.scanner.AllowPath(filePath) {
137+
return false
138+
}
139+
140+
return true
141+
}
142+
143+
func (a SecretAnalyzer) Type() analyzer.Type {
144+
return analyzer.TypeSecret
145+
}
146+
147+
func (a SecretAnalyzer) Version() int {
148+
return version
149+
}

‎analyzer/secret/secret_test.go‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
package secret
2+
3+
import (
4+
"context"
5+
"os"
6+
"testing"
7+
8+
"github.com/stretchr/testify/assert"
9+
"github.com/stretchr/testify/require"
10+
11+
"github.com/aquasecurity/fanal/analyzer"
12+
"github.com/aquasecurity/fanal/types"
13+
)
14+
15+
func TestSecretAnalyzer(t *testing.T) {
16+
wantFinding1 := types.SecretFinding{
17+
RuleID: "rule1",
18+
Category: "general",
19+
Title: "Generic Rule",
20+
Severity: "HIGH",
21+
StartLine: 2,
22+
EndLine: 3,
23+
Match: "generic secret line secret=\"*****\"",
24+
}
25+
wantFinding2 := types.SecretFinding{
26+
RuleID: "rule1",
27+
Category: "general",
28+
Title: "Generic Rule",
29+
Severity: "HIGH",
30+
StartLine: 4,
31+
EndLine: 5,
32+
Match: "secret=\"*****\"",
33+
}
34+
tests := []struct {
35+
name string
36+
configPath string
37+
filePath string
38+
want *analyzer.AnalysisResult
39+
}{
40+
{
41+
name: "return results",
42+
configPath: "testdata/config.yaml",
43+
filePath: "testdata/secret.txt",
44+
want: &analyzer.AnalysisResult{
45+
Secrets: []types.Secret{{
46+
FilePath: "testdata/secret.txt",
47+
Findings: []types.SecretFinding{wantFinding1, wantFinding2},
48+
},
49+
},
50+
},
51+
},
52+
{
53+
name: "return nil when no results",
54+
configPath: "",
55+
filePath: "testdata/secret.txt",
56+
want: nil,
57+
},
58+
{
59+
name: "skip binary file",
60+
configPath: "",
61+
filePath: "testdata/binaryfile",
62+
want: nil,
63+
},
64+
}
65+
66+
for _, tt := range tests {
67+
t.Run(tt.name, func(t *testing.T) {
68+
a, err := newSecretAnalyzer(tt.configPath)
69+
require.NoError(t, err)
70+
content, err := os.Open(tt.filePath)
71+
require.NoError(t, err)
72+
fi, err := content.Stat()
73+
require.NoError(t, err)
74+
75+
got, err := a.Analyze(context.TODO(), analyzer.AnalysisInput{
76+
FilePath: tt.filePath,
77+
Content: content,
78+
Info: fi,
79+
})
80+
81+
require.NoError(t, err)
82+
assert.Equal(t, tt.want, got)
83+
})
84+
}
85+
}
86+
87+
func TestSecretRequire(t *testing.T) {
88+
tests := []struct {
89+
name string
90+
filePath string
91+
want bool
92+
}{
93+
{
94+
name: "pass regular file",
95+
filePath: "testdata/secret.txt",
96+
want: true,
97+
},
98+
{
99+
name: "skip small file",
100+
filePath: "testdata/emptyfile",
101+
want: false,
102+
},
103+
{
104+
name: "skip folder",
105+
filePath: "testdata/node_modules/secret.txt",
106+
want: false,
107+
},
108+
{
109+
name: "skip file",
110+
filePath: "testdata/package-lock.json",
111+
want: false,
112+
},
113+
{
114+
name: "skip extension",
115+
filePath: "testdata/secret.doc",
116+
want: false,
117+
},
118+
}
119+
120+
for _, tt := range tests {
121+
t.Run(tt.name, func(t *testing.T) {
122+
a, err := newSecretAnalyzer("")
123+
require.NoError(t, err)
124+
125+
fi, err := os.Stat(tt.filePath)
126+
require.NoError(t, err)
127+
128+
got := a.Required(tt.filePath, fi)
129+
assert.Equal(t, tt.want, got)
130+
})
131+
}
132+
}

‎analyzer/secret/testdata/binaryfile‎

10 Bytes
Binary file not shown.

‎analyzer/secret/testdata/config.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
rules:
2+
- id: rule1
3+
category: general
4+
title: Generic Rule
5+
severity: HIGH
6+
regex: (?i)(?P<key>(secret))(=|:).{0,5}['"](?P<secret>[0-9a-zA-Z\-_=]{8,64})['"]
7+
secret-group-name: secret
8+
9+

‎analyzer/secret/testdata/emptyfile‎

Whitespace-only changes.

‎analyzer/secret/testdata/node_modules/secret.txt‎

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)