Skip to content

Commit b0937b6

Browse files
simar7knqyf263
simar7
and
knqyf263
authored
Add layer id info (merge to master) (fanal#88)
* analyzer: Include layerID as part of LayerInfo Signed-off-by: Simarpreet Singh <[email protected]> * Add LayerID to Package struct Signed-off-by: Simarpreet Singh <[email protected]> * analyzer: Remove ID from returned layerInfo Signed-off-by: Simarpreet Singh <[email protected]> * analyzer: Handle missing layer.ID from cached layer Signed-off-by: Simarpreet Singh <[email protected]> * extractor/docker: Cleanup logic to avoid extra slice usage Signed-off-by: Simarpreet Singh <[email protected]> * integration: Fix golden files to include LayerID Signed-off-by: Simarpreet Singh <[email protected]> * analyzer: Remove condition for adding layer.ID Signed-off-by: Simarpreet Singh <[email protected]> * types: Introduce types.LibraryInfo Signed-off-by: Simarpreet Singh <[email protected]> * docker: Add LayerID to each LibraryInfo Signed-off-by: Simarpreet Singh <[email protected]> * .github/bench: Bump up docker version Signed-off-by: Simarpreet Singh <[email protected]> * intergration/perf: Remove other OSes for the timebeing. Looks like Github CI is running out of space while running other tests. Until we find a better solution we need to comment out bigger OSes. Signed-off-by: Simarpreet Singh <[email protected]> * fix(image): call Close() via cleanup funcion * refactor(type): add omitempty * analyzer: Change to types.LibraryInfo in analzyer.go Signed-off-by: Simarpreet Singh <[email protected]> * wip: add CleanupDockerExtractorFn for cleanup Signed-off-by: Simarpreet Singh <[email protected]> * refactor(analyzer): remove un-needed function * test(cache): comment in * Revert "wip: add CleanupDockerExtractorFn for cleanup" This reverts commit dabfae104bf6d63492823c6c3eb94175d26eabad. * Revert ".github/bench: Bump up docker version" This reverts commit b982c46861e1cc0851d53621c0e68ac40918d755. * refactor(analyzer): sort imports * test(cache): remove debug code * test(cache): format * chore(image): remove debug code Co-authored-by: Teppei Fukuda <[email protected]>
1 parent bfa6e76 commit b0937b6

23 files changed

+2251
-1140
lines changed

‎analyzer/analyzer.go

Lines changed: 12 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -5,16 +5,14 @@ import (
55
"encoding/json"
66
"sort"
77

8-
"github.com/aquasecurity/fanal/extractor/docker"
98
"github.com/containers/image/v5/manifest"
109
digest "github.com/opencontainers/go-digest"
11-
12-
"github.com/aquasecurity/fanal/cache"
13-
"github.com/aquasecurity/fanal/types"
14-
1510
"golang.org/x/xerrors"
1611

12+
"github.com/aquasecurity/fanal/cache"
1713
"github.com/aquasecurity/fanal/extractor"
14+
"github.com/aquasecurity/fanal/extractor/docker"
15+
"github.com/aquasecurity/fanal/types"
1816
godeptypes "github.com/aquasecurity/go-dep-parser/pkg/types"
1917
)
2018

@@ -234,6 +232,7 @@ func (a Applier) ApplyLayers(imageID digest.Digest, layerIDs []string) (types.Im
234232
if layer.SchemaVersion == 0 {
235233
return types.ImageDetail{}, xerrors.Errorf("layer cache missing: %s", layerID)
236234
}
235+
layer.ID = digest.Digest(layerID)
237236
layers = append(layers, layer)
238237
}
239238

@@ -312,28 +311,20 @@ func GetLibraries(filesMap extractor.FileMap) ([]types.Application, error) {
312311
return nil, xerrors.Errorf("failed to get libraries: %w", err)
313312
}
314313

314+
var lis []types.LibraryInfo
315315
for filePath, libs := range libMap {
316+
for _, lib := range libs {
317+
lis = append(lis, types.LibraryInfo{
318+
Library: lib,
319+
})
320+
}
321+
316322
results = append(results, types.Application{
317323
Type: analyzer.Name(),
318324
FilePath: string(filePath),
319-
Libraries: libs,
325+
Libraries: lis,
320326
})
321327
}
322328
}
323329
return results, nil
324330
}
325-
326-
func mergePkgs(pkgs, pkgsFromCommands []types.Package) []types.Package {
327-
// pkg has priority over pkgsFromCommands
328-
uniqPkgs := map[string]struct{}{}
329-
for _, pkg := range pkgs {
330-
uniqPkgs[pkg.Name] = struct{}{}
331-
}
332-
for _, pkg := range pkgsFromCommands {
333-
if _, ok := uniqPkgs[pkg.Name]; ok {
334-
continue
335-
}
336-
pkgs = append(pkgs, pkg)
337-
}
338-
return pkgs
339-
}

‎analyzer/analyzer_test.go

Lines changed: 65 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -139,8 +139,25 @@ func TestConfig_Analyze(t *testing.T) {
139139
DecompressedLayerID: "sha256:24df0d4e20c0f42d3703bf1f1db2bdd77346c7956f74f423603d651e8e5ae8a7",
140140
LayerInfo: types.LayerInfo{
141141
SchemaVersion: 1,
142-
Applications: []types.Application{{Type: "composer", FilePath: "php-app/composer.lock", Libraries: []depTypes.Library{{Name: "guzzlehttp/guzzle", Version: "6.2.0"}, {Name: "guzzlehttp/promises", Version: "v1.3.1"}, {Name: "guzzlehttp/psr7", Version: "1.5.2"}, {Name: "laravel/installer", Version: "v2.0.1"}, {Name: "pear/log", Version: "1.13.1"}, {Name: "pear/pear_exception", Version: "v1.0.0"}, {Name: "psr/http-message", Version: "1.0.1"}, {Name: "ralouphie/getallheaders", Version: "2.0.5"}, {Name: "symfony/console", Version: "v4.2.7"}, {Name: "symfony/contracts", Version: "v1.0.2"}, {Name: "symfony/filesystem", Version: "v4.2.7"}, {Name: "symfony/polyfill-ctype", Version: "v1.11.0"}, {Name: "symfony/polyfill-mbstring", Version: "v1.11.0"}, {Name: "symfony/process", Version: "v4.2.7"}}}},
143-
OpaqueDirs: []string{"php-app/"},
142+
Applications: []types.Application{{Type: "composer", FilePath: "php-app/composer.lock",
143+
Libraries: []types.LibraryInfo{
144+
{Library: depTypes.Library{Name: "guzzlehttp/guzzle", Version: "6.2.0"}},
145+
{Library: depTypes.Library{Name: "guzzlehttp/promises", Version: "v1.3.1"}},
146+
{Library: depTypes.Library{Name: "guzzlehttp/psr7", Version: "1.5.2"}},
147+
{Library: depTypes.Library{Name: "laravel/installer", Version: "v2.0.1"}},
148+
{Library: depTypes.Library{Name: "pear/log", Version: "1.13.1"}},
149+
{Library: depTypes.Library{Name: "pear/pear_exception", Version: "v1.0.0"}},
150+
{Library: depTypes.Library{Name: "psr/http-message", Version: "1.0.1"}},
151+
{Library: depTypes.Library{Name: "ralouphie/getallheaders", Version: "2.0.5"}},
152+
{Library: depTypes.Library{Name: "symfony/console", Version: "v4.2.7"}},
153+
{Library: depTypes.Library{Name: "symfony/contracts", Version: "v1.0.2"}},
154+
{Library: depTypes.Library{Name: "symfony/filesystem", Version: "v4.2.7"}},
155+
{Library: depTypes.Library{Name: "symfony/polyfill-ctype", Version: "v1.11.0"}},
156+
{Library: depTypes.Library{Name: "symfony/polyfill-mbstring", Version: "v1.11.0"}},
157+
{Library: depTypes.Library{Name: "symfony/process", Version: "v4.2.7"}},
158+
},
159+
}},
160+
OpaqueDirs: []string{"php-app/"},
144161
},
145162
},
146163
},
@@ -209,8 +226,9 @@ func TestConfig_Analyze(t *testing.T) {
209226
mockCache.ApplyPutLayerExpectations(tt.putLayerExpectations)
210227
mockCache.ApplyPutImageExpectations(tt.putImageExpectations)
211228

212-
d, err := docker.NewDockerArchiveExtractor(context.Background(), tt.imagePath, types.DockerOption{})
229+
d, cleanup, err := docker.NewDockerArchiveExtractor(context.Background(), tt.imagePath, types.DockerOption{})
213230
require.NoError(t, err, tt.name)
231+
defer cleanup()
214232

215233
ac := analyzer.New(d, mockCache)
216234
got, err := ac.Analyze(context.Background())
@@ -265,6 +283,7 @@ func TestApplier_ApplyLayers(t *testing.T) {
265283
FilePath: "var/lib/dpkg/status.d/tzdata",
266284
Packages: []types.Package{
267285
{
286+
LayerID: "sha256:932da51564135c98a49a34a193d6cd363d8fa4184d957fde16c9d8527b3f3b02",
268287
Name: "tzdata",
269288
Version: "2019a-0+deb9u1",
270289
SrcName: "tzdata",
@@ -313,14 +332,18 @@ func TestApplier_ApplyLayers(t *testing.T) {
313332
{
314333
Type: "composer",
315334
FilePath: "php-app/composer.lock",
316-
Libraries: []depTypes.Library{
335+
Libraries: []types.LibraryInfo{
317336
{
318-
Name: "guzzlehttp/guzzle",
319-
Version: "6.2.0",
337+
Library: depTypes.Library{
338+
Name: "guzzlehttp/guzzle",
339+
Version: "6.2.0",
340+
},
320341
},
321342
{
322-
Name: "symfony/process",
323-
Version: "v4.2.7",
343+
Library: depTypes.Library{
344+
Name: "symfony/process",
345+
Version: "v4.2.7",
346+
},
324347
},
325348
},
326349
},
@@ -347,14 +370,33 @@ func TestApplier_ApplyLayers(t *testing.T) {
347370
Name: "9.9",
348371
},
349372
Packages: []types.Package{
350-
{Name: "libc6", Version: "2.24-11+deb9u4", SrcName: "glibc", SrcVersion: "2.24-11+deb9u4"},
351-
{Name: "tzdata", Version: "2019a-0+deb9u1", SrcName: "tzdata", SrcVersion: "2019a-0+deb9u1"},
373+
{
374+
Name: "libc6", Version: "2.24-11+deb9u4", SrcName: "glibc", SrcVersion: "2.24-11+deb9u4",
375+
LayerID: "sha256:dffd9992ca398466a663c87c92cfea2a2db0ae0cf33fcb99da60eec52addbfc5",
376+
},
377+
{
378+
Name: "tzdata", Version: "2019a-0+deb9u1", SrcName: "tzdata", SrcVersion: "2019a-0+deb9u1",
379+
LayerID: "sha256:932da51564135c98a49a34a193d6cd363d8fa4184d957fde16c9d8527b3f3b02",
380+
},
352381
},
353382
Applications: []types.Application{
354383
{
355384
Type: "composer", FilePath: "php-app/composer.lock",
356-
Libraries: []depTypes.Library{{Name: "guzzlehttp/guzzle", Version: "6.2.0"},
357-
{Name: "symfony/process", Version: "v4.2.7"},
385+
Libraries: []types.LibraryInfo{
386+
{
387+
Library: depTypes.Library{
388+
Name: "guzzlehttp/guzzle",
389+
Version: "6.2.0",
390+
},
391+
LayerID: "sha256:24df0d4e20c0f42d3703bf1f1db2bdd77346c7956f74f423603d651e8e5ae8a7",
392+
},
393+
{
394+
Library: depTypes.Library{
395+
Name: "symfony/process",
396+
Version: "v4.2.7",
397+
},
398+
LayerID: "sha256:24df0d4e20c0f42d3703bf1f1db2bdd77346c7956f74f423603d651e8e5ae8a7",
399+
},
358400
},
359401
},
360402
},
@@ -384,11 +426,11 @@ func TestApplier_ApplyLayers(t *testing.T) {
384426
{
385427
FilePath: "lib/apk/db/installed",
386428
Packages: []types.Package{
387-
{Name: "musl", Version: "1.1.22-r3"},
388-
{Name: "busybox", Version: "1.30.1-r3"},
389-
{Name: "openssl", Version: "1.1.1d-r2"},
390-
{Name: "libcrypto1.1", Version: "1.1.1d-r2"},
391-
{Name: "libssl1.1", Version: "1.1.1d-r2"},
429+
{Name: "musl", Version: "1.1.22-r3", LayerID: "sha256:531743b7098cb2aaf615641007a129173f63ed86ca32fe7b5a246a1c47286028"},
430+
{Name: "busybox", Version: "1.30.1-r3", LayerID: "sha256:531743b7098cb2aaf615641007a129173f63ed86ca32fe7b5a246a1c47286028"},
431+
{Name: "openssl", Version: "1.1.1d-r2", LayerID: "sha256:531743b7098cb2aaf615641007a129173f63ed86ca32fe7b5a246a1c47286028"},
432+
{Name: "libcrypto1.1", Version: "1.1.1d-r2", LayerID: "sha256:531743b7098cb2aaf615641007a129173f63ed86ca32fe7b5a246a1c47286028"},
433+
{Name: "libssl1.1", Version: "1.1.1d-r2", LayerID: "sha256:531743b7098cb2aaf615641007a129173f63ed86ca32fe7b5a246a1c47286028"},
392434
},
393435
},
394436
},
@@ -424,11 +466,11 @@ func TestApplier_ApplyLayers(t *testing.T) {
424466
Name: "3.10.4",
425467
},
426468
Packages: []types.Package{
427-
{Name: "busybox", Version: "1.30.1-r3"},
428-
{Name: "libcrypto1.1", Version: "1.1.1d-r2"},
429-
{Name: "libssl1.1", Version: "1.1.1d-r2"},
430-
{Name: "musl", Version: "1.1.22-r3"},
431-
{Name: "openssl", Version: "1.1.1d-r2"},
469+
{Name: "busybox", Version: "1.30.1-r3", LayerID: "sha256:531743b7098cb2aaf615641007a129173f63ed86ca32fe7b5a246a1c47286028"},
470+
{Name: "libcrypto1.1", Version: "1.1.1d-r2", LayerID: "sha256:531743b7098cb2aaf615641007a129173f63ed86ca32fe7b5a246a1c47286028"},
471+
{Name: "libssl1.1", Version: "1.1.1d-r2", LayerID: "sha256:531743b7098cb2aaf615641007a129173f63ed86ca32fe7b5a246a1c47286028"},
472+
{Name: "musl", Version: "1.1.22-r3", LayerID: "sha256:531743b7098cb2aaf615641007a129173f63ed86ca32fe7b5a246a1c47286028"},
473+
{Name: "openssl", Version: "1.1.1d-r2", LayerID: "sha256:531743b7098cb2aaf615641007a129173f63ed86ca32fe7b5a246a1c47286028"},
432474
},
433475
HistoryPackages: []types.Package{
434476
{Name: "musl", Version: "1.1.23"},
@@ -544,7 +586,7 @@ func TestApplier_ApplyLayers(t *testing.T) {
544586
})
545587
for _, app := range got.Applications {
546588
sort.Slice(app.Libraries, func(i, j int) bool {
547-
return app.Libraries[i].Name < app.Libraries[j].Name
589+
return app.Libraries[i].Library.Name < app.Libraries[j].Library.Name
548590
})
549591
}
550592
assert.Equal(t, tt.want, got)

‎cache/cache_test.go

Lines changed: 34 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -114,14 +114,18 @@ func TestFSCache_GetLayer(t *testing.T) {
114114
{
115115
Type: "composer",
116116
FilePath: "php-app/composer.lock",
117-
Libraries: []depTypes.Library{
117+
Libraries: []types.LibraryInfo{
118118
{
119-
Name: "guzzlehttp/guzzle",
120-
Version: "6.2.0",
119+
Library: depTypes.Library{
120+
Name: "guzzlehttp/guzzle",
121+
Version: "6.2.0",
122+
},
121123
},
122124
{
123-
Name: "guzzlehttp/promises",
124-
Version: "v1.3.1",
125+
Library: depTypes.Library{
126+
Name: "guzzlehttp/promises",
127+
Version: "v1.3.1",
128+
},
125129
},
126130
},
127131
},
@@ -215,9 +219,19 @@ func TestFSCache_PutLayer(t *testing.T) {
215219
{
216220
Type: "composer",
217221
FilePath: "php-app/composer.lock",
218-
Libraries: []depTypes.Library{
219-
{Name: "guzzlehttp/guzzle", Version: "6.2.0"},
220-
{Name: "guzzlehttp/promises", Version: "v1.3.1"},
222+
Libraries: []types.LibraryInfo{
223+
{
224+
Library: depTypes.Library{
225+
Name: "guzzlehttp/guzzle",
226+
Version: "6.2.0",
227+
},
228+
},
229+
{
230+
Library: depTypes.Library{
231+
Name: "guzzlehttp/promises",
232+
Version: "v1.3.1",
233+
},
234+
},
221235
},
222236
},
223237
},
@@ -248,14 +262,18 @@ func TestFSCache_PutLayer(t *testing.T) {
248262
"Type": "composer",
249263
"FilePath": "php-app/composer.lock",
250264
"Libraries": [
251-
{
252-
"Name": "guzzlehttp/guzzle",
253-
"Version": "6.2.0"
254-
},
255-
{
256-
"Name": "guzzlehttp/promises",
257-
"Version": "v1.3.1"
258-
}
265+
{
266+
"Library":{
267+
"Name":"guzzlehttp/guzzle",
268+
"Version":"6.2.0"
269+
}
270+
},
271+
{
272+
"Library":{
273+
"Name":"guzzlehttp/promises",
274+
"Version":"v1.3.1"
275+
}
276+
}
259277
]
260278
}
261279
],

‎cache/testdata/decompressed-layer-id.db

0 Bytes
Binary file not shown.

‎cmd/fanal/main.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,17 +68,19 @@ func run() (err error) {
6868
}
6969

7070
var ext extractor.Extractor
71+
var cleanup func()
7172
if len(args) > 0 {
72-
ext, err = docker.NewDockerExtractor(ctx, args[0], opt)
73+
ext, cleanup, err = docker.NewDockerExtractor(ctx, args[0], opt)
7374
if err != nil {
7475
return err
7576
}
7677
} else {
77-
ext, err = docker.NewDockerArchiveExtractor(ctx, *tarPath, opt)
78+
ext, cleanup, err = docker.NewDockerArchiveExtractor(ctx, *tarPath, opt)
7879
if err != nil {
7980
return err
8081
}
8182
}
83+
defer cleanup()
8284

8385
ac := analyzer.New(ext, c)
8486
imageInfo, err := ac.Analyze(ctx)

‎extractor/docker/docker.go

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -52,32 +52,36 @@ func init() {
5252
image.RegisterRegistry(&ecr.ECR{})
5353
}
5454

55-
func NewDockerExtractor(ctx context.Context, imageName string, option types.DockerOption) (Extractor, error) {
55+
func NewDockerExtractor(ctx context.Context, imageName string, option types.DockerOption) (Extractor, func(), error) {
5656
ref := image.Reference{Name: imageName, IsFile: false}
5757
transports := []string{"docker-daemon:", "docker://"}
5858
return newDockerExtractor(ctx, ref, transports, option)
5959
}
6060

61-
func NewDockerArchiveExtractor(ctx context.Context, fileName string, option types.DockerOption) (Extractor, error) {
61+
func NewDockerArchiveExtractor(ctx context.Context, fileName string, option types.DockerOption) (Extractor, func(), error) {
6262
ref := image.Reference{Name: fileName, IsFile: true}
6363
transports := []string{"docker-archive:"}
6464
return newDockerExtractor(ctx, ref, transports, option)
6565
}
6666

6767
func newDockerExtractor(ctx context.Context, imgRef image.Reference, transports []string,
68-
option types.DockerOption) (Extractor, error) {
68+
option types.DockerOption) (Extractor, func(), error) {
6969
ctx, cancel := context.WithTimeout(ctx, option.Timeout)
7070
defer cancel()
7171

7272
img, err := image.NewImage(ctx, imgRef, transports, option)
7373
if err != nil {
74-
return Extractor{}, xerrors.Errorf("unable to initialize a image struct: %w", err)
74+
return Extractor{}, nil, xerrors.Errorf("unable to initialize a image struct: %w", err)
75+
}
76+
77+
cleanup := func() {
78+
_ = img.Close()
7579
}
7680

7781
return Extractor{
7882
option: option,
7983
image: img,
80-
}, nil
84+
}, cleanup, nil
8185
}
8286

8387
func ApplyLayers(layers []types.LayerInfo) types.ImageDetail {
@@ -98,9 +102,15 @@ func ApplyLayers(layers []types.LayerInfo) types.ImageDetail {
98102
}
99103

100104
for _, pkgInfo := range layer.PackageInfos {
105+
for i := range pkgInfo.Packages {
106+
pkgInfo.Packages[i].LayerID = layer.ID
107+
}
101108
nestedMap.SetByString(pkgInfo.FilePath, sep, pkgInfo)
102109
}
103110
for _, app := range layer.Applications {
111+
for i := range app.Libraries {
112+
app.Libraries[i].LayerID = layer.ID
113+
}
104114
nestedMap.SetByString(app.FilePath, sep, app)
105115
}
106116
}

0 commit comments

Comments
 (0)