Skip to content

Commit c63e3aa

Browse files
simar7simar7
authored
integration: Add dockerless mode tests (fanal#81)
* integration: Add dockerless mode tests Signed-off-by: Simarpreet Singh <[email protected]> * .github: Fix typo Signed-off-by: Simarpreet Singh <[email protected]> * library_test: Improve cache assertions with containers/image Signed-off-by: Simarpreet Singh <[email protected]> * github: Update docker version before running Signed-off-by: Simarpreet Singh <[email protected]> * integration: Remove un-needed random num generation Signed-off-by: Simarpreet Singh <[email protected]> * rebase on latest master Signed-off-by: Simarpreet Singh <[email protected]> * integration: improve packages check Signed-off-by: Simarpreet Singh <[email protected]> * integration: Remove un-needed goldenfiles Signed-off-by: Simarpreet Singh <[email protected]> * library_test: Rename dockerlessImageName to remoteImageName Signed-off-by: Simarpreet Singh <[email protected]> * github: Remove un-needed bench target Signed-off-by: Simarpreet Singh <[email protected]> * library_test: Rename dockerlessImageName to remoteImageName Signed-off-by: Simarpreet Singh <[email protected]>
1 parent 3ac5042 commit c63e3aa

File tree

4 files changed

+99
-63
lines changed

4 files changed

+99
-63
lines changed

‎.github/workflows/test.yml

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ jobs:
99
name: Unit Test
1010
runs-on: ubuntu-latest
1111
steps:
12-
1312
- name: Set up Go 1.13
1413
uses: actions/setup-go@v1
1514
with:
@@ -28,7 +27,6 @@ jobs:
2827
name: Integration Test
2928
runs-on: ubuntu-latest
3029
steps:
31-
3230
- name: Set up Go 1.13
3331
uses: actions/setup-go@v1
3432
with:
@@ -41,5 +39,5 @@ jobs:
4139
- name: Install dependencies
4240
run: sudo apt-get install libdb-dev
4341

44-
- name: Run instegration tests
45-
run: make test-integration
42+
- name: Run integration tests
43+
run: make test-integration

‎integration/library_test.go

Lines changed: 97 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"io"
1010
"io/ioutil"
1111
"os"
12+
"sort"
1213
"strings"
1314
"testing"
1415
"time"
@@ -44,6 +45,7 @@ import (
4445
type testCase struct {
4546
name string
4647
imageName string
48+
remoteImageName string
4749
imageFile string
4850
expectedOS types.OS
4951
expectedPkgsFromCmds string
@@ -52,83 +54,109 @@ type testCase struct {
5254

5355
var testCases = []testCase{
5456
{
55-
name: "happy path, alpine:3.10",
56-
imageName: "alpine:3.10",
57-
imageFile: "testdata/fixtures/alpine-310.tar.gz",
58-
expectedOS: types.OS{Name: "3.10.2", Family: "alpine"},
57+
name: "happy path, alpine:3.10",
58+
imageName: "alpine:3.10",
59+
remoteImageName: "knqyf263/alpine:3.10",
60+
imageFile: "testdata/fixtures/alpine-310.tar.gz",
61+
expectedOS: types.OS{Name: "3.10.2", Family: "alpine"},
5962
},
6063
{
61-
name: "happy path, debian:buster",
62-
imageName: "debian:buster",
63-
imageFile: "testdata/fixtures/debian-buster.tar.gz",
64-
expectedOS: types.OS{Name: "10.1", Family: "debian"},
64+
name: "happy path, amazonlinux:2",
65+
imageName: "amazonlinux:2",
66+
remoteImageName: "knqyf263/amazonlinux:2",
67+
imageFile: "testdata/fixtures/amazon-2.tar.gz",
68+
expectedOS: types.OS{Name: "2 (Karoo)", Family: "amazon"},
6569
},
6670
{
67-
name: "happy path, ubuntu:18.04",
68-
imageName: "ubuntu:18.04",
69-
imageFile: "testdata/fixtures/ubuntu-1804.tar.gz",
70-
expectedOS: types.OS{Name: "18.04", Family: "ubuntu"},
71+
name: "happy path, debian:buster",
72+
imageName: "debian:buster",
73+
remoteImageName: "knqyf263/debian:buster",
74+
imageFile: "testdata/fixtures/debian-buster.tar.gz",
75+
expectedOS: types.OS{Name: "10.1", Family: "debian"},
7176
},
7277
{
73-
name: "happy path, centos6",
74-
imageName: "centos:6",
75-
imageFile: "testdata/fixtures/centos-6.tar.gz",
76-
expectedOS: types.OS{Name: "6.10", Family: "centos"},
78+
name: "happy path, photon:1.0",
79+
imageName: "photon:1.0-20190823",
80+
remoteImageName: "knqyf263/photon:1.0-20190823",
81+
imageFile: "testdata/fixtures/photon-10.tar.gz",
82+
expectedOS: types.OS{Name: "1.0", Family: "photon"},
7783
},
7884
{
79-
name: "happy path, registry.redhat.io/ubi7",
80-
imageName: "registry.redhat.io/ubi7",
81-
imageFile: "testdata/fixtures/ubi-7.tar.gz",
82-
expectedOS: types.OS{Name: "7.7", Family: "redhat"},
85+
name: "happy path, registry.redhat.io/ubi7",
86+
imageName: "registry.redhat.io/ubi7",
87+
remoteImageName: "knqyf263/registry.redhat.io-ubi7:latest",
88+
imageFile: "testdata/fixtures/ubi-7.tar.gz",
89+
expectedOS: types.OS{Name: "7.7", Family: "redhat"},
8390
},
8491
{
85-
name: "happy path, amazonlinux:2",
86-
imageName: "amazonlinux:2",
87-
imageFile: "testdata/fixtures/amazon-2.tar.gz",
88-
expectedOS: types.OS{Name: "2 (Karoo)", Family: "amazon"},
89-
},
90-
{
91-
name: "happy path, oracle:8",
92-
imageName: "oraclelinux:8-slim",
93-
imageFile: "testdata/fixtures/oraclelinux-8-slim.tar.gz",
94-
expectedOS: types.OS{Name: "8.0", Family: "oracle"},
95-
},
96-
{
97-
name: "happy path, distroless",
98-
imageName: "gcr.io/distroless/base:latest",
99-
imageFile: "testdata/fixtures/distroless-base.tar.gz",
100-
expectedOS: types.OS{Name: "9.9", Family: "debian"},
101-
},
102-
{
103-
name: "happy path, photon:1.0",
104-
imageName: "photon:1.0-20190823",
105-
imageFile: "testdata/fixtures/photon-10.tar.gz",
106-
expectedOS: types.OS{Name: "1.0", Family: "photon"},
107-
},
108-
{
109-
name: "happy path, opensuse leap 15.1",
110-
imageName: "opensuse/leap:latest",
111-
imageFile: "testdata/fixtures/opensuse-leap-151.tar.gz",
112-
expectedOS: types.OS{Name: "15.1", Family: "opensuse.leap"},
92+
name: "happy path, opensuse leap 15.1",
93+
imageName: "opensuse/leap:latest",
94+
remoteImageName: "knqyf263/opensuse-leap:latest",
95+
imageFile: "testdata/fixtures/opensuse-leap-151.tar.gz",
96+
expectedOS: types.OS{Name: "15.1", Family: "opensuse.leap"},
11397
},
11498
{
11599
name: "happy path, vulnimage with lock files",
116100
imageName: "knqyf263/vuln-image:1.2.3",
101+
remoteImageName: "knqyf263/vuln-image:1.2.3",
117102
imageFile: "testdata/fixtures/vulnimage.tar.gz",
118103
expectedOS: types.OS{Name: "3.7.1", Family: "alpine"},
119-
expectedLibraries: "testdata/goldens/knqyf263vuln-image1.2.3.expectedlibs.golden",
120-
expectedPkgsFromCmds: "testdata/goldens/knqyf263vuln-image1.2.3.expectedpkgsfromcmds.golden",
104+
expectedLibraries: "testdata/goldens/vuln-image1.2.3.expectedlibs.golden",
105+
expectedPkgsFromCmds: "testdata/goldens/vuln-image1.2.3.expectedpkgsfromcmds.golden",
121106
},
122107
}
123108

109+
func TestFanal_Library_DockerLessMode(t *testing.T) {
110+
for _, tc := range testCases {
111+
tc := tc
112+
t.Run(tc.name, func(t *testing.T) {
113+
t.Parallel()
114+
ctx := context.Background()
115+
d, _ := ioutil.TempDir("", "TestFanal_Library_DockerLessMode_*")
116+
defer os.RemoveAll(d)
117+
118+
c, err := cache.NewFSCache(d)
119+
require.NoError(t, err, tc.name)
120+
121+
opt := types.DockerOption{
122+
Timeout: 600 * time.Second,
123+
SkipPing: true,
124+
}
125+
126+
cli, err := client.NewClientWithOpts(client.FromEnv)
127+
require.NoError(t, err, tc.name)
128+
129+
// remove existing Image if any
130+
_, _ = cli.ImageRemove(ctx, tc.remoteImageName, dtypes.ImageRemoveOptions{
131+
Force: true,
132+
PruneChildren: true,
133+
})
134+
135+
ext, cleanup, err := docker.NewDockerExtractor(ctx, tc.remoteImageName, opt)
136+
require.NoError(t, err, tc.name)
137+
defer cleanup()
138+
139+
ac := analyzer.New(ext, c)
140+
applier := analyzer.NewApplier(c)
141+
142+
// run tests twice, one without cache and with cache
143+
for i := 1; i <= 2; i++ {
144+
runChecks(t, ctx, ac, applier, tc)
145+
}
146+
147+
// clear Cache
148+
require.NoError(t, c.Clear(), tc.name)
149+
})
150+
}
151+
}
152+
124153
func TestFanal_Library_DockerMode(t *testing.T) {
125-
t.Parallel()
126154
for _, tc := range testCases {
127-
tc := tc // save a copy of tc for use in t.Run https://gist.github.com/posener/92a55c4cd441fc5e5e85f27bca008721
155+
tc := tc
128156
t.Run(tc.name, func(t *testing.T) {
129157
t.Parallel()
130158
ctx := context.Background()
131-
d, _ := ioutil.TempDir("", "TestFanal_Library_")
159+
d, _ := ioutil.TempDir("", "TestFanal_Library_DockerMode_*")
132160
defer os.RemoveAll(d)
133161
c, err := cache.NewFSCache(d)
134162
require.NoError(t, err)
@@ -186,13 +214,12 @@ func TestFanal_Library_DockerMode(t *testing.T) {
186214
}
187215

188216
func TestFanal_Library_TarMode(t *testing.T) {
189-
t.Parallel()
190217
for _, tc := range testCases {
191-
tc := tc // save a copy of tc for use in t.Run https://gist.github.com/posener/92a55c4cd441fc5e5e85f27bca008721
218+
tc := tc
192219
t.Run(tc.name, func(t *testing.T) {
193220
t.Parallel()
194221
ctx := context.Background()
195-
d, _ := ioutil.TempDir("", "TestFanal_Library_")
222+
d, _ := ioutil.TempDir("", "TestFanal_Library_TarMode_*")
196223
defer os.RemoveAll(d)
197224

198225
c, err := cache.NewFSCache(d)
@@ -236,18 +263,28 @@ func commonChecks(t *testing.T, detail types.ImageDetail, tc testCase) {
236263
func checkPackages(t *testing.T, detail types.ImageDetail, tc testCase) {
237264
r := strings.NewReplacer("/", "-", ":", "-")
238265
goldenFile := fmt.Sprintf("testdata/goldens/packages/%s.json.golden", r.Replace(tc.imageName))
266+
data, err := ioutil.ReadFile(goldenFile)
267+
require.NoError(t, err, tc.name)
239268

240-
data, _ := ioutil.ReadFile(goldenFile)
241269
var expectedPkgs []types.Package
242-
err := json.Unmarshal(data, &expectedPkgs)
270+
err = json.Unmarshal(data, &expectedPkgs)
243271
require.NoError(t, err)
244-
assert.ElementsMatch(t, expectedPkgs, detail.Packages, tc.name)
272+
273+
require.Equal(t, len(expectedPkgs), len(detail.Packages), tc.name)
274+
sort.Slice(expectedPkgs, func(i, j int) bool { return expectedPkgs[i].Name < expectedPkgs[j].Name })
275+
sort.Slice(detail.Packages, func(i, j int) bool { return detail.Packages[i].Name < detail.Packages[j].Name })
276+
277+
for i := 0; i < len(expectedPkgs); i++ {
278+
require.Equal(t, expectedPkgs[i].Name, detail.Packages[i].Name, tc.name)
279+
require.Equal(t, expectedPkgs[i].Version, detail.Packages[i].Version, tc.name)
280+
}
245281
}
246282

247283
func checkLibraries(detail types.ImageDetail, t *testing.T, tc testCase) {
248284
if tc.expectedLibraries != "" {
249285
data, _ := ioutil.ReadFile(tc.expectedLibraries)
250286
var expectedLibraries map[types.FilePath][]godeptypes.Library
287+
251288
json.Unmarshal(data, &expectedLibraries)
252289
require.Equal(t, len(expectedLibraries), len(detail.Applications), tc.name)
253290
} else {
@@ -259,6 +296,7 @@ func checkPackageFromCommands(t *testing.T, detail types.ImageDetail, tc testCas
259296
if tc.expectedPkgsFromCmds != "" {
260297
data, _ := ioutil.ReadFile(tc.expectedPkgsFromCmds)
261298
var expectedPkgsFromCmds []types.Package
299+
262300
json.Unmarshal(data, &expectedPkgsFromCmds)
263301
assert.ElementsMatch(t, expectedPkgsFromCmds, detail.HistoryPackages, tc.name)
264302
} else {

‎integration/testdata/goldens/knqyf263vuln-image1.2.3.expectedlibs.golden renamed to ‎integration/testdata/goldens/vuln-image1.2.3.expectedlibs.golden

File renamed without changes.

‎integration/testdata/goldens/knqyf263vuln-image1.2.3.expectedpkgsfromcmds.golden renamed to ‎integration/testdata/goldens/vuln-image1.2.3.expectedpkgsfromcmds.golden

File renamed without changes.

0 commit comments

Comments
 (0)