refactor: replace genuinetools/reg with containers/image (fanal#70)

* chore(ci): remove unused lines

* feat(cache): add SetBytes

* refactor(cache): replace Initialize with New

* fix(cache): use ReadCloser instead of Reader

* fix(option): update options according to containers/image

* feat(image): add struct to manipulate an image

* refactor(token): move the directory

* chore(Makefile): fix test

* chore(Makefile): add containers_image_storage_stub tag

* refactor(docker): use Image

* refactor(docker): remove unused functions

* refactor(docker): update imports

* test(docker): fix tests

* refactor(analyer): use containers/image

* chore(mod): update dependencies

* fix(extractor): update interface

* fix(main): use updated functions

* test(integration): fix

* refactor(image): remove unused definition

* refactor(error): wrap errors

* test(image): add TestNewImage

* test(mock): prepare interfaces

* test(mock): generate mocks

* test(image): add TestImage_LayerInfos

* test(image): add TestImage_ConfigBlob

* test(image): add TestImage_GetBlob

* chore(mod): update dependencies

* refactor(error): wrap errors

* fix(auth): pass nil when auth is empty

* chore(Makefile): add a tag

* test(bench): fix

* chore(bench): introduce cob

* chore(ci): restrict a push trigger

* chore(bench): run benchmarks 10 times

* test(bench): use a random tag

* test(integration): remove ImageRemove

* chore(cob): set threshold to 0.7

* image_test: Add unhappy paths for GetBlob

Signed-off-by: Simarpreet Singh <[email protected]>

* refactor(image): remove unused fuction

* fix(image): close io.ReadCloser via cleanup function

* test(image): do not skip populateSource

Co-authored-by: Simarpreet Singh <[email protected]>

Author: knqyf263

.github/workflows/test.yml

@@ -1,5 +1,9 @@
 name: Test
-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - master
+  pull_request:
 jobs:
   unittest:
     name: Unit Test
@@ -13,8 +17,7 @@ jobs:
       id: go
 
     - name: Check out code into the Go module directory
-      uses: actions/checkout@v1
-
+      uses: actions/checkout@v1 
     - name: Install dependencies
       run: sudo apt-get install libdb-dev
 
@@ -58,5 +61,8 @@ jobs:
     - name: Install dependencies
       run: sudo apt-get install libdb-dev
 
+    - name: Install cob
+      run: curl -sfL https://raw.githubusercontent.com/knqyf263/cob/master/install.sh | sudo sh -s -- -b /usr/local/bin
+
     - name: Run Benchmark
-      run: make test-performance
+      run: cob --threshold 0.7 --base origin/master --bench-cmd make --bench-args test-performance

Makefile

@@ -4,7 +4,7 @@ deps:
 
 .PHONY: test
 test:
-	go test ./...
+	go test -tags="containers_image_storage_stub" ./...
 
 .PHONY: lint
 lint: devel-deps
@@ -20,8 +20,8 @@ integration/testdata/fixtures/*.tar.gz:
 
 .PHONY: test-integration
 test-integration: integration/testdata/fixtures/*.tar.gz
-	go test -v -tags=integration ./integration/...
+	go test -v -tags="integration containers_image_storage_stub" ./integration/...
 
 .PHONY: test-performance
 test-performance: integration/testdata/fixtures/*.tar.gz
-	go test -v -tags=performance -bench=. ./integration/...
+	go test -v -benchtime=10x -tags="performance containers_image_storage_stub" -bench=. ./integration/...

analyzer/analyzer.go

@@ -1,19 +1,13 @@
 package analyzer
 
 import (
-	"bufio"
-	"compress/gzip"
 	"context"
-	"io"
-	"os"
-
-	"github.com/aquasecurity/fanal/utils"
-
-	"github.com/aquasecurity/fanal/types"
 
 	"golang.org/x/xerrors"
 
 	"github.com/aquasecurity/fanal/extractor"
+	"github.com/aquasecurity/fanal/extractor/image"
+	"github.com/aquasecurity/fanal/types"
 	godeptypes "github.com/aquasecurity/go-dep-parser/pkg/types"
 )
 
@@ -118,37 +112,21 @@ func RequiredFilenames() []string {
 
 // TODO: Remove opts as they're no longer needed
 func (ac Config) Analyze(ctx context.Context, imageName string, opts ...types.DockerOption) (fileMap extractor.FileMap, err error) {
-	r, err := ac.Extractor.SaveLocalImage(ctx, imageName)
-	if err != nil {
-		// when no docker daemon is installed or no image exists in the local machine
-		fileMap, err = ac.Extractor.Extract(ctx, imageName, RequiredFilenames())
-		if err != nil {
-			return nil, xerrors.Errorf("failed to extract files: %w", err)
-		}
-		return fileMap, nil
-	}
-
-	fileMap, err = ac.Extractor.ExtractFromFile(ctx, r, RequiredFilenames())
+	transports := []string{"docker-daemon:", "docker://"}
+	ref := image.Reference{Name: imageName, IsFile: false}
+	fileMap, err = ac.Extractor.Extract(ctx, ref, transports, RequiredFilenames())
 	if err != nil {
-		return nil, xerrors.Errorf("failed to extract files from saved tar: %w", err)
+		return nil, xerrors.Errorf("failed to extract files: %w", err)
 	}
 	return fileMap, nil
 }
 
-func (ac Config) AnalyzeFile(ctx context.Context, f *os.File) (fileMap extractor.FileMap, err error) {
-	var r io.Reader
-	br := bufio.NewReader(f)
-	if utils.IsGzip(br) {
-		r, err = gzip.NewReader(br)
-		if err != nil {
-			return nil, xerrors.Errorf("failed to open gzip: %w", err)
-		}
-	} else {
-		r = br
-	}
-	fileMap, err = ac.Extractor.ExtractFromFile(ctx, r, RequiredFilenames())
+func (ac Config) AnalyzeFile(ctx context.Context, filePath string) (fileMap extractor.FileMap, err error) {
+	transports := []string{"docker-archive:"}
+	ref := image.Reference{Name: filePath, IsFile: true}
+	fileMap, err = ac.Extractor.Extract(ctx, ref, transports, RequiredFilenames())
 	if err != nil {
-		return nil, xerrors.Errorf("failed to extract files from tar: %w", err)
+		return nil, xerrors.Errorf("failed to extract files: %w", err)
 	}
 	return fileMap, nil
 }

analyzer/analyzer_test.go

@@ -4,41 +4,25 @@ import (
 	"context"
 	"errors"
 	"io"
-	"os"
 	"testing"
 
-	"github.com/aquasecurity/fanal/extractor"
-
 	"github.com/stretchr/testify/assert"
+
+	"github.com/aquasecurity/fanal/extractor"
+	"github.com/aquasecurity/fanal/extractor/image"
 )
 
 type mockDockerExtractor struct {
-	saveLocalImage  func(ctx context.Context, imageName string) (io.Reader, error)
-	extractFromFile func(ctx context.Context, r io.Reader, filenames []string) (extractor.FileMap, error)
-	extract         func(ctx context.Context, imageName string, filenames []string) (extractor.FileMap, error)
+	extract func(ctx context.Context, imageRef image.Reference, transports, filenames []string) (extractor.FileMap, error)
 }
 
-func (mde mockDockerExtractor) Extract(ctx context.Context, imageName string, filenames []string) (extractor.FileMap, error) {
+func (mde mockDockerExtractor) Extract(ctx context.Context, imageRef image.Reference, transports, filenames []string) (extractor.FileMap, error) {
 	if mde.extract != nil {
-		return mde.extract(ctx, imageName, filenames)
-	}
-	return extractor.FileMap{}, nil
-}
-
-func (mde mockDockerExtractor) ExtractFromFile(ctx context.Context, r io.Reader, filenames []string) (extractor.FileMap, error) {
-	if mde.extractFromFile != nil {
-		return mde.extractFromFile(ctx, r, filenames)
+		return mde.extract(ctx, imageRef, transports, filenames)
 	}
 	return extractor.FileMap{}, nil
 }
 
-func (mde mockDockerExtractor) SaveLocalImage(ctx context.Context, imageName string) (io.Reader, error) {
-	if mde.saveLocalImage != nil {
-		return mde.saveLocalImage(ctx, imageName)
-	}
-	return nil, nil
-}
-
 func (mde mockDockerExtractor) ExtractFiles(layer io.Reader, filenames []string) (extractor.FileMap, extractor.OPQDirs, error) {
 	panic("implement me")
 }
@@ -55,32 +39,14 @@ func (m mockOSAnalyzer) RequiredFiles() []string {
 
 func TestConfig_Analyze(t *testing.T) {
 	testCases := []struct {
-		name                string
-		saveLocalImageFunc  func(ctx context.Context, imageName string) (io.Reader, error)
-		extractFunc         func(ctx context.Context, imageName string, filenames []string) (extractor.FileMap, error)
-		extractFromFileFunc func(ctx context.Context, r io.Reader, filenames []string) (maps extractor.FileMap, e error)
-		expectedError       error
-		expectedFileMap     extractor.FileMap
+		name            string
+		extractFunc     func(ctx context.Context, imageRef image.Reference, transports, filenames []string) (extractor.FileMap, error)
+		expectedError   error
+		expectedFileMap extractor.FileMap
 	}{
-		{
-			name: "happy path with docker installed and image found",
-			extractFromFileFunc: func(ctx context.Context, r io.Reader, filenames []string) (maps extractor.FileMap, e error) {
-				return extractor.FileMap{
-					"file1": []byte{0x1, 0x2, 0x3},
-					"file2": []byte{0x4, 0x5, 0x6},
-				}, nil
-			},
-			expectedFileMap: extractor.FileMap{
-				"file1": []byte{0x1, 0x2, 0x3},
-				"file2": []byte{0x4, 0x5, 0x6},
-			},
-		},
 		{
 			name: "happy path with no docker installed or no image found",
-			saveLocalImageFunc: func(ctx context.Context, imageName string) (reader io.Reader, e error) {
-				return nil, errors.New("couldn't save local image")
-			},
-			extractFunc: func(ctx context.Context, imageName string, filenames []string) (maps extractor.FileMap, e error) {
+			extractFunc: func(ctx context.Context, imageRef image.Reference, transports, filenames []string) (maps extractor.FileMap, e error) {
 				return extractor.FileMap{
 					"file1": []byte{0x1, 0x2, 0x3},
 					"file2": []byte{0x4, 0x5, 0x6},
@@ -97,9 +63,7 @@ func TestConfig_Analyze(t *testing.T) {
 		RegisterOSAnalyzer(mockOSAnalyzer{})
 
 		ac := Config{Extractor: mockDockerExtractor{
-			extractFromFile: tc.extractFromFileFunc,
-			extract:         tc.extractFunc,
-			saveLocalImage:  tc.saveLocalImageFunc,
+			extract: tc.extractFunc,
 		}}
 		fm, err := ac.Analyze(context.TODO(), "fooimage")
 		assert.Equal(t, tc.expectedError, err, tc.name)
@@ -112,11 +76,11 @@ func TestConfig_Analyze(t *testing.T) {
 
 func TestConfig_AnalyzeFile(t *testing.T) {
 	testCases := []struct {
-		name                string
-		extractFromFileFunc func(ctx context.Context, r io.Reader, filenames []string) (fileMap extractor.FileMap, err error)
-		inputFile           string
-		expectedError       error
-		expectedFileMap     extractor.FileMap
+		name            string
+		extractFunc     func(ctx context.Context, imageReference image.Reference, transports, filenames []string) (extractor.FileMap, error)
+		inputFile       string
+		expectedError   error
+		expectedFileMap extractor.FileMap
 	}{
 		{
 			name:            "happy path, valid tar.gz file",
@@ -130,8 +94,8 @@ func TestConfig_AnalyzeFile(t *testing.T) {
 		},
 		{
 			name:          "sad path, valid file but ExtractFromFile fails",
-			expectedError: errors.New("failed to extract files from tar: extract from file failed"),
-			extractFromFileFunc: func(ctx context.Context, r io.Reader, filenames []string) (fileMap extractor.FileMap, err error) {
+			expectedError: errors.New("failed to extract files: extract from file failed"),
+			extractFunc: func(ctx context.Context, imageRef image.Reference, transports, filenames []string) (fileMap extractor.FileMap, err error) {
 				return nil, errors.New("extract from file failed")
 			},
 		},
@@ -140,13 +104,11 @@ func TestConfig_AnalyzeFile(t *testing.T) {
 	for _, tc := range testCases {
 		ac := Config{
 			Extractor: mockDockerExtractor{
-				extractFromFile: tc.extractFromFileFunc,
+				extract: tc.extractFunc,
 			},
 		}
 
-		f, _ := os.Open(tc.inputFile)
-		defer f.Close()
-		fm, err := ac.AnalyzeFile(context.TODO(), f)
+		fm, err := ac.AnalyzeFile(context.Background(), tc.inputFile)
 		switch {
 		case tc.expectedError != nil:
 			assert.Equal(t, tc.expectedError.Error(), err.Error(), tc.name)

cache/cache.go

@@ -18,20 +18,21 @@ var (
 )
 
 type Cache interface {
-	Get(key string) io.Reader
-	Set(key string, file io.Reader) (io.Reader, error)
-	Clear() error
+	Get(key string) (reader io.ReadCloser)
+	Set(key string, file io.Reader) (reader io.Reader, err error)
+	SetBytes(key string, value []byte) (err error)
+	Clear() (err error)
 }
 
 type FSCache struct {
 	directory string
 }
 
-func Initialize(cacheDir string) Cache {
+func New(cacheDir string) Cache {
 	return &FSCache{directory: filepath.Join(cacheDir, cacheDirName)}
 }
 
-func (fs FSCache) Get(key string) io.Reader {
+func (fs FSCache) Get(key string) io.ReadCloser {
 	filePath := filepath.Join(fs.directory, replacer.Replace(key))
 	f, err := os.Open(filePath)
 	if err != nil {
@@ -40,20 +41,36 @@ func (fs FSCache) Get(key string) io.Reader {
 	return f
 }
 
-func (fs FSCache) Set(key string, file io.Reader) (io.Reader, error) {
+func (fs FSCache) Set(key string, r io.Reader) (io.Reader, error) {
 	filePath := filepath.Join(fs.directory, replacer.Replace(key))
 	if err := os.MkdirAll(fs.directory, os.ModePerm); err != nil {
 		return nil, xerrors.Errorf("failed to mkdir all: %w", err)
 	}
 	cacheFile, err := os.Create(filePath)
 	if err != nil {
-		return file, xerrors.Errorf("failed to create cache file: %w", err)
+		return r, xerrors.Errorf("failed to create cache file: %w", err)
 	}
 
-	tee := io.TeeReader(file, cacheFile)
+	tee := io.TeeReader(r, cacheFile)
 	return tee, nil
 }
 
+func (fs FSCache) SetBytes(key string, b []byte) error {
+	filePath := filepath.Join(fs.directory, replacer.Replace(key))
+	if err := os.MkdirAll(fs.directory, os.ModePerm); err != nil {
+		return xerrors.Errorf("failed to mkdir all: %w", err)
+	}
+	cacheFile, err := os.Create(filePath)
+	if err != nil {
+		return xerrors.Errorf("failed to create cache file: %w", err)
+	}
+
+	if _, err := cacheFile.Write(b); err != nil {
+		return xerrors.Errorf("cache write error: %w", err)
+	}
+	return nil
+}
+
 func (fs FSCache) Clear() error {
 	if err := os.RemoveAll(fs.directory); err != nil {
 		return xerrors.New("failed to remove cache")

cache/cache_test.go

@@ -13,7 +13,7 @@ func TestSetAndGetAndClear(t *testing.T) {
 	tempCacheDir, _ := ioutil.TempDir("", "TestCacheDir-*")
 	f, _ := ioutil.TempFile(tempCacheDir, "foo.bar.baz-*")
 
-	c := Initialize(tempCacheDir)
+	c := New(tempCacheDir)
 
 	// set
 	expectedCacheContents := "foo bar baz"