feat(scan): support --offline-scan option (#1511)

knqyf263 · web-flow · commit 59957d4c6b63 · 2021-12-24T12:20:21.000+02:00
diff --git a/docs/advanced/air-gap.md b/docs/advanced/air-gap.md
@@ -50,11 +50,12 @@ $ rm trivy-offline.db.tgz
 
 In an air-gapped environment it is your responsibility to update the Trivy database on a regular basis, so that the scanner can detect recently-identified vulnerabilities. 
 
-### Run Trivy with --skip-update option
+### Run Trivy with --skip-update and --offline-scan option
 In an air-gapped environment, specify `--skip-update` so that Trivy doesn't attempt to download the latest database file.
+In addition, if you want to scan Java dependencies such as JAR and pom.xml, you need to specify `--offline-scan` since Trivy tries to issue API requests for scanning Java applications by default.
 
 ```
-$ trivy image --skip-update alpine:3.12
+$ trivy image --skip-update --offline-scan alpine:3.12
 ```
 
 ## Air-Gapped Environment for misconfigurations
diff --git a/docs/getting-started/cli/client.md b/docs/getting-started/cli/client.md
@@ -22,6 +22,7 @@ OPTIONS:
    --timeout value             timeout (default: 5m0s) [$TRIVY_TIMEOUT]
    --ignore-policy value       specify the Rego file to evaluate each vulnerability [$TRIVY_IGNORE_POLICY]
    --list-all-pkgs             enabling the option will output all packages regardless of vulnerability (default: false) [$TRIVY_LIST_ALL_PKGS]
+   --offline-scan              do not issue API requests to identify dependencies (default: false) [$TRIVY_OFFLINE_SCAN]
    --token value               for authentication [$TRIVY_TOKEN]
    --token-header value        specify a header name for token (default: "Trivy-Token") [$TRIVY_TOKEN_HEADER]
    --remote value              server address (default: "http://localhost:4954") [$TRIVY_REMOTE]
diff --git a/docs/getting-started/cli/fs.md b/docs/getting-started/cli/fs.md
@@ -25,6 +25,7 @@ OPTIONS:
    --no-progress                                  suppress progress bar (default: false) [$TRIVY_NO_PROGRESS]
    --ignore-policy value                          specify the Rego file to evaluate each vulnerability [$TRIVY_IGNORE_POLICY]
    --list-all-pkgs                                enabling the option will output all packages regardless of vulnerability (default: false) [$TRIVY_LIST_ALL_PKGS]
+   --offline-scan                                 do not issue API requests to identify dependencies (default: false) [$TRIVY_OFFLINE_SCAN]
    --skip-files value                             specify the file paths to skip traversal [$TRIVY_SKIP_FILES]
    --skip-dirs value                              specify the directories where the traversal is skipped [$TRIVY_SKIP_DIRS]
    --config-policy value                          specify paths to the Rego policy files directory, applying config files [$TRIVY_CONFIG_POLICY]
diff --git a/docs/getting-started/cli/image.md b/docs/getting-started/cli/image.md
@@ -27,6 +27,7 @@ OPTIONS:
    --light                     light mode: it's faster, but vulnerability descriptions and references are not displayed (default: false) [$TRIVY_LIGHT]
    --ignore-policy value       specify the Rego file to evaluate each vulnerability [$TRIVY_IGNORE_POLICY]
    --list-all-pkgs             enabling the option will output all packages regardless of vulnerability (default: false) [$TRIVY_LIST_ALL_PKGS]
+   --offline-scan              do not issue API requests to identify dependencies (default: false) [$TRIVY_OFFLINE_SCAN]
    --skip-files value          specify the file path to skip traversal [$TRIVY_SKIP_FILES]
    --skip-dirs value           specify the directory where the traversal is skipped [$TRIVY_SKIP_DIRS]
    --cache-backend value       cache backend (e.g. redis://localhost:6379) (default: "fs") [$TRIVY_CACHE_BACKEND]
diff --git a/docs/getting-started/cli/repo.md b/docs/getting-started/cli/repo.md
@@ -25,6 +25,7 @@ OPTIONS:
    --no-progress               suppress progress bar (default: false) [$TRIVY_NO_PROGRESS]
    --ignore-policy value       specify the Rego file to evaluate each vulnerability [$TRIVY_IGNORE_POLICY]
    --list-all-pkgs             enabling the option will output all packages regardless of vulnerability (default: false) [$TRIVY_LIST_ALL_PKGS]
+   --offline-scan              do not issue API requests to identify dependencies (default: false) [$TRIVY_OFFLINE_SCAN]
    --skip-files value          specify the file path to skip traversal [$TRIVY_SKIP_FILES]
    --skip-dirs value           specify the directory where the traversal is skipped [$TRIVY_SKIP_DIRS]
    --help, -h                  show help (default: false)
diff --git a/docs/getting-started/cli/rootfs.md b/docs/getting-started/cli/rootfs.md
@@ -25,6 +25,7 @@ OPTIONS:
    --no-progress                                  suppress progress bar (default: false) [$TRIVY_NO_PROGRESS]
    --ignore-policy value                          specify the Rego file to evaluate each vulnerability [$TRIVY_IGNORE_POLICY]
    --list-all-pkgs                                enabling the option will output all packages regardless of vulnerability (default: false) [$TRIVY_LIST_ALL_PKGS]
+   --offline-scan                                 do not issue API requests to identify dependencies (default: false) [$TRIVY_OFFLINE_SCAN]
    --skip-files value                             specify the file paths to skip traversal [$TRIVY_SKIP_FILES]
    --skip-dirs value                              specify the directories where the traversal is skipped [$TRIVY_SKIP_DIRS]
    --config-policy value                          specify paths to the Rego policy files directory, applying config files [$TRIVY_CONFIG_POLICY]
diff --git a/docs/getting-started/troubleshooting.md b/docs/getting-started/troubleshooting.md
@@ -39,6 +39,22 @@ https://developer.github.com/v3/#rate-limiting
 $ GITHUB_TOKEN=XXXXXXXXXX trivy alpine:3.10
 ```
 
+### Maven rate limiting
+
+!!! error
+    ``` bash
+    $ trivy image ...
+    ...
+    status 403 Forbidden from http://search.maven.org/solrsearch/select
+    ```
+
+Trivy calls Maven API for better detection of JAR files, but many requests may exceed rate limiting.
+If it happens frequently, try the `--offline-scan` option to stop Trivy from making API requests.
+This option affects only vulnerability scanning. The vulnerability database and builtin policies are downloaded as usual.
+If you want to skip them as well, you can try `--skip-update` and `--skip-policy-update`.
+
+Note that a number of vulnerabilities might be fewer than without the `--offline-scan` option.
+
 ### Running in parallel takes same time as series run
 When running trivy on multiple images simultaneously, it will take same time as running trivy in series.  
 This is because of a limitation of boltdb.
diff --git a/go.mod b/go.mod
@@ -7,8 +7,8 @@ require (
 	github.com/Masterminds/sprig v2.22.0+incompatible
 	github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46
 	github.com/aquasecurity/bolt-fixtures v0.0.0-20200903104109-d34e7f983986
-	github.com/aquasecurity/fanal v0.0.0-20211223181536-672696605858
-	github.com/aquasecurity/go-dep-parser v0.0.0-20211223152202-b497b40cd9d2
+	github.com/aquasecurity/fanal v0.0.0-20211224062610-102e2bce2240
+	github.com/aquasecurity/go-dep-parser v0.0.0-20211224061556-d0e33761a8ab
 	github.com/aquasecurity/go-gem-version v0.0.0-20201115065557-8eed6fe000ce
 	github.com/aquasecurity/go-npm-version v0.0.0-20201110091526-0b796d180798
 	github.com/aquasecurity/go-pep440-version v0.0.0-20210121094942-22b2f8951d46
diff --git a/go.sum b/go.sum
@@ -207,10 +207,10 @@ github.com/aquasecurity/cfsec v0.2.2 h1:hq6MZlg7XFZsrerCv297N4HRlnJM7K6LLd/l/xCz
 github.com/aquasecurity/cfsec v0.2.2/go.mod h1:sUELRJqIPXTOZiHUx7TzyyFFzuk0W22IG6IWAoV8T6U=
 github.com/aquasecurity/defsec v0.0.37 h1:zdZndlKrW257b8VLK1UwfmXiyPuDrNA+wzBilHRk1LA=
 github.com/aquasecurity/defsec v0.0.37/go.mod h1:csaBEcJ3AKy44expnW0dCANEZcS/c1vcJjwBCbnKWBM=
-github.com/aquasecurity/fanal v0.0.0-20211223181536-672696605858 h1:K+OhavtHOe6weJpCvSDDiObrJDBk4hXtcqBBJ0mTzjE=
-github.com/aquasecurity/fanal v0.0.0-20211223181536-672696605858/go.mod h1:cLmcWHV2gIXcwNEOVVVoas/5wSyhIvMHJACbenvGUCg=
-github.com/aquasecurity/go-dep-parser v0.0.0-20211223152202-b497b40cd9d2 h1:B+lL7tKxen+aWygRCv5YRjwq08YokAEHMrTsrujURrc=
-github.com/aquasecurity/go-dep-parser v0.0.0-20211223152202-b497b40cd9d2/go.mod h1:mYbm6nW+oy1o7gGYngbki6y2VPUf6BPt5U7+O9C78sI=
+github.com/aquasecurity/fanal v0.0.0-20211224062610-102e2bce2240 h1:wxeId0nDv3i3Ih98oFZE7Q6OeNY1R+itxOpkmpbaiek=
+github.com/aquasecurity/fanal v0.0.0-20211224062610-102e2bce2240/go.mod h1:Uj+SCSOPxrU4xrxu9fFVvRWimkktPXv/VWzSfMx/dog=
+github.com/aquasecurity/go-dep-parser v0.0.0-20211224061556-d0e33761a8ab h1:/i0NsV3rYRcW0hkcCCrHmppX5rAr3rlWVIGKdeKBThU=
+github.com/aquasecurity/go-dep-parser v0.0.0-20211224061556-d0e33761a8ab/go.mod h1:mYbm6nW+oy1o7gGYngbki6y2VPUf6BPt5U7+O9C78sI=
 github.com/aquasecurity/go-gem-version v0.0.0-20201115065557-8eed6fe000ce h1:QgBRgJvtEOBtUXilDb1MLi1p1MWoyFDXAu5DEUl5nwM=
 github.com/aquasecurity/go-gem-version v0.0.0-20201115065557-8eed6fe000ce/go.mod h1:HXgVzOPvXhVGLJs4ZKO817idqr/xhwsTcj17CLYY74s=
 github.com/aquasecurity/go-npm-version v0.0.0-20201110091526-0b796d180798 h1:eveqE9ivrt30CJ7dOajOfBavhZ4zPqHcZe/4tKp0alc=
diff --git a/pkg/commands/app.go b/pkg/commands/app.go
@@ -225,6 +225,12 @@ var (
 		EnvVars: []string{"TRIVY_SKIP_DIRS"},
 	}
 
+	offlineScan = cli.BoolFlag{
+		Name:    "offline-scan",
+		Usage:   "do not issue API requests to identify dependencies",
+		EnvVars: []string{"TRIVY_OFFLINE_SCAN"},
+	}
+
 	// For misconfigurations
 	configPolicy = cli.StringSliceFlag{
 		Name:    "config-policy",
@@ -309,6 +315,7 @@ var (
 		&ignorePolicy,
 		&listAllPackages,
 		&cacheBackendFlag,
+		&offlineScan,
 		stringSliceFlag(skipFiles),
 		stringSliceFlag(skipDirs),
 	}
@@ -465,6 +472,7 @@ func NewFilesystemCommand() *cli.Command {
 			&noProgressFlag,
 			&ignorePolicy,
 			&listAllPackages,
+			&offlineScan,
 			stringSliceFlag(skipFiles),
 			stringSliceFlag(skipDirs),
 			stringSliceFlag(configPolicy),
@@ -499,6 +507,7 @@ func NewRootfsCommand() *cli.Command {
 			&noProgressFlag,
 			&ignorePolicy,
 			&listAllPackages,
+			&offlineScan,
 			stringSliceFlag(skipFiles),
 			stringSliceFlag(skipDirs),
 			stringSliceFlag(configPolicy),
@@ -536,6 +545,7 @@ func NewRepositoryCommand() *cli.Command {
 			&noProgressFlag,
 			&ignorePolicy,
 			&listAllPackages,
+			&offlineScan,
 			stringSliceFlag(skipFiles),
 			stringSliceFlag(skipDirs),
 		},
@@ -569,6 +579,7 @@ func NewClientCommand() *cli.Command {
 			stringSliceFlag(skipDirs),
 			stringSliceFlag(configPolicy),
 			&listAllPackages,
+			&offlineScan,
 
 			// original flags
 			&token,
diff --git a/pkg/commands/artifact/run.go b/pkg/commands/artifact/run.go
@@ -203,6 +203,7 @@ func scan(ctx context.Context, opt Option, initializeScanner InitializeScanner,
 		DisabledAnalyzers: disabledAnalyzers(opt),
 		SkipFiles:         opt.SkipFiles,
 		SkipDirs:          opt.SkipDirs,
+		Offline:           opt.OfflineScan,
 	}
 
 	s, cleanup, err := initializeScanner(ctx, target, cacheClient, cacheClient, opt.Timeout, artifactOpt, configScannerOptions)
diff --git a/pkg/commands/client/run.go b/pkg/commands/client/run.go
@@ -143,7 +143,7 @@ func initializeScanner(ctx context.Context, opt Option) (scanner.Scanner, func()
 	// ScannerOptions is filled only when config scanning is enabled.
 	var configScannerOptions config.ScannerOption
 	if utils.StringInSlice(types.SecurityCheckConfig, opt.SecurityChecks) {
-		builtinPolicyPaths, err := operation.InitBuiltinPolicies(ctx, false)
+		builtinPolicyPaths, err := operation.InitBuiltinPolicies(ctx, opt.SkipPolicyUpdate)
 		if err != nil {
 			return scanner.Scanner{}, nil, xerrors.Errorf("failed to initialize default policies: %w", err)
 		}
@@ -161,6 +161,7 @@ func initializeScanner(ctx context.Context, opt Option) (scanner.Scanner, func()
 		DisabledAnalyzers: disabledAnalyzers(opt),
 		SkipFiles:         opt.SkipFiles,
 		SkipDirs:          opt.SkipDirs,
+		Offline:           opt.OfflineScan,
 	}
 
 	if opt.Input != "" {
diff --git a/pkg/commands/option/artifact.go b/pkg/commands/option/artifact.go
@@ -15,8 +15,9 @@ type ArtifactOption struct {
 	Timeout    time.Duration
 	ClearCache bool
 
-	SkipDirs  []string
-	SkipFiles []string
+	SkipDirs    []string
+	SkipFiles   []string
+	OfflineScan bool
 
 	// this field is populated in Init()
 	Target string
@@ -25,11 +26,12 @@ type ArtifactOption struct {
 // NewArtifactOption is the factory method to return artifact option
 func NewArtifactOption(c *cli.Context) ArtifactOption {
 	return ArtifactOption{
-		Input:      c.String("input"),
-		Timeout:    c.Duration("timeout"),
-		ClearCache: c.Bool("clear-cache"),
-		SkipFiles:  c.StringSlice("skip-files"),
-		SkipDirs:   c.StringSlice("skip-dirs"),
+		Input:       c.String("input"),
+		Timeout:     c.Duration("timeout"),
+		ClearCache:  c.Bool("clear-cache"),
+		SkipFiles:   c.StringSlice("skip-files"),
+		SkipDirs:    c.StringSlice("skip-dirs"),
+		OfflineScan: c.Bool("offline-scan"),
 	}
 }
 
